water-column-sonar-processing 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

water_column_sonar_processing/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import
+
+from . import aws, cruise, geometry, index, model, utility, process
+from .model import ZarrManager
+from .process import Process
+
+__all__ = [
+    "aws",
+    "cruise",
+    "geometry",
+    "index",
+    "model",
+    "utility",
+    "process",
+    "Process",
+]

water_column_sonar_processing/aws/__init__.py

@@ -1,4 +1,7 @@
-# from .dynamodb_manager import DynamoDBManager
-# from .s3_manager import S3Manager
-# from .s3fs_manager import S3FSManager
-# from .sns_manager import SNSManager
+from .dynamodb_manager import DynamoDBManager
+from .s3_manager import S3Manager
+from .s3fs_manager import S3FSManager
+from .sns_manager import SNSManager
+from .sqs_manager import SQSManager
+
+__all__ = ["DynamoDBManager", "S3Manager", "S3FSManager", "SNSManager", "SQSManager"]

water_column_sonar_processing/aws/dynamodb_manager.py

@@ -1,7 +1,8 @@
 import os
+
 import boto3
 import pandas as pd
-from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
+from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
 
 
 #########################################################################
@@ -9,9 +10,9 @@ class DynamoDBManager:
     #####################################################################
     def __init__(self):
         self.__dynamodb_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=os.environ.get("AWS_REGION", default="us-east-1")
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
+            region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
         self.__dynamodb_resource = self.__dynamodb_session.resource(
             service_name="dynamodb",
@@ -35,10 +36,10 @@ class DynamoDBManager:
 
     #####################################################################
     def create_table(
-            self,
-            table_name,
-            key_schema,
-            attribute_definitions,
+        self,
+        table_name,
+        key_schema,
+        attribute_definitions,
     ):
         self.__dynamodb_client.create_table(
             AttributeDefinitions=attribute_definitions,
@@ -52,98 +53,118 @@ class DynamoDBManager:
         )
 
     #####################################################################
-    def get_item(
-            self,
-            table_name,
-            key
+    def create_water_column_sonar_table(
+        self,
+        table_name,
     ):
+        self.create_table(
+            table_name=table_name,
+            key_schema=[
+                {
+                    "AttributeName": "FILE_NAME",
+                    "KeyType": "HASH",
+                },
+                {
+                    "AttributeName": "CRUISE_NAME",
+                    "KeyType": "RANGE",
+                },
+            ],
+            attribute_definitions=[
+                {"AttributeName": "FILE_NAME", "AttributeType": "S"},
+                {"AttributeName": "CRUISE_NAME", "AttributeType": "S"},
+            ],
+        )
+
+    #####################################################################
+    def get_item(self, table_name, key):
         response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
         item = None
-        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
-            if 'Item' in response:
-                item = response['Item']
+        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+            if "Item" in response:
+                item = response["Item"]
         return item
 
     #####################################################################
     def update_item(
-            self,
-            table_name,
-            key,
-            expression_attribute_names,
-            expression_attribute_values,
-            update_expression
+        self,
+        table_name,
+        key,
+        expression_attribute_names,
+        expression_attribute_values,
+        update_expression,
     ):
         response = self.__dynamodb_client.update_item(
             TableName=table_name,
             Key=key,
             ExpressionAttributeNames=expression_attribute_names,
             ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression
+            UpdateExpression=update_expression,
         )
-        status_code = response['ResponseMetadata']['HTTPStatusCode']
+        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
         # TODO: change to exception
-        assert (status_code == 200), "Problem, unable to update dynamodb table."
+        assert status_code == 200, "Problem, unable to update dynamodb table."
 
     #####################################################################
     def get_table_as_df(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        table_name,
     ):
         expression_attribute_values = {
-            ':cr': {'S': cruise_name},
-            ':se': {'S': sensor_name},
-            ':sh': {'S': ship_name},
+            ":cr": {"S": cruise_name},
+            ":se": {"S": sensor_name},
+            ":sh": {"S": ship_name},
         }
 
-        filter_expression = 'CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh'
+        filter_expression = (
+            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
+        )
         response = self.__dynamodb_client.scan(
             TableName=table_name,
-            Select='ALL_ATTRIBUTES',
+            Select="ALL_ATTRIBUTES",
             ExpressionAttributeValues=expression_attribute_values,
             FilterExpression=filter_expression,
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
-        data = response['Items']
+        data = response["Items"]
 
-        while 'LastEvaluatedKey' in response:
+        while "LastEvaluatedKey" in response:
             response = self.__dynamodb_client.scan(
                 TableName=table_name,
-                Select='ALL_ATTRIBUTES',
+                Select="ALL_ATTRIBUTES",
                 ExpressionAttributeValues=expression_attribute_values,
                 FilterExpression=filter_expression,
-                ExclusiveStartKey=response['LastEvaluatedKey']
+                ExclusiveStartKey=response["LastEvaluatedKey"],
             )
-            data.extend(response['Items'])
+            data.extend(response["Items"])
 
         deserializer = self.type_deserializer
         df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
 
-        return df.sort_values(by='START_TIME', ignore_index=True)
+        return df.sort_values(by="START_TIME", ignore_index=True)
 
     #####################################################################
     # is this used?
     def get_table_item(
-            self,
-            table_name,
-            key,
+        self,
+        table_name,
+        key,
     ):
         # a bit more high level, uses resource to get table item
         table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(
-            Key=key
-        )
+        response = table.get_item(Key=key)
         return response
 
     #####################################################################
     # TODO: add helper method to delete the data
     def delete_cruise(
-            self,
-            table_name,
-            cruise_name,
+        self,
+        table_name,
+        cruise_name,
     ):
         pass
 
+
 #########################################################################

water_column_sonar_processing/aws/s3_manager.py

@@ -1,47 +1,45 @@
 import json
 import os
-import boto3
-# import pandas as pd
 from collections.abc import Generator
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
-# import geopandas
-from botocore.config import Config
+import boto3
 from boto3.s3.transfer import TransferConfig
+from botocore.config import Config
 from botocore.exceptions import ClientError
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import as_completed
 
 MAX_POOL_CONNECTIONS = 64
 MAX_CONCURRENCY = 64
 MAX_WORKERS = 64
-GB = 1024 ** 3
+GB = 1024**3
+
 
 #########################################################################
 def chunked(ll: list, n: int) -> Generator:
     # Yields successively n-sized chunks from ll.
     for i in range(0, len(ll), n):
-        yield ll[i:i + n]
+        yield ll[i : i + n]
 
 
 class S3Manager:
     #####################################################################
     def __init__(
-            self,
-            # TODO: Need to allow passing in of credentials when writing to protected bucket
+        self,
+        # TODO: Need to allow passing in of credentials when writing to protected bucket
     ):
-        self.input_bucket_name = os.environ.get('INPUT_BUCKET_NAME')
-        self.output_bucket_name = os.environ.get('OUTPUT_BUCKET_NAME')
+        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
+        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
         self.s3_transfer_config = TransferConfig(
             max_concurrency=MAX_CONCURRENCY,
             use_threads=True,
             max_bandwidth=None,
-            multipart_threshold=10 * GB
+            multipart_threshold=10 * GB,
         )
         self.s3_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
             region_name=self.s3_region,
         )
         self.s3_client = self.s3_session.client(
@@ -57,8 +55,8 @@ class S3Manager:
         # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
         # TODO: create both "s3_client_input" and "s3_client_output" ???
         self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
-            aws_access_key_id=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            aws_secret_access_key=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
+            aws_access_key_id=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
+            aws_secret_access_key=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
             region_name=self.s3_region,
         )
         self.s3_client_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.client(
@@ -66,15 +64,15 @@ class S3Manager:
             config=self.s3_client_config,
             region_name=self.s3_region,
         )
-        self.s3_resource_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.resource(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
+        self.s3_resource_noaa_wcsd_zarr_pds = (
+            self.s3_session_noaa_wcsd_zarr_pds.resource(
+                service_name="s3",
+                config=self.s3_client_config,
+                region_name=self.s3_region,
+            )
         )
 
-    def get_client(
-            self
-    ):
+    def get_client(self):
         return self.s3_session.client(
             service_name="s3",
             config=self.__s3_client_config,
@@ -83,8 +81,8 @@ class S3Manager:
 
     #####################################################################
     def create_bucket(
-            self,
-            bucket_name: str,
+        self,
+        bucket_name: str,
     ):
         self.s3_client.create_bucket(
             Bucket=bucket_name,
@@ -95,18 +93,16 @@ class S3Manager:
         )
 
     #####################################################################
-    def list_buckets(
-            self
-    ):
+    def list_buckets(self):
         # client = self.get_client()
         client = self.s3_client
         return client.list_buckets()
 
     #####################################################################
     def upload_nodd_file(
-            self,
-            file_name: str,
-            key: str,
+        self,
+        file_name: str,
+        key: str,
     ):
         self.s3_client_noaa_wcsd_zarr_pds.upload_file(
             Filename=file_name,
@@ -117,115 +113,120 @@ class S3Manager:
 
     #####################################################################
     def upload_files_with_thread_pool_executor(
-            self,
-            all_files: list,
+        self,
+        all_files: list,
     ):
         # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
         all_uploads = []
         try:  # TODO: problem with threadpool here, missing child files
             with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-                futures = [executor.submit(
-                    self.upload_nodd_file,
-                    all_file[0],            # file_name
-                    all_file[1]             # key
-                ) for all_file in all_files]
+                futures = [
+                    executor.submit(
+                        self.upload_nodd_file,
+                        all_file[0],  # file_name
+                        all_file[1],  # key
+                    )
+                    for all_file in all_files
+                ]
                 for future in as_completed(futures):
                     result = future.result()
                     if result:
                         all_uploads.extend(result)
         except Exception as err:
             print(err)
-        print('Done uploading files using threading pool.')
+        print("Done uploading files using threading pool.")
         return all_uploads
 
     #####################################################################
     def upload_zarr_files_to_bucket(  # noaa-wcsd-model-pds
-            self,
-            local_directory,
-            remote_directory,
+        self,
+        local_directory,
+        remote_directory,
     ):
         # Right now this is just for uploading a model store to s3
-        print('Uploading files to output bucket.')
+        print("Uploading files to output bucket.")
         store_name = os.path.basename(local_directory)
         all_files = []
         for subdir, dirs, files in os.walk(local_directory):
             for file in files:
                 local_path = os.path.join(subdir, file)
                 # s3_key = os.path.join(object_prefix, local_path)
-                s3_key = os.path.join(remote_directory, store_name, subdir.split(store_name)[-1].strip('/'))
+                s3_key = os.path.join(
+                    remote_directory,
+                    store_name,
+                    subdir.split(store_name)[-1].strip("/"),
+                )
                 all_files.append([local_path, s3_key])
 
         all_uploads = self.upload_files_with_thread_pool_executor(
             all_files=all_files,
         )
-        print('Done uploading files to output bucket.')
+        print("Done uploading files to output bucket.")
         return all_uploads
 
     #####################################################################
     # used: raw-to-model
     def list_objects(  # noaa-wcsd-pds and noaa-wcsd-model-pds
-            self,
-            bucket_name,
-            prefix
+        self, bucket_name, prefix
     ):
         # analog to "find_children_objects"
         # Returns a list of key strings for each object in bucket defined by prefix
         s3_client = self.s3_client
         keys = []
-        paginator = s3_client.get_paginator('list_objects_v2')
+        paginator = s3_client.get_paginator("list_objects_v2")
         page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
         for page in page_iterator:
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
+            if "Contents" in page.keys():
+                keys.extend([k["Key"] for k in page["Contents"]])
         return keys
 
     def list_nodd_objects(  # These are used by the geometry for uploading data
-            self,
-            prefix,
+        self,
+        prefix,
     ):
         # Returns a list of key strings for each object in bucket defined by prefix
         keys = []
-        paginator = self.s3_client_noaa_wcsd_zarr_pds.get_paginator('list_objects_v2')
+        paginator = self.s3_client_noaa_wcsd_zarr_pds.get_paginator("list_objects_v2")
         for page in paginator.paginate(Bucket=self.output_bucket_name, Prefix=prefix):
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
+            if "Contents" in page.keys():
+                keys.extend([k["Key"] for k in page["Contents"]])
         return keys
 
     #####################################################################
     # TODO: change name to "directory"
-    def folder_exists_and_not_empty(
-            self,
-            bucket_name: str,
-            path: str
-    ) -> bool:
-        if not path.endswith('/'):
-            path = path + '/'
+    def folder_exists_and_not_empty(self, bucket_name: str, path: str) -> bool:
+        if not path.endswith("/"):
+            path = path + "/"
         s3_client = self.s3_client
-        resp = self.list_objects(bucket_name=bucket_name, prefix=path)  # TODO: this is returning root folder and doesn't include children or hidden folders
-        #resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        return 'Contents' in resp
+        resp = self.list_objects(
+            bucket_name=bucket_name, prefix=path
+        )  # TODO: this is returning root folder and doesn't include children or hidden folders
+        # resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
+        return "Contents" in resp
 
     #####################################################################
     # used
     def __paginate_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str = None,
+        self,
+        bucket_name: str,
+        sub_prefix: str = None,
     ) -> list:
-        page_iterator = self.s3_client.get_paginator('list_objects_v2').paginate(Bucket=bucket_name, Prefix=sub_prefix)
+        page_iterator = self.s3_client.get_paginator("list_objects_v2").paginate(
+            Bucket=bucket_name, Prefix=sub_prefix
+        )
         objects = []
         for page in page_iterator:
-            if 'Contents' in page.keys():
-                objects.extend(page['Contents'])
+            if "Contents" in page.keys():
+                objects.extend(page["Contents"])
         return objects
 
     def get_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str,
-            file_suffix: str = None,
+        self,
+        bucket_name: str,
+        sub_prefix: str,
+        file_suffix: str = None,
     ) -> list:
-        print('Getting child objects')
+        print("Getting child objects")
         raw_files = []
         try:
             children = self.__paginate_child_objects(
@@ -238,10 +239,10 @@ class S3Manager:
                 for child in children:
                     # Note: Any files with predicate 'NOISE' are to be ignored
                     # see: "Bell_M._Shimada/SH1507" cruise for more details.
-                    if child['Key'].endswith(file_suffix) and not os.path.basename(child['Key']).startswith(
-                        'NOISE'
-                    ):
-                        raw_files.append(child['Key'])
+                    if child["Key"].endswith(file_suffix) and not os.path.basename(
+                        child["Key"]
+                    ).startswith("NOISE"):
+                        raw_files.append(child["Key"])
                 return raw_files
         except ClientError as err:
             print(f"Problem was encountered while getting s3 files: {err}")
@@ -251,10 +252,10 @@ class S3Manager:
 
     #####################################################################
     def get_object(  # TODO: Move this to index.py
-                     # noaa-wcsd-pds or noaa-wcsd-model-pds
-            self,
-            bucket_name,
-            key_name,
+        # noaa-wcsd-pds or noaa-wcsd-model-pds
+        self,
+        bucket_name,
+        key_name,
     ):
         # Meant for getting singular objects from a bucket, used by indexing lambda
         print(f"Getting object {key_name} from {bucket_name}")
@@ -274,18 +275,14 @@ class S3Manager:
     #####################################################################
     # used raw-to-model
     def download_file(  # TODO: change to download_object
-                        # noaa-wcsd-pds or noaa-wcsd-model-pds
-            self,
-            bucket_name,
-            key,
-            file_name,
+        # noaa-wcsd-pds or noaa-wcsd-model-pds
+        self,
+        bucket_name,
+        key,
+        file_name,
     ):
-        self.s3_client.download_file(
-            Bucket=bucket_name,
-            Key=key,
-            Filename=file_name
-        )
-        print('downloaded file')
+        self.s3_client.download_file(Bucket=bucket_name, Key=key, Filename=file_name)
+        print("downloaded file")
 
     #####################################################################
     # not used
@@ -299,19 +296,20 @@ class S3Manager:
 
     #####################################################################
     def delete_nodd_objects(  # nodd-bucket
-            self,
-            objects: list,
+        self,
+        objects: list,
     ):
         try:
-            print(f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches.")
+            print(
+                f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches."
+            )
             objects_to_delete = []
             for obj in objects:
-                objects_to_delete.append({'Key': obj['Key']})
+                objects_to_delete.append({"Key": obj["Key"]})
             # Note: request can contain a list of up to 1000 keys
             for batch in chunked(ll=objects_to_delete, n=1000):
                 self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name,
-                    Delete={'Objects': batch}
+                    Bucket=self.output_bucket_name, Delete={"Objects": batch}
                 )
             print(f"Deleted files.")
         except Exception as err:
@@ -319,38 +317,30 @@ class S3Manager:
 
     #####################################################################
     # not used TODO: remove
-    def put(  # noaa-wcsd-model-pds
-            self,
-            bucket_name,
-            key,
-            body
-    ):
-        self.s3_client.put_object(
-            Bucket=bucket_name,
-            Key=key,
-            Body=body
-        )
+    def put(self, bucket_name, key, body):  # noaa-wcsd-model-pds
+        self.s3_client.put_object(Bucket=bucket_name, Key=key, Body=body)
 
     #####################################################################
     def read_s3_json(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name_stem,
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        file_name_stem,
     ) -> str:
         try:
             content_object = self.s3_resource_noaa_wcsd_zarr_pds.Object(
                 bucket_name=self.output_bucket_name,
-                key=f'spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.json'
+                key=f"spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.json",
             ).get()
-            file_content = content_object['Body'].read().decode('utf-8')
+            file_content = content_object["Body"].read().decode("utf-8")
             json_content = json.loads(file_content)
             return json_content
         except Exception as err:  # Failure
-            print(f'Exception encountered reading s3 GeoJSON: {err}')
+            print(f"Exception encountered reading s3 GeoJSON: {err}")
             raise
 
     #####################################################################
 
+
 #########################################################################