water-column-sonar-processing 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

water_column_sonar_processing/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import
+
+from . import aws, cruise, geometry, index, model, utility, process
+from .model import ZarrManager
+from .process import Process
+
+__all__ = [
+    "aws",
+    "cruise",
+    "geometry",
+    "index",
+    "model",
+    "utility",
+    "process",
+    "Process",
+]

water_column_sonar_processing/aws/__init__.py

@@ -0,0 +1,7 @@
+from .dynamodb_manager import DynamoDBManager
+from .s3_manager import S3Manager
+from .s3fs_manager import S3FSManager
+from .sns_manager import SNSManager
+from .sqs_manager import SQSManager
+
+__all__ = ["DynamoDBManager", "S3Manager", "S3FSManager", "SNSManager", "SQSManager"]

{aws_manager → water_column_sonar_processing/aws}/dynamodb_manager.py

@@ -1,7 +1,8 @@
 import os
+
 import boto3
 import pandas as pd
-from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
+from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
 
 
 #########################################################################
@@ -9,9 +10,9 @@ class DynamoDBManager:
     #####################################################################
     def __init__(self):
         self.__dynamodb_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=os.environ.get("AWS_REGION", default="us-east-1")
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
+            region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
         self.__dynamodb_resource = self.__dynamodb_session.resource(
             service_name="dynamodb",
@@ -23,7 +24,7 @@ class DynamoDBManager:
         self.type_deserializer = TypeDeserializer()
 
     #####################################################################
-    ### defined in raw-to-zarr_manager, not used
+    ### defined in raw-to-model, not used
     # def put_item(
     #         self,
     #         table_name,
@@ -35,10 +36,10 @@ class DynamoDBManager:
 
     #####################################################################
     def create_table(
-            self,
-            table_name,
-            key_schema,
-            attribute_definitions,
+        self,
+        table_name,
+        key_schema,
+        attribute_definitions,
     ):
         self.__dynamodb_client.create_table(
             AttributeDefinitions=attribute_definitions,
@@ -52,98 +53,118 @@ class DynamoDBManager:
         )
 
     #####################################################################
-    def get_item(
-            self,
-            table_name,
-            key
+    def create_water_column_sonar_table(
+        self,
+        table_name,
     ):
+        self.create_table(
+            table_name=table_name,
+            key_schema=[
+                {
+                    "AttributeName": "FILE_NAME",
+                    "KeyType": "HASH",
+                },
+                {
+                    "AttributeName": "CRUISE_NAME",
+                    "KeyType": "RANGE",
+                },
+            ],
+            attribute_definitions=[
+                {"AttributeName": "FILE_NAME", "AttributeType": "S"},
+                {"AttributeName": "CRUISE_NAME", "AttributeType": "S"},
+            ],
+        )
+
+    #####################################################################
+    def get_item(self, table_name, key):
         response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
         item = None
-        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
-            if 'Item' in response:
-                item = response['Item']
+        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+            if "Item" in response:
+                item = response["Item"]
         return item
 
     #####################################################################
     def update_item(
-            self,
-            table_name,
-            key,
-            expression_attribute_names,
-            expression_attribute_values,
-            update_expression
+        self,
+        table_name,
+        key,
+        expression_attribute_names,
+        expression_attribute_values,
+        update_expression,
     ):
         response = self.__dynamodb_client.update_item(
             TableName=table_name,
             Key=key,
             ExpressionAttributeNames=expression_attribute_names,
             ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression
+            UpdateExpression=update_expression,
         )
-        status_code = response['ResponseMetadata']['HTTPStatusCode']
+        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
         # TODO: change to exception
-        assert (status_code == 200), "Problem, unable to update dynamodb table."
+        assert status_code == 200, "Problem, unable to update dynamodb table."
 
     #####################################################################
     def get_table_as_df(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        table_name,
     ):
         expression_attribute_values = {
-            ':cr': {'S': cruise_name},
-            ':se': {'S': sensor_name},
-            ':sh': {'S': ship_name},
+            ":cr": {"S": cruise_name},
+            ":se": {"S": sensor_name},
+            ":sh": {"S": ship_name},
         }
 
-        filter_expression = 'CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh'
+        filter_expression = (
+            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
+        )
         response = self.__dynamodb_client.scan(
             TableName=table_name,
-            Select='ALL_ATTRIBUTES',
+            Select="ALL_ATTRIBUTES",
             ExpressionAttributeValues=expression_attribute_values,
             FilterExpression=filter_expression,
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
-        data = response['Items']
+        data = response["Items"]
 
-        while 'LastEvaluatedKey' in response:
+        while "LastEvaluatedKey" in response:
             response = self.__dynamodb_client.scan(
                 TableName=table_name,
-                Select='ALL_ATTRIBUTES',
+                Select="ALL_ATTRIBUTES",
                 ExpressionAttributeValues=expression_attribute_values,
                 FilterExpression=filter_expression,
-                ExclusiveStartKey=response['LastEvaluatedKey']
+                ExclusiveStartKey=response["LastEvaluatedKey"],
             )
-            data.extend(response['Items'])
+            data.extend(response["Items"])
 
         deserializer = self.type_deserializer
         df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
 
-        return df.sort_values(by='START_TIME', ignore_index=True)
+        return df.sort_values(by="START_TIME", ignore_index=True)
 
     #####################################################################
     # is this used?
     def get_table_item(
-            self,
-            table_name,
-            key,
+        self,
+        table_name,
+        key,
     ):
         # a bit more high level, uses resource to get table item
         table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(
-            Key=key
-        )
+        response = table.get_item(Key=key)
         return response
 
     #####################################################################
     # TODO: add helper method to delete the data
     def delete_cruise(
-            self,
-            table_name,
-            cruise_name,
+        self,
+        table_name,
+        cruise_name,
     ):
         pass
 
+
 #########################################################################