water-column-sonar-processing 0.0.9__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/aws/dynamodb_manager.py

@@ -9,20 +9,20 @@ from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
 class DynamoDBManager:
     #####################################################################
     def __init__(
-            self,
-            # endpoint_url
+        self,
+        # endpoint_url
     ):
         # self.endpoint_url = endpoint_url
-        self.__dynamodb_session = boto3.Session(
+        self.dynamodb_session = boto3.Session(
             aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
             aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
             region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
-        self.__dynamodb_resource = self.__dynamodb_session.resource(
+        self.dynamodb_resource = self.dynamodb_session.resource(
             service_name="dynamodb",
             # endpoint_url=self.endpoint_url
         )
-        self.__dynamodb_client = self.__dynamodb_session.client(
+        self.dynamodb_client = self.dynamodb_session.client(
             service_name="dynamodb",
             # endpoint_url=self.endpoint_url
         )
@@ -46,7 +46,7 @@ class DynamoDBManager:
         self,
         table_name,
     ):
-        self.__dynamodb_client.create_table(
+        self.dynamodb_client.create_table(
             TableName=table_name,
             KeySchema=[
                 {
@@ -62,15 +62,17 @@ class DynamoDBManager:
                 {"AttributeName": "FILE_NAME", "AttributeType": "S"},
                 {"AttributeName": "CRUISE_NAME", "AttributeType": "S"},
             ],
-            BillingMode="PAY_PER_REQUEST"
+            BillingMode="PAY_PER_REQUEST",
             # ProvisionedThroughput={
             #     'ReadCapacityUnits': 1_000,
             #     'WriteCapacityUnits': 1_000
             # }
         )
         # TODO: after creating status is 'CREATING', wait until 'ACTIVE'
-        response = self.__dynamodb_client.describe_table(TableName=table_name)
-        print(response) # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/describe_table.html
+        response = self.dynamodb_client.describe_table(TableName=table_name)
+        print(
+            response
+        )  # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/describe_table.html
         # sleep then response['Table']['TableStatus'] == 'ACTIVE'
 
     #####################################################################
@@ -80,7 +82,7 @@ class DynamoDBManager:
     #         table_name,
     #         key
     # ):
-    #     response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
+    #     response = self.dynamodb_client.get_item(TableName=table_name, Key=key)
     #     item = None
     #     if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
     #         if "Item" in response:
@@ -96,7 +98,7 @@ class DynamoDBManager:
         """
         Gets a single row from the db.
         """
-        table = self.__dynamodb_resource.Table(table_name)
+        table = self.dynamodb_resource.Table(table_name)
         response = table.get_item(Key=key)
         # TODO:
         # if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
@@ -111,58 +113,83 @@ class DynamoDBManager:
         expression_attribute_names,
         expression_attribute_values,
         update_expression,
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression,
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        assert response['ConsumedCapacity']['TableName'] == table_name
-        assert status_code == 200, "Problem, unable to update dynamodb table."
+    ):  # TODO: convert to boolean
+        try:
+            response = self.dynamodb_client.update_item(
+                TableName=table_name,
+                Key=key,
+                ExpressionAttributeNames=expression_attribute_names,
+                ExpressionAttributeValues=expression_attribute_values,
+                UpdateExpression=update_expression,
+            )
+            return response["ResponseMetadata"]["HTTPStatusCode"]  # TODO: should be 200
+            # print(f"HTTPStatusCode: {status_code}")
+            # assert status_code == 200, "Problem, unable to update dynamodb table."
+            # assert response['ConsumedCapacity']['TableName'] == table_name
+        except Exception as err:
+            raise RuntimeError(f"Problem was encountered while updating item, {err}")
 
     #####################################################################
     # TODO: change to "get_cruise_as_df"
     def get_table_as_df(
         self,
-        ship_name,
+        # ship_name,
         cruise_name,
-        sensor_name,
+        # sensor_name, # TODO: need to add this back for EK80
         table_name,
     ) -> pd.DataFrame:
         """
         To be used to initialize a cruise, deletes all entries associated with that cruise
         in the database.
+        #TODO: cruise names isn't good enough, there could be two instrument for a cruise...
         """
-        expression_attribute_values = {
-            ":cr": {"S": cruise_name},
-            ":se": {"S": sensor_name},
-            ":sh": {"S": ship_name},
-        }
-
-        filter_expression = (
-            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
-        )
-        response = self.__dynamodb_client.scan(
+        filter_expression = "CRUISE_NAME = :cr"
+        response = self.dynamodb_client.scan(
             TableName=table_name,
-            Select="ALL_ATTRIBUTES",
-            ExpressionAttributeValues=expression_attribute_values,
+            # Limit=1000,
+            Select="ALL_ATTRIBUTES",  # or 'SPECIFIC_ATTRIBUTES',
+            # ExclusiveStartKey=where to pick up
+            # ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+            # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
             FilterExpression=filter_expression,
+            # ExpressionAttributeNames={
+            #     '#SH': 'SHIP_NAME',
+            #     '#CR': 'CRUISE_NAME',
+            #     '#FN': 'FILE_NAME',
+            # },
+            ExpressionAttributeValues={  # criteria
+                ":cr": {
+                    "S": cruise_name,
+                },
+            },
+            ConsistentRead=True,
+            # ExclusiveStartKey=response["LastEvaluatedKey"],
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
-        if len(response["Items"]) == 0:
-            return pd.DataFrame() # If no results, return empty dataframe
+
+        if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
+            return pd.DataFrame()  # If no results, return empty dataframe
 
         data = response["Items"]
 
-        while "LastEvaluatedKey" in response:
-            response = self.__dynamodb_client.scan(
+        while response.get("LastEvaluatedKey"):  # "LastEvaluatedKey" in response:
+            response = self.dynamodb_client.scan(
                 TableName=table_name,
-                Select="ALL_ATTRIBUTES",
-                ExpressionAttributeValues=expression_attribute_values,
+                ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+                Select="ALL_ATTRIBUTES",  # or 'SPECIFIC_ATTRIBUTES',
                 FilterExpression=filter_expression,
+                # ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+                # ExpressionAttributeNames={ # would need to specify all cols in df
+                #     '#SH': 'SHIP_NAME',
+                #     '#CR': 'CRUISE_NAME',
+                #     '#FN': 'FILE_NAME',
+                # },
+                ExpressionAttributeValues={  # criteria
+                    ":cr": {
+                        "S": cruise_name,
+                    },
+                },
+                ConsistentRead=True,
                 ExclusiveStartKey=response["LastEvaluatedKey"],
             )
             data.extend(response["Items"])
@@ -172,6 +199,66 @@ class DynamoDBManager:
 
         return df.sort_values(by="START_TIME", ignore_index=True)
 
+    #####################################################################
+    # def get_cruise_list(
+    #     self,
+    #     table_name,
+    # ) -> list:
+    #     """
+    #     Experimental, gets all cruise names as list
+    #     """
+    #     filter_expression = "CRUISE_NAME = :cr"
+    #     response = self.dynamodb_client.scan(
+    #         TableName=table_name,
+    #         Select='SPECIFIC_ATTRIBUTES',
+    #         #ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+    #         # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
+    #         FilterExpression=filter_expression,
+    #         # ExpressionAttributeNames={
+    #         #     '#SH': 'SHIP_NAME',
+    #         #     '#CR': 'CRUISE_NAME',
+    #         #     '#FN': 'FILE_NAME',
+    #         # },
+    #         # ExpressionAttributeValues={ # criteria
+    #         #     ':cr': {
+    #         #         'S': cruise_name,
+    #         #     },
+    #         # },
+    #     )
+    #     # Note: table.scan() has 1 MB limit on results so pagination is used
+    #
+    #     if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
+    #         return pd.DataFrame() # If no results, return empty dataframe
+    #
+    #     dataset = response["Items"]
+    #
+    #     while response.get('LastEvaluatedKey'): #"LastEvaluatedKey" in response:
+    #         response = self.dynamodb_client.scan(
+    #             TableName=table_name,
+    #             ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+    #             Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
+    #             FilterExpression=filter_expression,
+    #             #ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+    #             # ExpressionAttributeNames={ # would need to specify all cols in df
+    #             #     '#SH': 'SHIP_NAME',
+    #             #     '#CR': 'CRUISE_NAME',
+    #             #     '#FN': 'FILE_NAME',
+    #             # },
+    #             ExpressionAttributeValues={  # criteria
+    #                 ':cr': {
+    #                     'S': cruise_name,
+    #                 },
+    #             },
+    #             ConsistentRead=True,
+    #             ExclusiveStartKey=response["LastEvaluatedKey"],
+    #         )
+    #         dataset.extend(response["Items"])
+    #
+    #     deserializer = self.type_deserializer
+    #     df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in dataset])
+    #
+    #     return df.sort_values(by="START_TIME", ignore_index=True)
+
     #####################################################################
     # TODO: WIP
     def delete_item(
@@ -183,17 +270,10 @@ class DynamoDBManager:
         """
         Finds all rows associated with a cruise and deletes them.
         """
-        response = self.__dynamodb_client.delete_item(
-            Key={
-                "CRUISE_NAME": {
-                    "S": cruise_name
-                },
-                "FILE_NAME": {
-                    "S": file_name
-                }
-            },
+        response = self.dynamodb_client.delete_item(
+            Key={"CRUISE_NAME": {"S": cruise_name}, "FILE_NAME": {"S": file_name}},
             TableName=table_name,
-            ReturnConsumedCapacity="TOTALS",
+            ReturnConsumedCapacity="TOTAL",
         )
         # TODO: there should be attributes included in response but they are missing
         # if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
@@ -202,18 +282,16 @@ class DynamoDBManager:
 
     #####################################################################
     def describe_table(
-            self,
-            table_name,
+        self,
+        table_name,
     ):
         """
         Get a description of the table. Used to verify that records were added/removed.
         """
-        response = self.__dynamodb_client.describe_table(TableName=table_name)
+        response = self.dynamodb_client.describe_table(TableName=table_name)
         print(response)
         return response
 
-
-
     #####################################################################
     # TODO: from test_raw_to_zarr get enum and use here
     # def __update_processing_status(
@@ -226,7 +304,7 @@ class DynamoDBManager:
     #     print(f"Updating processing status to {pipeline_status}.")
     #     if error_message:
     #         print(f"Error message: {error_message}")
-    #         self.__dynamo.update_item(
+    #         self.dynamo.update_item(
     #             table_name=self.__table_name,
     #             key={
     #                 'FILE_NAME': {'S': file_name},  # Partition Key
@@ -251,7 +329,7 @@ class DynamoDBManager:
     #             }
     #         )
     #     else:
-    #         self.__dynamo.update_item(
+    #         self.dynamo.update_item(
     #             table_name=self.__table_name,
     #             key={
     #                 'FILE_NAME': {'S': file_name},  # Partition Key
@@ -273,4 +351,5 @@ class DynamoDBManager:
     #         )
     #     print("Done updating processing status.")
 
+
 #########################################################################