water-column-sonar-processing 0.0.6__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/__init__.py

@@ -1,8 +1,6 @@
 from __future__ import absolute_import
 
-from . import aws, cruise, geometry, index, model, utility, process
-from .model import ZarrManager
-from .process import Process
+from . import aws, cruise, geometry, index, model, processing, utility
 
 __all__ = [
     "aws",
@@ -10,7 +8,6 @@ __all__ = [
     "geometry",
     "index",
     "model",
+    "processing",
     "utility",
-    "process",
-    "Process",
 ]

water_column_sonar_processing/aws/__init__.py

@@ -1,7 +1,7 @@
 from .dynamodb_manager import DynamoDBManager
-from .s3_manager import S3Manager
+from .s3_manager import S3Manager, chunked
 from .s3fs_manager import S3FSManager
 from .sns_manager import SNSManager
 from .sqs_manager import SQSManager
 
-__all__ = ["DynamoDBManager", "S3Manager", "S3FSManager", "SNSManager", "SQSManager"]
+__all__ = ["DynamoDBManager", "S3Manager", "chunked", "S3FSManager", "SNSManager", "SQSManager"]

water_column_sonar_processing/aws/dynamodb_manager.py

@@ -8,17 +8,23 @@ from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
 #########################################################################
 class DynamoDBManager:
     #####################################################################
-    def __init__(self):
-        self.__dynamodb_session = boto3.Session(
+    def __init__(
+        self,
+        # endpoint_url
+    ):
+        # self.endpoint_url = endpoint_url
+        self.dynamodb_session = boto3.Session(
             aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
             aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
             region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
-        self.__dynamodb_resource = self.__dynamodb_session.resource(
+        self.dynamodb_resource = self.dynamodb_session.resource(
             service_name="dynamodb",
+            # endpoint_url=self.endpoint_url
         )
-        self.__dynamodb_client = self.__dynamodb_session.client(
+        self.dynamodb_client = self.dynamodb_session.client(
             service_name="dynamodb",
+            # endpoint_url=self.endpoint_url
         )
         self.type_serializer = TypeSerializer()  # https://stackoverflow.com/a/46738251
         self.type_deserializer = TypeDeserializer()
@@ -35,31 +41,14 @@ class DynamoDBManager:
     #     assert (status_code == 200), "Problem, unable to update dynamodb table."
 
     #####################################################################
-    def create_table(
-        self,
-        table_name,
-        key_schema,
-        attribute_definitions,
-    ):
-        self.__dynamodb_client.create_table(
-            AttributeDefinitions=attribute_definitions,
-            TableName=table_name,
-            KeySchema=key_schema,
-            BillingMode="PAY_PER_REQUEST",  # "PROVISIONED",
-            # ProvisionedThroughput={
-            #     'ReadCapacityUnits': 1_000,
-            #     'WriteCapacityUnits': 1_000
-            # }
-        )
-
     #####################################################################
     def create_water_column_sonar_table(
         self,
         table_name,
     ):
-        self.create_table(
-            table_name=table_name,
-            key_schema=[
+        self.dynamodb_client.create_table(
+            TableName=table_name,
+            KeySchema=[
                 {
                     "AttributeName": "FILE_NAME",
                     "KeyType": "HASH",
@@ -69,20 +58,52 @@ class DynamoDBManager:
                     "KeyType": "RANGE",
                 },
             ],
-            attribute_definitions=[
+            AttributeDefinitions=[
                 {"AttributeName": "FILE_NAME", "AttributeType": "S"},
                 {"AttributeName": "CRUISE_NAME", "AttributeType": "S"},
             ],
+            BillingMode="PAY_PER_REQUEST",
+            # ProvisionedThroughput={
+            #     'ReadCapacityUnits': 1_000,
+            #     'WriteCapacityUnits': 1_000
+            # }
         )
+        # TODO: after creating status is 'CREATING', wait until 'ACTIVE'
+        response = self.dynamodb_client.describe_table(TableName=table_name)
+        print(
+            response
+        )  # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/describe_table.html
+        # sleep then response['Table']['TableStatus'] == 'ACTIVE'
+
+    #####################################################################
+    # don't think this is used?
+    # def get_item(
+    #         self,
+    #         table_name,
+    #         key
+    # ):
+    #     response = self.dynamodb_client.get_item(TableName=table_name, Key=key)
+    #     item = None
+    #     if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+    #         if "Item" in response:
+    #             item = response["Item"]
+    #     return item
 
     #####################################################################
-    def get_item(self, table_name, key):
-        response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
-        item = None
-        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
-            if "Item" in response:
-                item = response["Item"]
-        return item
+    def get_table_item(
+        self,
+        table_name,
+        key,
+    ):
+        """
+        Gets a single row from the db.
+        """
+        table = self.dynamodb_resource.Table(table_name)
+        response = table.get_item(Key=key)
+        # TODO:
+        # if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
+        #     throw error
+        return response
 
     #####################################################################
     def update_item(
@@ -92,50 +113,83 @@ class DynamoDBManager:
         expression_attribute_names,
         expression_attribute_values,
         update_expression,
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression,
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        # TODO: change to exception
-        assert status_code == 200, "Problem, unable to update dynamodb table."
+    ):  # TODO: convert to boolean
+        try:
+            response = self.dynamodb_client.update_item(
+                TableName=table_name,
+                Key=key,
+                ExpressionAttributeNames=expression_attribute_names,
+                ExpressionAttributeValues=expression_attribute_values,
+                UpdateExpression=update_expression,
+            )
+            return response["ResponseMetadata"]["HTTPStatusCode"]  # TODO: should be 200
+            # print(f"HTTPStatusCode: {status_code}")
+            # assert status_code == 200, "Problem, unable to update dynamodb table."
+            # assert response['ConsumedCapacity']['TableName'] == table_name
+        except Exception as err:
+            raise RuntimeError(f"Problem was encountered while updating item, {err}")
 
     #####################################################################
+    # TODO: change to "get_cruise_as_df"
     def get_table_as_df(
         self,
-        ship_name,
+        # ship_name,
         cruise_name,
-        sensor_name,
+        # sensor_name, # TODO: need to add this back for EK80
         table_name,
-    ):
-        expression_attribute_values = {
-            ":cr": {"S": cruise_name},
-            ":se": {"S": sensor_name},
-            ":sh": {"S": ship_name},
-        }
-
-        filter_expression = (
-            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
-        )
-        response = self.__dynamodb_client.scan(
+    ) -> pd.DataFrame:
+        """
+        To be used to initialize a cruise, deletes all entries associated with that cruise
+        in the database.
+        #TODO: cruise names isn't good enough, there could be two instrument for a cruise...
+        """
+        filter_expression = "CRUISE_NAME = :cr"
+        response = self.dynamodb_client.scan(
             TableName=table_name,
-            Select="ALL_ATTRIBUTES",
-            ExpressionAttributeValues=expression_attribute_values,
+            # Limit=1000,
+            Select="ALL_ATTRIBUTES",  # or 'SPECIFIC_ATTRIBUTES',
+            # ExclusiveStartKey=where to pick up
+            # ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+            # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
             FilterExpression=filter_expression,
+            # ExpressionAttributeNames={
+            #     '#SH': 'SHIP_NAME',
+            #     '#CR': 'CRUISE_NAME',
+            #     '#FN': 'FILE_NAME',
+            # },
+            ExpressionAttributeValues={  # criteria
+                ":cr": {
+                    "S": cruise_name,
+                },
+            },
+            ConsistentRead=True,
+            # ExclusiveStartKey=response["LastEvaluatedKey"],
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
+
+        if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
+            return pd.DataFrame()  # If no results, return empty dataframe
+
         data = response["Items"]
 
-        while "LastEvaluatedKey" in response:
-            response = self.__dynamodb_client.scan(
+        while response.get("LastEvaluatedKey"):  # "LastEvaluatedKey" in response:
+            response = self.dynamodb_client.scan(
                 TableName=table_name,
-                Select="ALL_ATTRIBUTES",
-                ExpressionAttributeValues=expression_attribute_values,
+                ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+                Select="ALL_ATTRIBUTES",  # or 'SPECIFIC_ATTRIBUTES',
                 FilterExpression=filter_expression,
+                # ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+                # ExpressionAttributeNames={ # would need to specify all cols in df
+                #     '#SH': 'SHIP_NAME',
+                #     '#CR': 'CRUISE_NAME',
+                #     '#FN': 'FILE_NAME',
+                # },
+                ExpressionAttributeValues={  # criteria
+                    ":cr": {
+                        "S": cruise_name,
+                    },
+                },
+                ConsistentRead=True,
                 ExclusiveStartKey=response["LastEvaluatedKey"],
             )
             data.extend(response["Items"])
@@ -146,25 +200,156 @@ class DynamoDBManager:
         return df.sort_values(by="START_TIME", ignore_index=True)
 
     #####################################################################
-    # is this used?
-    def get_table_item(
+    # def get_cruise_list(
+    #     self,
+    #     table_name,
+    # ) -> list:
+    #     """
+    #     Experimental, gets all cruise names as list
+    #     """
+    #     filter_expression = "CRUISE_NAME = :cr"
+    #     response = self.dynamodb_client.scan(
+    #         TableName=table_name,
+    #         Select='SPECIFIC_ATTRIBUTES',
+    #         #ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+    #         # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
+    #         FilterExpression=filter_expression,
+    #         # ExpressionAttributeNames={
+    #         #     '#SH': 'SHIP_NAME',
+    #         #     '#CR': 'CRUISE_NAME',
+    #         #     '#FN': 'FILE_NAME',
+    #         # },
+    #         # ExpressionAttributeValues={ # criteria
+    #         #     ':cr': {
+    #         #         'S': cruise_name,
+    #         #     },
+    #         # },
+    #     )
+    #     # Note: table.scan() has 1 MB limit on results so pagination is used
+    #
+    #     if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
+    #         return pd.DataFrame() # If no results, return empty dataframe
+    #
+    #     dataset = response["Items"]
+    #
+    #     while response.get('LastEvaluatedKey'): #"LastEvaluatedKey" in response:
+    #         response = self.dynamodb_client.scan(
+    #             TableName=table_name,
+    #             ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+    #             Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
+    #             FilterExpression=filter_expression,
+    #             #ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+    #             # ExpressionAttributeNames={ # would need to specify all cols in df
+    #             #     '#SH': 'SHIP_NAME',
+    #             #     '#CR': 'CRUISE_NAME',
+    #             #     '#FN': 'FILE_NAME',
+    #             # },
+    #             ExpressionAttributeValues={  # criteria
+    #                 ':cr': {
+    #                     'S': cruise_name,
+    #                 },
+    #             },
+    #             ConsistentRead=True,
+    #             ExclusiveStartKey=response["LastEvaluatedKey"],
+    #         )
+    #         dataset.extend(response["Items"])
+    #
+    #     deserializer = self.type_deserializer
+    #     df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in dataset])
+    #
+    #     return df.sort_values(by="START_TIME", ignore_index=True)
+
+    #####################################################################
+    # TODO: WIP
+    def delete_item(
         self,
         table_name,
-        key,
+        cruise_name,
+        file_name,
     ):
-        # a bit more high level, uses resource to get table item
-        table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(Key=key)
+        """
+        Finds all rows associated with a cruise and deletes them.
+        """
+        response = self.dynamodb_client.delete_item(
+            Key={"CRUISE_NAME": {"S": cruise_name}, "FILE_NAME": {"S": file_name}},
+            TableName=table_name,
+            ReturnConsumedCapacity="TOTAL",
+        )
+        # TODO: there should be attributes included in response but they are missing
+        # if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
+        #     throw error
         return response
 
     #####################################################################
-    # TODO: add helper method to delete the data
-    def delete_cruise(
+    def describe_table(
         self,
         table_name,
-        cruise_name,
     ):
-        pass
+        """
+        Get a description of the table. Used to verify that records were added/removed.
+        """
+        response = self.dynamodb_client.describe_table(TableName=table_name)
+        print(response)
+        return response
+
+    #####################################################################
+    # TODO: from test_raw_to_zarr get enum and use here
+    # def __update_processing_status(
+    #         self,
+    #         file_name: str,
+    #         cruise_name: str,
+    #         pipeline_status: str,
+    #         error_message: str = None,
+    # ):
+    #     print(f"Updating processing status to {pipeline_status}.")
+    #     if error_message:
+    #         print(f"Error message: {error_message}")
+    #         self.dynamo.update_item(
+    #             table_name=self.__table_name,
+    #             key={
+    #                 'FILE_NAME': {'S': file_name},  # Partition Key
+    #                 'CRUISE_NAME': {'S': cruise_name},  # Sort Key
+    #             },
+    #             attribute_names={
+    #                 '#PT': 'PIPELINE_TIME',
+    #                 '#PS': 'PIPELINE_STATUS',
+    #                 '#EM': 'ERROR_MESSAGE',
+    #             },
+    #             expression='SET #PT = :pt, #PS = :ps, #EM = :em',
+    #             attribute_values={
+    #                 ':pt': {
+    #                     'S': datetime.now().isoformat(timespec="seconds") + "Z"
+    #                 },
+    #                 ':ps': {
+    #                     'S': pipeline_status
+    #                 },
+    #                 ':em': {
+    #                     'S': error_message
+    #                 }
+    #             }
+    #         )
+    #     else:
+    #         self.dynamo.update_item(
+    #             table_name=self.__table_name,
+    #             key={
+    #                 'FILE_NAME': {'S': file_name},  # Partition Key
+    #                 'CRUISE_NAME': {'S': cruise_name},  # Sort Key
+    #             },
+    #             attribute_names={
+    #                 '#PT': 'PIPELINE_TIME',
+    #                 '#PS': 'PIPELINE_STATUS',
+    #             },
+    #             expression='SET #PT = :pt, #PS = :ps',
+    #             attribute_values={
+    #                 ':pt': {
+    #                     'S': datetime.now().isoformat(timespec="seconds") + "Z"
+    #                 },
+    #                 ':ps': {
+    #                     'S': pipeline_status
+    #                 }
+    #             }
+    #         )
+    #     print("Done updating processing status.")
 
 
 #########################################################################