water-column-sonar-processing 0.0.1__py3-none-any.whl → 26.1.14__py3-none-any.whl

model/aws/dynamodb_manager.py

@@ -1,149 +0,0 @@
-import os
-import boto3
-import pandas as pd
-from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
-
-
-#########################################################################
-class DynamoDBManager:
-    #####################################################################
-    def __init__(self):
-        self.__dynamodb_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=os.environ.get("AWS_REGION", default="us-east-1")
-        )
-        self.__dynamodb_resource = self.__dynamodb_session.resource(
-            service_name="dynamodb",
-        )
-        self.__dynamodb_client = self.__dynamodb_session.client(
-            service_name="dynamodb",
-        )
-        self.type_serializer = TypeSerializer()  # https://stackoverflow.com/a/46738251
-        self.type_deserializer = TypeDeserializer()
-
-    #####################################################################
-    ### defined in raw-to-zarr, not used
-    # def put_item(
-    #         self,
-    #         table_name,
-    #         item
-    # ):
-    #     response = boto3.Session().client(service_name='dynamodb').put_item(TableName=table_name, Item=item)
-    #     status_code = response['ResponseMetadata']['HTTPStatusCode']
-    #     assert (status_code == 200), "Problem, unable to update dynamodb table."
-
-    #####################################################################
-    def create_table(
-            self,
-            table_name,
-            key_schema,
-            attribute_definitions,
-    ):
-        self.__dynamodb_client.create_table(
-            AttributeDefinitions=attribute_definitions,
-            TableName=table_name,
-            KeySchema=key_schema,
-            BillingMode="PAY_PER_REQUEST",  # "PROVISIONED",
-            # ProvisionedThroughput={
-            #     'ReadCapacityUnits': 1_000,
-            #     'WriteCapacityUnits': 1_000
-            # }
-        )
-
-    #####################################################################
-    def get_item(
-            self,
-            table_name,
-            key
-    ):
-        response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
-        item = None
-        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
-            if 'Item' in response:
-                item = response['Item']
-        return item
-
-    #####################################################################
-    def update_item(
-            self,
-            table_name,
-            key,
-            expression_attribute_names,
-            expression_attribute_values,
-            update_expression
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression
-        )
-        status_code = response['ResponseMetadata']['HTTPStatusCode']
-        # TODO: change to exception
-        assert (status_code == 200), "Problem, unable to update dynamodb table."
-
-    #####################################################################
-    def get_table_as_df(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
-    ):
-        expression_attribute_values = {
-            ':cr': {'S': cruise_name},
-            ':se': {'S': sensor_name},
-            ':sh': {'S': ship_name},
-        }
-
-        filter_expression = 'CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh'
-        response = self.__dynamodb_client.scan(
-            TableName=table_name,
-            Select='ALL_ATTRIBUTES',
-            ExpressionAttributeValues=expression_attribute_values,
-            FilterExpression=filter_expression,
-        )
-        # Note: table.scan() has 1 MB limit on results so pagination is used
-        data = response['Items']
-
-        while 'LastEvaluatedKey' in response:
-            response = self.__dynamodb_client.scan(
-                TableName=table_name,
-                Select='ALL_ATTRIBUTES',
-                ExpressionAttributeValues=expression_attribute_values,
-                FilterExpression=filter_expression,
-                ExclusiveStartKey=response['LastEvaluatedKey']
-            )
-            data.extend(response['Items'])
-
-        deserializer = self.type_deserializer
-        df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
-
-        return df.sort_values(by='START_TIME', ignore_index=True)
-
-    #####################################################################
-    # is this used?
-    def get_table_item(
-            self,
-            table_name,
-            key,
-    ):
-        # a bit more high level, uses resource to get table item
-        table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(
-            Key=key
-        )
-        return response
-
-    #####################################################################
-    # TODO: add helper method to delete the data
-    def delete_cruise(
-            self,
-            table_name,
-            cruise_name,
-    ):
-        pass
-
-#########################################################################

model/aws/s3_manager.py

@@ -1,356 +0,0 @@
-import json
-import os
-import boto3
-import pandas as pd
-from collections.abc import Generator
-
-import geopandas
-from botocore.config import Config
-from boto3.s3.transfer import TransferConfig
-from botocore.exceptions import ClientError
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import as_completed
-
-MAX_POOL_CONNECTIONS = 64
-MAX_CONCURRENCY = 64
-MAX_WORKERS = 64
-GB = 1024 ** 3
-
-#########################################################################
-def chunked(ll: list, n: int) -> Generator:
-    # Yields successively n-sized chunks from ll.
-    for i in range(0, len(ll), n):
-        yield ll[i:i + n]
-
-
-class S3Manager:
-    #####################################################################
-    def __init__(
-            self,
-            # TODO: Need to allow passing in of credentials when writing to protected bucket
-    ):
-        self.input_bucket_name = os.environ.get('INPUT_BUCKET_NAME')
-        self.output_bucket_name = os.environ.get('OUTPUT_BUCKET_NAME')
-        self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
-        self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
-        self.s3_transfer_config = TransferConfig(
-            max_concurrency=MAX_CONCURRENCY,
-            use_threads=True,
-            max_bandwidth=None,
-            multipart_threshold=10 * GB
-        )
-        self.s3_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=self.s3_region,
-        )
-        self.s3_client = self.s3_session.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        self.s3_resource = boto3.resource(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
-        # TODO: create both "s3_client_input" and "s3_client_output" ???
-        self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
-            aws_access_key_id=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            aws_secret_access_key=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
-            region_name=self.s3_region,
-        )
-        self.s3_client_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        self.s3_resource_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.resource(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-
-    def get_client(
-            self
-    ):
-        return self.s3_session.client(
-            service_name="s3",
-            config=self.__s3_client_config,
-            region_name=self.s3_region,
-        )
-
-    #####################################################################
-    def create_bucket(
-            self,
-            bucket_name: str,
-    ):
-        self.s3_client.create_bucket(
-            Bucket=bucket_name,
-            # Required when region is different then us-east-1
-            #
-            # TODO: if region is us-east-1, don't include this line somehow
-            # CreateBucketConfiguration={'LocationConstraint': self.__s3_region}
-        )
-
-    #####################################################################
-    def list_buckets(
-            self
-    ):
-        # client = self.get_client()
-        client = self.s3_client
-        return client.list_buckets()
-
-    #####################################################################
-    def upload_nodd_file(
-            self,
-            file_name: str,
-            key: str,
-    ):
-        self.s3_client_noaa_wcsd_zarr_pds.upload_file(
-            Filename=file_name,
-            Bucket=self.output_bucket_name,
-            Key=key,
-        )
-        return key
-
-    #####################################################################
-    def upload_files_with_thread_pool_executor(
-            self,
-            all_files: list,
-    ):
-        # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
-        all_uploads = []
-        try:  # TODO: problem with threadpool here, missing child files
-            with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-                futures = [executor.submit(
-                    self.upload_nodd_file,
-                    all_file[0],            # file_name
-                    all_file[1]             # key
-                ) for all_file in all_files]
-                for future in as_completed(futures):
-                    result = future.result()
-                    if result:
-                        all_uploads.extend(result)
-        except Exception as err:
-            print(err)
-        print('Done uploading files using threading pool.')
-        return all_uploads
-
-    #####################################################################
-    def upload_zarr_files_to_bucket(  # noaa-wcsd-zarr-pds
-            self,
-            local_directory,
-            remote_directory,
-    ):
-        # Right now this is just for uploading a zarr store to s3
-        print('Uploading files to output bucket.')
-        store_name = os.path.basename(local_directory)
-        all_files = []
-        for subdir, dirs, files in os.walk(local_directory):
-            for file in files:
-                local_path = os.path.join(subdir, file)
-                # s3_key = os.path.join(object_prefix, local_path)
-                s3_key = os.path.join(remote_directory, store_name, subdir.split(store_name)[-1].strip('/'))
-                all_files.append([local_path, s3_key])
-
-        all_uploads = self.upload_files_with_thread_pool_executor(
-            all_files=all_files,
-        )
-        print('Done uploading files to output bucket.')
-        return all_uploads
-
-    #####################################################################
-    # used: raw-to-zarr
-    def list_objects(  # noaa-wcsd-pds and noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            prefix
-    ):
-        # analog to "find_children_objects"
-        # Returns a list of key strings for each object in bucket defined by prefix
-        s3_client = self.s3_client
-        keys = []
-        paginator = s3_client.get_paginator('list_objects_v2')
-        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
-        return keys
-
-    def list_nodd_objects(  # These are used by the geometry_manager for uploading data
-            self,
-            prefix,
-    ):
-        # Returns a list of key strings for each object in bucket defined by prefix
-        keys = []
-        paginator = self.s3_client_noaa_wcsd_zarr_pds.get_paginator('list_objects_v2')
-        for page in paginator.paginate(Bucket=self.output_bucket_name, Prefix=prefix):
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
-        return keys
-
-    #####################################################################
-    # TODO: change name to "directory"
-    def folder_exists_and_not_empty(
-            self,
-            bucket_name: str,
-            path: str
-    ) -> bool:
-        if not path.endswith('/'):
-            path = path + '/'
-        s3_client = self.s3_client
-        resp = self.list_objects(bucket_name=bucket_name, prefix=path)  # TODO: this is returning root folder and doesn't include children or hidden folders
-        #resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        return 'Contents' in resp
-
-    #####################################################################
-    # used
-    def __paginate_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str = None,
-    ) -> list:
-        page_iterator = self.s3_client.get_paginator('list_objects_v2').paginate(Bucket=bucket_name, Prefix=sub_prefix)
-        objects = []
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                objects.extend(page['Contents'])
-        return objects
-
-    def get_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str,
-            file_suffix: str = None,
-    ) -> list:
-        print('Getting child objects')
-        raw_files = []
-        try:
-            children = self.__paginate_child_objects(
-                bucket_name=bucket_name,
-                sub_prefix=sub_prefix,
-            )
-            if file_suffix is None:
-                raw_files = children
-            else:
-                for child in children:
-                    # Note: Any files with predicate 'NOISE' are to be ignored
-                    # see: "Bell_M._Shimada/SH1507" cruise for more details.
-                    if child['Key'].endswith(file_suffix) and not os.path.basename(child['Key']).startswith(
-                        'NOISE'
-                    ):
-                        raw_files.append(child['Key'])
-                return raw_files
-        except ClientError as err:
-            print(f"Problem was encountered while getting s3 files: {err}")
-            raise
-        print(f"Found {len(raw_files)} files.")
-        return raw_files
-
-    #####################################################################
-    def get_object(  # TODO: Move this to index.py
-                     # noaa-wcsd-pds or noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key_name,
-    ):
-        # Meant for getting singular objects from a bucket, used by indexing lambda
-        print(f"Getting object {key_name} from {bucket_name}")
-        try:
-            response = self.s3_client.get_object(
-                Bucket=bucket_name,
-                Key=key_name,
-            )
-            # status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
-            # if status == 200:
-        except ClientError as err:
-            print(f"Problem was encountered while getting s3 file: {err}")
-            raise
-        print(f"Done getting object {key_name} from {bucket_name}")
-        return response
-
-    #####################################################################
-    # used raw-to-zarr
-    def download_file(  # TODO: change to download_object
-                        # noaa-wcsd-pds or noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key,
-            file_name,
-    ):
-        self.s3_client.download_file(
-            Bucket=bucket_name,
-            Key=key,
-            Filename=file_name
-        )
-        print('downloaded file')
-
-    #####################################################################
-    # not used
-    # def delete_nodd_object(  # noaa-wcsd-zarr-pds
-    #         self,
-    #         bucket_name,
-    #         key
-    # ):  # -> dict:
-    #     #return self.__s3_client.delete_object(Bucket=bucket_name, Key=key)
-    #     self.s3_client.delete_object(Bucket=bucket_name, Key=key)
-
-    #####################################################################
-    def delete_nodd_objects(  # nodd-bucket
-            self,
-            objects: list,
-    ):
-        try:
-            print(f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches.")
-            objects_to_delete = []
-            for obj in objects:
-                objects_to_delete.append({'Key': obj['Key']})
-            # Note: request can contain a list of up to 1000 keys
-            for batch in chunked(ll=objects_to_delete, n=1000):
-                self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name,
-                    Delete={'Objects': batch}
-                )
-            print(f"Deleted files.")
-        except Exception as err:
-            print(f"Problem was encountered while deleting objects: {err}")
-
-    #####################################################################
-    # not used TODO: remove
-    def put(  # noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key,
-            body
-    ):
-        self.s3_client.put_object(
-            Bucket=bucket_name,
-            Key=key,
-            Body=body
-        )
-
-    #####################################################################
-    def read_s3_json(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name_stem,
-    ) -> str:
-        try:
-            content_object = self.s3_resource_noaa_wcsd_zarr_pds.Object(
-                bucket_name=self.output_bucket_name,
-                key=f'spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.json'
-            ).get()
-            file_content = content_object['Body'].read().decode('utf-8')
-            json_content = json.loads(file_content)
-            return json_content
-        except Exception as err:  # Failure
-            print(f'Exception encountered reading s3 GeoJSON: {err}')
-            raise
-
-    #####################################################################
-
-#########################################################################

model/aws/s3fs_manager.py

@@ -1,74 +0,0 @@
-import os
-import s3fs
-
-
-# TODO: S3FS_LOGGING_LEVEL=DEBUG
-
-
-class S3FSManager:
-    #####################################################################
-    def __init__(
-            self,
-    ):
-        self.__s3_region = os.environ.get("AWS_REGION", default="us-east-1")
-        self.s3fs = s3fs.S3FileSystem(
-            key=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            secret=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
-            # asynchronous=True
-            # use_ssl=False,
-            # skip_instance_cache=True,
-            # default_block_size='100MB',  # if no specific value is given at all time. The built-in default is 5MB
-            # client_kwargs={
-            #     "region_name": self.__s3_region
-            # }
-        )
-
-    #####################################################################
-    def add_file(
-            self,
-            filename
-    ):
-        full_path = f"{os.getenv('OUTPUT_BUCKET_NAME')}/testing/{filename}"
-        print(full_path)
-
-        self.s3fs.touch(full_path)
-        ff = self.s3fs.ls(f"{os.getenv('OUTPUT_BUCKET_NAME')}/")
-
-        print(ff)
-
-    #####################################################################
-    def upload_data(
-            self,
-            bucket_name,
-            file_path,
-            prefix
-    ):
-        # TODO: this works in theory but use boto3 to upload files
-        s3_path = f"s3://{bucket_name}/{prefix}/"
-        s3_file_system = self.s3fs
-        s3_file_system.put(file_path, s3_path, recursive=True)
-
-    #####################################################################
-    def s3_map(
-            self,
-            s3_zarr_store_path,  # f's3://{bucket}/{input_zarr_path}'
-    ):
-        # The "s3_zarr_store_path" is defined as f's3://{bucket}/{input_zarr_path}'
-        # create=False, not false because will be writing
-        # return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs, check=True)
-        return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs)  # create=False, not false because will be writing
-
-    #####################################################################
-    def exists(
-            self,
-            geo_json_s3_path,
-    ):
-        s3_file_system = self.s3fs
-        return s3_file_system.exists(path=geo_json_s3_path)
-
-    #####################################################################
-    # def put(
-    #         self
-    # ):
-    #     s3_file_system = self.s3fs
-    #     return