water-column-sonar-processing 0.0.1__py3-none-any.whl → 25.11.1__py3-none-any.whl

model/aws/s3_manager.py

@@ -1,356 +0,0 @@
-import json
-import os
-import boto3
-import pandas as pd
-from collections.abc import Generator
-
-import geopandas
-from botocore.config import Config
-from boto3.s3.transfer import TransferConfig
-from botocore.exceptions import ClientError
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import as_completed
-
-MAX_POOL_CONNECTIONS = 64
-MAX_CONCURRENCY = 64
-MAX_WORKERS = 64
-GB = 1024 ** 3
-
-#########################################################################
-def chunked(ll: list, n: int) -> Generator:
-    # Yields successively n-sized chunks from ll.
-    for i in range(0, len(ll), n):
-        yield ll[i:i + n]
-
-
-class S3Manager:
-    #####################################################################
-    def __init__(
-            self,
-            # TODO: Need to allow passing in of credentials when writing to protected bucket
-    ):
-        self.input_bucket_name = os.environ.get('INPUT_BUCKET_NAME')
-        self.output_bucket_name = os.environ.get('OUTPUT_BUCKET_NAME')
-        self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
-        self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
-        self.s3_transfer_config = TransferConfig(
-            max_concurrency=MAX_CONCURRENCY,
-            use_threads=True,
-            max_bandwidth=None,
-            multipart_threshold=10 * GB
-        )
-        self.s3_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=self.s3_region,
-        )
-        self.s3_client = self.s3_session.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        self.s3_resource = boto3.resource(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
-        # TODO: create both "s3_client_input" and "s3_client_output" ???
-        self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
-            aws_access_key_id=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            aws_secret_access_key=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
-            region_name=self.s3_region,
-        )
-        self.s3_client_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.client(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-        self.s3_resource_noaa_wcsd_zarr_pds = self.s3_session_noaa_wcsd_zarr_pds.resource(
-            service_name="s3",
-            config=self.s3_client_config,
-            region_name=self.s3_region,
-        )
-
-    def get_client(
-            self
-    ):
-        return self.s3_session.client(
-            service_name="s3",
-            config=self.__s3_client_config,
-            region_name=self.s3_region,
-        )
-
-    #####################################################################
-    def create_bucket(
-            self,
-            bucket_name: str,
-    ):
-        self.s3_client.create_bucket(
-            Bucket=bucket_name,
-            # Required when region is different then us-east-1
-            #
-            # TODO: if region is us-east-1, don't include this line somehow
-            # CreateBucketConfiguration={'LocationConstraint': self.__s3_region}
-        )
-
-    #####################################################################
-    def list_buckets(
-            self
-    ):
-        # client = self.get_client()
-        client = self.s3_client
-        return client.list_buckets()
-
-    #####################################################################
-    def upload_nodd_file(
-            self,
-            file_name: str,
-            key: str,
-    ):
-        self.s3_client_noaa_wcsd_zarr_pds.upload_file(
-            Filename=file_name,
-            Bucket=self.output_bucket_name,
-            Key=key,
-        )
-        return key
-
-    #####################################################################
-    def upload_files_with_thread_pool_executor(
-            self,
-            all_files: list,
-    ):
-        # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
-        all_uploads = []
-        try:  # TODO: problem with threadpool here, missing child files
-            with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-                futures = [executor.submit(
-                    self.upload_nodd_file,
-                    all_file[0],            # file_name
-                    all_file[1]             # key
-                ) for all_file in all_files]
-                for future in as_completed(futures):
-                    result = future.result()
-                    if result:
-                        all_uploads.extend(result)
-        except Exception as err:
-            print(err)
-        print('Done uploading files using threading pool.')
-        return all_uploads
-
-    #####################################################################
-    def upload_zarr_files_to_bucket(  # noaa-wcsd-zarr-pds
-            self,
-            local_directory,
-            remote_directory,
-    ):
-        # Right now this is just for uploading a zarr store to s3
-        print('Uploading files to output bucket.')
-        store_name = os.path.basename(local_directory)
-        all_files = []
-        for subdir, dirs, files in os.walk(local_directory):
-            for file in files:
-                local_path = os.path.join(subdir, file)
-                # s3_key = os.path.join(object_prefix, local_path)
-                s3_key = os.path.join(remote_directory, store_name, subdir.split(store_name)[-1].strip('/'))
-                all_files.append([local_path, s3_key])
-
-        all_uploads = self.upload_files_with_thread_pool_executor(
-            all_files=all_files,
-        )
-        print('Done uploading files to output bucket.')
-        return all_uploads
-
-    #####################################################################
-    # used: raw-to-zarr
-    def list_objects(  # noaa-wcsd-pds and noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            prefix
-    ):
-        # analog to "find_children_objects"
-        # Returns a list of key strings for each object in bucket defined by prefix
-        s3_client = self.s3_client
-        keys = []
-        paginator = s3_client.get_paginator('list_objects_v2')
-        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
-        return keys
-
-    def list_nodd_objects(  # These are used by the geometry_manager for uploading data
-            self,
-            prefix,
-    ):
-        # Returns a list of key strings for each object in bucket defined by prefix
-        keys = []
-        paginator = self.s3_client_noaa_wcsd_zarr_pds.get_paginator('list_objects_v2')
-        for page in paginator.paginate(Bucket=self.output_bucket_name, Prefix=prefix):
-            if 'Contents' in page.keys():
-                keys.extend([k['Key'] for k in page['Contents']])
-        return keys
-
-    #####################################################################
-    # TODO: change name to "directory"
-    def folder_exists_and_not_empty(
-            self,
-            bucket_name: str,
-            path: str
-    ) -> bool:
-        if not path.endswith('/'):
-            path = path + '/'
-        s3_client = self.s3_client
-        resp = self.list_objects(bucket_name=bucket_name, prefix=path)  # TODO: this is returning root folder and doesn't include children or hidden folders
-        #resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        return 'Contents' in resp
-
-    #####################################################################
-    # used
-    def __paginate_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str = None,
-    ) -> list:
-        page_iterator = self.s3_client.get_paginator('list_objects_v2').paginate(Bucket=bucket_name, Prefix=sub_prefix)
-        objects = []
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                objects.extend(page['Contents'])
-        return objects
-
-    def get_child_objects(
-            self,
-            bucket_name: str,
-            sub_prefix: str,
-            file_suffix: str = None,
-    ) -> list:
-        print('Getting child objects')
-        raw_files = []
-        try:
-            children = self.__paginate_child_objects(
-                bucket_name=bucket_name,
-                sub_prefix=sub_prefix,
-            )
-            if file_suffix is None:
-                raw_files = children
-            else:
-                for child in children:
-                    # Note: Any files with predicate 'NOISE' are to be ignored
-                    # see: "Bell_M._Shimada/SH1507" cruise for more details.
-                    if child['Key'].endswith(file_suffix) and not os.path.basename(child['Key']).startswith(
-                        'NOISE'
-                    ):
-                        raw_files.append(child['Key'])
-                return raw_files
-        except ClientError as err:
-            print(f"Problem was encountered while getting s3 files: {err}")
-            raise
-        print(f"Found {len(raw_files)} files.")
-        return raw_files
-
-    #####################################################################
-    def get_object(  # TODO: Move this to index.py
-                     # noaa-wcsd-pds or noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key_name,
-    ):
-        # Meant for getting singular objects from a bucket, used by indexing lambda
-        print(f"Getting object {key_name} from {bucket_name}")
-        try:
-            response = self.s3_client.get_object(
-                Bucket=bucket_name,
-                Key=key_name,
-            )
-            # status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
-            # if status == 200:
-        except ClientError as err:
-            print(f"Problem was encountered while getting s3 file: {err}")
-            raise
-        print(f"Done getting object {key_name} from {bucket_name}")
-        return response
-
-    #####################################################################
-    # used raw-to-zarr
-    def download_file(  # TODO: change to download_object
-                        # noaa-wcsd-pds or noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key,
-            file_name,
-    ):
-        self.s3_client.download_file(
-            Bucket=bucket_name,
-            Key=key,
-            Filename=file_name
-        )
-        print('downloaded file')
-
-    #####################################################################
-    # not used
-    # def delete_nodd_object(  # noaa-wcsd-zarr-pds
-    #         self,
-    #         bucket_name,
-    #         key
-    # ):  # -> dict:
-    #     #return self.__s3_client.delete_object(Bucket=bucket_name, Key=key)
-    #     self.s3_client.delete_object(Bucket=bucket_name, Key=key)
-
-    #####################################################################
-    def delete_nodd_objects(  # nodd-bucket
-            self,
-            objects: list,
-    ):
-        try:
-            print(f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches.")
-            objects_to_delete = []
-            for obj in objects:
-                objects_to_delete.append({'Key': obj['Key']})
-            # Note: request can contain a list of up to 1000 keys
-            for batch in chunked(ll=objects_to_delete, n=1000):
-                self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name,
-                    Delete={'Objects': batch}
-                )
-            print(f"Deleted files.")
-        except Exception as err:
-            print(f"Problem was encountered while deleting objects: {err}")
-
-    #####################################################################
-    # not used TODO: remove
-    def put(  # noaa-wcsd-zarr-pds
-            self,
-            bucket_name,
-            key,
-            body
-    ):
-        self.s3_client.put_object(
-            Bucket=bucket_name,
-            Key=key,
-            Body=body
-        )
-
-    #####################################################################
-    def read_s3_json(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name_stem,
-    ) -> str:
-        try:
-            content_object = self.s3_resource_noaa_wcsd_zarr_pds.Object(
-                bucket_name=self.output_bucket_name,
-                key=f'spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.json'
-            ).get()
-            file_content = content_object['Body'].read().decode('utf-8')
-            json_content = json.loads(file_content)
-            return json_content
-        except Exception as err:  # Failure
-            print(f'Exception encountered reading s3 GeoJSON: {err}')
-            raise
-
-    #####################################################################
-
-#########################################################################

model/aws/s3fs_manager.py

@@ -1,74 +0,0 @@
-import os
-import s3fs
-
-
-# TODO: S3FS_LOGGING_LEVEL=DEBUG
-
-
-class S3FSManager:
-    #####################################################################
-    def __init__(
-            self,
-    ):
-        self.__s3_region = os.environ.get("AWS_REGION", default="us-east-1")
-        self.s3fs = s3fs.S3FileSystem(
-            key=os.environ.get('OUTPUT_BUCKET_ACCESS_KEY'),
-            secret=os.environ.get('OUTPUT_BUCKET_SECRET_ACCESS_KEY'),
-            # asynchronous=True
-            # use_ssl=False,
-            # skip_instance_cache=True,
-            # default_block_size='100MB',  # if no specific value is given at all time. The built-in default is 5MB
-            # client_kwargs={
-            #     "region_name": self.__s3_region
-            # }
-        )
-
-    #####################################################################
-    def add_file(
-            self,
-            filename
-    ):
-        full_path = f"{os.getenv('OUTPUT_BUCKET_NAME')}/testing/{filename}"
-        print(full_path)
-
-        self.s3fs.touch(full_path)
-        ff = self.s3fs.ls(f"{os.getenv('OUTPUT_BUCKET_NAME')}/")
-
-        print(ff)
-
-    #####################################################################
-    def upload_data(
-            self,
-            bucket_name,
-            file_path,
-            prefix
-    ):
-        # TODO: this works in theory but use boto3 to upload files
-        s3_path = f"s3://{bucket_name}/{prefix}/"
-        s3_file_system = self.s3fs
-        s3_file_system.put(file_path, s3_path, recursive=True)
-
-    #####################################################################
-    def s3_map(
-            self,
-            s3_zarr_store_path,  # f's3://{bucket}/{input_zarr_path}'
-    ):
-        # The "s3_zarr_store_path" is defined as f's3://{bucket}/{input_zarr_path}'
-        # create=False, not false because will be writing
-        # return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs, check=True)
-        return s3fs.S3Map(root=s3_zarr_store_path, s3=self.s3fs)  # create=False, not false because will be writing
-
-    #####################################################################
-    def exists(
-            self,
-            geo_json_s3_path,
-    ):
-        s3_file_system = self.s3fs
-        return s3_file_system.exists(path=geo_json_s3_path)
-
-    #####################################################################
-    # def put(
-    #         self
-    # ):
-    #     s3_file_system = self.s3fs
-    #     return

model/cruise/create_empty_zarr_store.py

@@ -1,166 +0,0 @@
-import os
-import numcodecs
-import numpy as np
-from ..utility.cleaner import Cleaner
-from ..aws.dynamodb_manager import DynamoDBManager
-from ..aws.s3_manager import S3Manager
-from ..zarr.zarr_manager import ZarrManager
-
-numcodecs.blosc.use_threads = False
-numcodecs.blosc.set_nthreads(1)
-
-TEMPDIR = "/tmp"
-
-# TODO: when ready switch to version 3 of zarr spec
-# ZARR_V3_EXPERIMENTAL_API = 1
-# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
-
-class CreateEmptyZarrStore:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        self.__overwrite = True
-        # TODO: create output_bucket and input_bucket variables here?
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-
-    #######################################################
-
-    def upload_zarr_store_to_s3(
-            self,
-            local_directory: str,
-            object_prefix: str,
-            cruise_name: str,
-    ) -> None:
-        print('uploading zarr store to s3')
-        s3_manager = S3Manager()
-        #
-        print('Starting upload with thread pool executor.')
-        # # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
-        all_files = []
-        for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr"):
-            for file in files:
-                local_path = os.path.join(subdir, file)
-                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr/.zattrs'
-                s3_key = f'{object_prefix}/{cruise_name}.zarr{local_path.split(f"{cruise_name}.zarr")[-1]}'
-                all_files.append([local_path, s3_key])
-        #
-        # print(all_files)
-        s3_manager.upload_files_with_thread_pool_executor(
-            all_files=all_files,
-        )
-        print('Done uploading with thread pool executor.')
-        # TODO: move to common place
-
-    #######################################################
-    def create_cruise_level_zarr_store(
-            self,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            table_name: str
-    ) -> None:
-        try:
-            # HB0806 - 123, HB0903 - 220
-            dynamo_db_manager = DynamoDBManager()
-
-            df = dynamo_db_manager.get_table_as_df(
-                table_name=table_name,
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name
-            )
-
-            # filter the dataframe just for enums >= LEVEL_1_PROCESSING
-            # df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
-
-            # TODO: VERIFY GEOJSON EXISTS as prerequisite!!!
-
-            print(f"DataFrame shape: {df.shape}")
-            cruise_channels = list(set([i for sublist in df['CHANNELS'].dropna() for i in sublist]))
-            cruise_channels.sort()
-
-            consolidated_zarr_width = np.sum(df['NUM_PING_TIME_DROPNA'].dropna().astype(int))
-
-            # [3] calculate the max/min measurement resolutions for the whole cruise
-            cruise_min_echo_range = float(np.min(df['MIN_ECHO_RANGE'].dropna().astype(float)))
-
-            # [4] calculate the maximum of the max depth values
-            cruise_max_echo_range = float(np.max(df['MAX_ECHO_RANGE'].dropna().astype(float)))
-            print(f"cruise_min_echo_range: {cruise_min_echo_range}, cruise_max_echo_range: {cruise_max_echo_range}")
-
-            # [5] get number of channels
-            cruise_frequencies = [float(i) for i in df['FREQUENCIES'].dropna().values.flatten()[0]]
-            print(cruise_frequencies)
-
-            new_width = int(consolidated_zarr_width)
-            print(f"new_width: {new_width}")
-            #################################################################
-            store_name = f"{cruise_name}.zarr"
-            print(store_name)
-            ################################################################
-            # Delete existing zarr store if it exists
-            s3_manager = S3Manager()
-            zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
-            child_objects = s3_manager.get_child_objects(
-                bucket_name=self.output_bucket_name,
-                sub_prefix=zarr_prefix,
-            )
-            if len(child_objects) > 0:
-                s3_manager.delete_nodd_objects(
-                    objects=child_objects,
-                )
-            ################################################################
-            # Create new zarr store
-            zarr_manager = ZarrManager()
-            new_height = len(zarr_manager.get_depth_values(
-                min_echo_range=cruise_min_echo_range,
-                max_echo_range=cruise_max_echo_range
-            ))
-            print(f"new_height: {new_height}")
-
-            zarr_manager.create_zarr_store(
-                path=TEMPDIR,
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name,
-                frequencies=cruise_frequencies,
-                width=new_width,
-                min_echo_range=cruise_min_echo_range,
-                max_echo_range=cruise_max_echo_range,
-                calibration_status=True,
-            )
-            #################################################################
-            self.upload_zarr_store_to_s3(
-                local_directory=TEMPDIR,
-                object_prefix=zarr_prefix,
-                cruise_name=cruise_name,
-            )
-            # https://noaa-wcsd-zarr-pds.s3.amazonaws.com/index.html
-            #################################################################
-            # Verify count of the files uploaded
-            # count = self.__get_file_count(store_name=store_name)
-            # #
-            # raw_zarr_files = self.__get_s3_files(  # TODO: just need count
-            #     bucket_name=self.__output_bucket,
-            #     sub_prefix=os.path.join(zarr_prefix, store_name),
-            # )
-            # if len(raw_zarr_files) != count:
-            #     print(f'Problem writing {store_name} with proper count {count}.')
-            #     raise Exception("File count doesnt equal number of s3 Zarr store files.")
-            # else:
-            #     print("File counts match.")
-            #################################################################
-            # Success
-            # TODO: update enum in dynamodb
-            #################################################################
-        except Exception as err:
-            print(f"Problem trying to create new cruise zarr store: {err}")
-        finally:
-            cleaner = Cleaner()
-            cleaner.delete_local_files()
-        print("Done creating cruise level zarr store")
-
-
-###########################################################