water-column-sonar-processing 0.0.6__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/aws/s3_manager.py

@@ -2,8 +2,11 @@ import json
 import os
 from collections.abc import Generator
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from time import sleep
+from typing import Optional
 
 import boto3
+import botocore
 from boto3.s3.transfer import TransferConfig
 from botocore.config import Config
 from botocore.exceptions import ClientError
@@ -25,10 +28,9 @@ class S3Manager:
     #####################################################################
     def __init__(
         self,
-        # TODO: Need to allow passing in of credentials when writing to protected bucket
+        endpoint_url: Optional[str] = None,
     ):
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
+        self.endpoint_url = endpoint_url
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3_client_config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)
         self.s3_transfer_config = TransferConfig(
@@ -46,14 +48,14 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
+            endpoint_url=self.endpoint_url,
         )
         self.s3_resource = boto3.resource(
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
+            endpoint_url=self.endpoint_url,
         )
-        # self.paginator = self.s3_client.get_paginator(operation_name='list_objects_v2')
-        # TODO: create both "s3_client_input" and "s3_client_output" ???
         self.s3_session_noaa_wcsd_zarr_pds = boto3.Session(
             aws_access_key_id=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             aws_secret_access_key=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
@@ -63,38 +65,54 @@ class S3Manager:
             service_name="s3",
             config=self.s3_client_config,
             region_name=self.s3_region,
+            endpoint_url=self.endpoint_url,
         )
         self.s3_resource_noaa_wcsd_zarr_pds = (
             self.s3_session_noaa_wcsd_zarr_pds.resource(
                 service_name="s3",
                 config=self.s3_client_config,
                 region_name=self.s3_region,
+                endpoint_url=self.endpoint_url,
             )
         )
-
-    def get_client(self):
-        return self.s3_session.client(
-            service_name="s3",
-            config=self.__s3_client_config,
-            region_name=self.s3_region,
+        #
+        self.paginator = self.s3_client.get_paginator("list_objects_v2")
+        self.paginator_noaa_wcsd_zarr_pds = (
+            self.s3_client_noaa_wcsd_zarr_pds.get_paginator("list_objects_v2")
         )
 
     #####################################################################
+    # tested
     def create_bucket(
         self,
         bucket_name: str,
     ):
-        self.s3_client.create_bucket(
-            Bucket=bucket_name,
-            # Required when region is different then us-east-1
-            #
-            # TODO: if region is us-east-1, don't include this line somehow
-            # CreateBucketConfiguration={'LocationConstraint': self.__s3_region}
-        )
+        """
+        Note: this function is only really meant to be used for creating test
+        buckets. It allows public read of all objects.
+        """
+        # https://github.com/aodn/aodn_cloud_optimised/blob/e5035495e782783cc8b9e58711d63ed466420350/test_aodn_cloud_optimised/test_schema.py#L7
+        # public_policy = {
+        #     "Version": "2012-10-17",
+        #     "Statement": [
+        #         {
+        #             "Effect": "Allow",
+        #             "Principal": "*",
+        #             "Action": "s3:GetObject",
+        #             "Resource": f"arn:aws:s3:::{bucket_name}/*",
+        #         }
+        #     ],
+        # }
+        response1 = self.s3_client.create_bucket(Bucket=bucket_name, ACL="public-read")
+        print(response1)
+        # response = self.s3_client.put_bucket_policy(
+        #     Bucket=bucket_name, Policy=json.dumps(public_policy)
+        # )
+        # print(response)
 
     #####################################################################
+    # tested
     def list_buckets(self):
-        # client = self.get_client()
         client = self.s3_client
         return client.list_buckets()
 
@@ -103,17 +121,20 @@ class S3Manager:
         self,
         file_name: str,
         key: str,
+        output_bucket_name: str,
     ):
-        self.s3_client_noaa_wcsd_zarr_pds.upload_file(
-            Filename=file_name,
-            Bucket=self.output_bucket_name,
-            Key=key,
+        """
+        Used to upload a single file, e.g. the GeoJSON file to the NODD bucket
+        """
+        self.s3_resource_noaa_wcsd_zarr_pds.Bucket(output_bucket_name).upload_file(
+            Filename=file_name, Key=key
         )
         return key
 
     #####################################################################
     def upload_files_with_thread_pool_executor(
         self,
+        output_bucket_name: str,
         all_files: list,
     ):
         # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
@@ -122,90 +143,118 @@ class S3Manager:
             with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
                 futures = [
                     executor.submit(
-                        self.upload_nodd_file,
+                        self.upload_nodd_file,  # TODO: verify which one is using this
                         all_file[0],  # file_name
                         all_file[1],  # key
+                        output_bucket_name,  # output_bucket_name
                     )
                     for all_file in all_files
                 ]
                 for future in as_completed(futures):
                     result = future.result()
                     if result:
-                        all_uploads.extend(result)
+                        all_uploads.extend([result])
         except Exception as err:
-            print(err)
+            raise RuntimeError(f"Problem, {err}")
+
         print("Done uploading files using threading pool.")
         return all_uploads
 
     #####################################################################
-    def upload_zarr_files_to_bucket(  # noaa-wcsd-model-pds
+    # tested
+    def upload_zarr_store_to_s3(
+        self,
+        output_bucket_name: str,
+        local_directory: str,
+        object_prefix: str,
+        cruise_name: str,
+    ) -> None:
+        print("uploading model store to s3")
+        try:
+            #
+            print("Starting upload with thread pool executor.")
+            # # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
+            all_files = []
+            for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr"):
+                for file in files:
+                    local_path = os.path.join(subdir, file)
+                    # TODO: find a better method for splitting strings here:
+                    # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr/.zattrs'
+                    # s3_key = f"{object_prefix}/{cruise_name}.zarr{local_path.split(f'{cruise_name}.zarr')[-1]}"
+                    s3_key = os.path.join(
+                        object_prefix,
+                        os.path.join(
+                            subdir[subdir.find(f"{cruise_name}.zarr") :], file
+                        ),
+                    )
+                    all_files.append([local_path, s3_key])
+            self.upload_files_with_thread_pool_executor(
+                output_bucket_name=output_bucket_name,
+                all_files=all_files,
+            )
+            print("Done uploading with thread pool executor.")
+        except Exception as err:
+            raise RuntimeError(f"Problem uploading zarr store to s3, {err}")
+
+    #####################################################################
+    # tested
+    def upload_file(
         self,
-        local_directory,
-        remote_directory,
+        filename: str,
+        bucket_name: str,
+        key: str,
     ):
-        # Right now this is just for uploading a model store to s3
-        print("Uploading files to output bucket.")
-        store_name = os.path.basename(local_directory)
-        all_files = []
-        for subdir, dirs, files in os.walk(local_directory):
-            for file in files:
-                local_path = os.path.join(subdir, file)
-                # s3_key = os.path.join(object_prefix, local_path)
-                s3_key = os.path.join(
-                    remote_directory,
-                    store_name,
-                    subdir.split(store_name)[-1].strip("/"),
-                )
-                all_files.append([local_path, s3_key])
+        self.s3_resource.Bucket(bucket_name).upload_file(Filename=filename, Key=key)
 
-        all_uploads = self.upload_files_with_thread_pool_executor(
-            all_files=all_files,
-        )
-        print("Done uploading files to output bucket.")
-        return all_uploads
+    #####################################################################
+    # tested
+    def check_if_object_exists(self, bucket_name, key_name) -> bool:
+        s3_manager2 = S3Manager()
+        s3_manager2.list_objects(bucket_name=bucket_name, prefix=key_name)
+        s3_client_noaa_wcsd_zarr_pds = self.s3_client_noaa_wcsd_zarr_pds
+        try:
+            s3_client_noaa_wcsd_zarr_pds.head_object(Bucket=bucket_name, Key=key_name)
+            return True
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                # The object does not exist.
+                return False
+            elif e.response["Error"]["Code"] == 403:
+                # Unauthorized, including invalid bucket
+                return False
+            else:
+                # Something else has gone wrong.
+                raise
 
     #####################################################################
-    # used: raw-to-model
-    def list_objects(  # noaa-wcsd-pds and noaa-wcsd-model-pds
-        self, bucket_name, prefix
-    ):
+    # tested
+    def list_objects(self, bucket_name, prefix):  # noaa-wcsd-pds and noaa-wcsd-zarr-pds
+        # TODO: this isn't working for geojson detecting objects!!!!!!!
         # analog to "find_children_objects"
         # Returns a list of key strings for each object in bucket defined by prefix
-        s3_client = self.s3_client
+        # s3_client = self.s3_client
         keys = []
-        paginator = s3_client.get_paginator("list_objects_v2")
-        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
+        # paginator = s3_client.get_paginator("list_objects_v2")
+        page_iterator = self.paginator.paginate(Bucket=bucket_name, Prefix=prefix)
         for page in page_iterator:
             if "Contents" in page.keys():
                 keys.extend([k["Key"] for k in page["Contents"]])
         return keys
 
-    def list_nodd_objects(  # These are used by the geometry for uploading data
-        self,
-        prefix,
-    ):
-        # Returns a list of key strings for each object in bucket defined by prefix
-        keys = []
-        paginator = self.s3_client_noaa_wcsd_zarr_pds.get_paginator("list_objects_v2")
-        for page in paginator.paginate(Bucket=self.output_bucket_name, Prefix=prefix):
-            if "Contents" in page.keys():
-                keys.extend([k["Key"] for k in page["Contents"]])
-        return keys
-
     #####################################################################
     # TODO: change name to "directory"
-    def folder_exists_and_not_empty(self, bucket_name: str, path: str) -> bool:
-        if not path.endswith("/"):
-            path = path + "/"
-        s3_client = self.s3_client
-        resp = self.list_objects(
-            bucket_name=bucket_name, prefix=path
-        )  # TODO: this is returning root folder and doesn't include children or hidden folders
-        # resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        return "Contents" in resp
+    # def folder_exists_and_not_empty(self, bucket_name: str, path: str) -> bool:
+    #     if not path.endswith("/"):
+    #         path = path + "/"
+    #     # s3_client = self.s3_client
+    #     resp = self.list_objects(
+    #         bucket_name=bucket_name, prefix=path
+    #     )  # TODO: this is returning root folder and doesn't include children or hidden folders
+    #     # resp = s3_client.list_objects(Bucket=bucket, Prefix=path, Delimiter='/', MaxKeys=1)
+    #     return "Contents" in resp
 
     #####################################################################
-    # used
+    # private
     def __paginate_child_objects(
         self,
         bucket_name: str,
@@ -220,6 +269,8 @@ class S3Manager:
                 objects.extend(page["Contents"])
         return objects
 
+    #####################################################################
+    # tested
     def get_child_objects(
         self,
         bucket_name: str,
@@ -251,13 +302,14 @@ class S3Manager:
         return raw_files
 
     #####################################################################
-    def get_object(  # TODO: Move this to index.py
-        # noaa-wcsd-pds or noaa-wcsd-model-pds
+    # tested
+    def get_object(  # noaa-wcsd-pds or noaa-wcsd-zarr-pds
         self,
         bucket_name,
         key_name,
     ):
         # Meant for getting singular objects from a bucket, used by indexing lambda
+        # can also return byte range potentially.
         print(f"Getting object {key_name} from {bucket_name}")
         try:
             response = self.s3_client.get_object(
@@ -266,81 +318,101 @@ class S3Manager:
             )
             # status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
             # if status == 200:
+            print(f"Done getting object {key_name} from {bucket_name}")
+            return response
         except ClientError as err:
             print(f"Problem was encountered while getting s3 file: {err}")
             raise
-        print(f"Done getting object {key_name} from {bucket_name}")
-        return response
 
     #####################################################################
-    # used raw-to-model
-    def download_file(  # TODO: change to download_object
-        # noaa-wcsd-pds or noaa-wcsd-model-pds
+    # tested
+    def download_file(
         self,
         bucket_name,
         key,
-        file_name,
+        file_name,  # path to where the file will be saved
     ):
-        self.s3_client.download_file(Bucket=bucket_name, Key=key, Filename=file_name)
-        print("downloaded file")
-
-    #####################################################################
-    # not used
-    # def delete_nodd_object(  # noaa-wcsd-model-pds
-    #         self,
-    #         bucket_name,
-    #         key
-    # ):  # -> dict:
-    #     #return self.__s3_client.delete_object(Bucket=bucket_name, Key=key)
-    #     self.s3_client.delete_object(Bucket=bucket_name, Key=key)
+        try:
+            self.s3_client.download_file(
+                Bucket=bucket_name, Key=key, Filename=file_name
+            )
+            # TODO: if bottom file doesn't exist, don't fail downloader
+            print("downloaded file")
+        except Exception as err:
+            raise RuntimeError(f"Problem was encountered while downloading_file, {err}")
 
     #####################################################################
+    # tested
     def delete_nodd_objects(  # nodd-bucket
         self,
+        bucket_name,
         objects: list,
     ):
         try:
-            print(
-                f"Deleting {len(objects)} objects in {self.output_bucket_name} in batches."
-            )
+            print(f"Deleting {len(objects)} objects in {bucket_name} in batches.")
             objects_to_delete = []
             for obj in objects:
                 objects_to_delete.append({"Key": obj["Key"]})
             # Note: request can contain a list of up to 1000 keys
             for batch in chunked(ll=objects_to_delete, n=1000):
+                # An error occurred (SlowDown) when calling the DeleteObjects operation (reached max retries: 4):
+                # Please reduce your request rate.
+                sleep(0.5)
+                #
                 self.s3_client_noaa_wcsd_zarr_pds.delete_objects(
-                    Bucket=self.output_bucket_name, Delete={"Objects": batch}
+                    Bucket=bucket_name, Delete={"Objects": batch}
                 )
-            print(f"Deleted files.")
+            print("Deleted files.")
         except Exception as err:
-            print(f"Problem was encountered while deleting objects: {err}")
+            raise RuntimeError(f"Problem was encountered while deleting objects, {err}")
 
     #####################################################################
-    # not used TODO: remove
+    # tested
+    def delete_nodd_object(  # only used to delete geojson it looks like?! Remove.
+        self,
+        bucket_name,
+        key_name,
+    ):
+        try:
+            print(f"Deleting {key_name} objects in {bucket_name}.")
+            self.s3_client_noaa_wcsd_zarr_pds.delete_object(
+                Bucket=bucket_name, Key=key_name
+            )
+            print("Deleted file.")
+        except Exception as err:
+            raise RuntimeError(f"Problem was encountered while deleting objects, {err}")
+
+    #####################################################################
+    # tested
     def put(self, bucket_name, key, body):  # noaa-wcsd-model-pds
-        self.s3_client.put_object(Bucket=bucket_name, Key=key, Body=body)
+        try:
+            self.s3_client.put_object(
+                Bucket=bucket_name, Key=key, Body=body
+            )  # "Body" can be a file
+        except Exception as err:
+            raise RuntimeError(f"Problem was encountered putting object, {err}")
 
     #####################################################################
+    # tested
     def read_s3_json(
         self,
         ship_name,
         cruise_name,
         sensor_name,
         file_name_stem,
+        output_bucket_name,  # TODO: change to just bucket_name
     ) -> str:
         try:
-            content_object = self.s3_resource_noaa_wcsd_zarr_pds.Object(
-                bucket_name=self.output_bucket_name,
+            resource = self.s3_resource_noaa_wcsd_zarr_pds
+            content_object = resource.Object(
+                bucket_name=output_bucket_name,
                 key=f"spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.json",
             ).get()
             file_content = content_object["Body"].read().decode("utf-8")
             json_content = json.loads(file_content)
             return json_content
-        except Exception as err:  # Failure
-            print(f"Exception encountered reading s3 GeoJSON: {err}")
-            raise
-
-    #####################################################################
+        except Exception as err:
+            raise RuntimeError(f"Exception encountered reading s3 GeoJSON, {err}")
 
 
 #########################################################################

water_column_sonar_processing/aws/s3fs_manager.py

@@ -1,45 +1,29 @@
 import os
+from typing import Optional
 
 import s3fs
 
+
 # TODO: S3FS_LOGGING_LEVEL=DEBUG
+# S3FS_LOGGING_LEVEL=DEBUG
 
 
 class S3FSManager:
     #####################################################################
     def __init__(
         self,
+        endpoint_url: Optional[str] = None,
     ):
-        self.__s3_region = os.environ.get("AWS_REGION", default="us-east-1")
+        self.endpoint_url = endpoint_url
+        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
+        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
+        self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3fs = s3fs.S3FileSystem(
+            endpoint_url=endpoint_url,
             key=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             secret=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),
-            # asynchronous=True
-            # use_ssl=False,
-            # skip_instance_cache=True,
-            # default_block_size='100MB',  # if no specific value is given at all time. The built-in default is 5MB
-            # client_kwargs={
-            #     "region_name": self.__s3_region
-            # }
         )
 
-    #####################################################################
-    def add_file(self, filename):
-        full_path = f"{os.getenv('OUTPUT_BUCKET_NAME')}/testing/{filename}"
-        print(full_path)
-
-        self.s3fs.touch(full_path)
-        ff = self.s3fs.ls(f"{os.getenv('OUTPUT_BUCKET_NAME')}/")
-
-        print(ff)
-
-    #####################################################################
-    def upload_data(self, bucket_name, file_path, prefix):
-        # TODO: this works in theory but use boto3 to upload files
-        s3_path = f"s3://{bucket_name}/{prefix}/"
-        s3_file_system = self.s3fs
-        s3_file_system.put(file_path, s3_path, recursive=True)
-
     #####################################################################
     def s3_map(
         self,
@@ -52,17 +36,29 @@ class S3FSManager:
             root=s3_zarr_store_path, s3=self.s3fs
         )  # create=False, not false because will be writing
 
+    #####################################################################
+    # def add_file(self, filename):
+    #     full_path = f"{os.getenv('OUTPUT_BUCKET_NAME')}/testing/{filename}"
+    #     print(full_path)
+    #
+    #     self.s3fs.touch(full_path)
+    #     ff = self.s3fs.ls(f"{os.getenv('OUTPUT_BUCKET_NAME')}/")
+    #
+    #     print(ff)
+
+    #####################################################################
+    def upload_data(self, bucket_name, file_path, prefix):
+        # TODO: this works in theory but use boto3 to upload files
+        s3_path = f"s3://{bucket_name}/{prefix}/"
+        s3_file_system = self.s3fs
+        s3_file_system.put(file_path, s3_path, recursive=True)
+
     #####################################################################
     def exists(
         self,
-        geo_json_s3_path,
+        s3_path,
     ):
-        s3_file_system = self.s3fs
-        return s3_file_system.exists(path=geo_json_s3_path)
+        # s3_file_system =
+        return self.s3fs.exists(s3_path)
 
     #####################################################################
-    # def put(
-    #         self
-    # ):
-    #     s3_file_system = self.s3fs
-    #     return

water_column_sonar_processing/aws/sqs_manager.py

@@ -35,7 +35,7 @@ class SQSManager:
     #######################################################
     def list_queues(self, queue_name_prefix):
         # Note: SQS control plane is eventually consistent, meaning that it
-        # takes a while to propagate the data accross the systems.
+        # takes a while to propagate the dataset accross the systems.
         response = self.__sqs_client.list_queues(QueueNamePrefix=queue_name_prefix)
         print(response)