water-column-sonar-processing 0.0.9__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/index/index_manager.py

@@ -1,19 +1,27 @@
 import os
 import re
-import pandas as pd
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import as_completed
+
+# import networkx as nx
+import pandas as pd
+
 from water_column_sonar_processing.aws import S3Manager
 
+MAX_POOL_CONNECTIONS = 64
+MAX_CONCURRENCY = 64
+MAX_WORKERS = 64
+GB = 1024**3
+
 
 class IndexManager:
+    # TODO: index into dynamodb instead of csv files
 
     def __init__(self, input_bucket_name, calibration_bucket, calibration_key):
         self.input_bucket_name = input_bucket_name
         self.calibration_bucket = calibration_bucket
-        self.calibration_key = calibration_key
-        self.s3_manager = S3Manager()
+        self.calibration_key = calibration_key  # TODO: make optional?
+        self.s3_manager = S3Manager()  # TODO: make anonymous?
 
     #################################################################
     def list_ships(
@@ -50,6 +58,9 @@ class IndexManager:
         self,
         cruise_prefixes,
     ):
+        """
+        This returns a list of ek60 prefixed cruises.
+        """
         cruise_sensors = []  # includes all sensor types
         for cruise_prefix in cruise_prefixes:
             page_iterator = self.s3_manager.paginator.paginate(
@@ -67,6 +78,7 @@ class IndexManager:
         cruise_name,
         sensor_name,
     ):
+        # Gets all raw files for a cruise under the given prefix
         prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
         page_iterator = self.s3_manager.paginator.paginate(
             Bucket=self.input_bucket_name, Prefix=prefix, Delimiter="/"
@@ -77,6 +89,61 @@ class IndexManager:
                 all_files.extend([i["Key"] for i in page["Contents"]])
         return [i for i in all_files if i.endswith(".raw")]
 
+    def get_first_raw_file(
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+    ):
+        # Same as above but only needs to get the first raw file
+        # because we are only interested in the first datagram of one file
+        # TODO: "dataset?"
+        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
+        # page_iterator = self.s3_manager.paginator.paginate(
+        #     Bucket=self.input_bucket_name,
+        #     Prefix=prefix,
+        #     Delimiter="/",
+        #     PaginationConfig={ 'MaxItems': 5 }
+        # ) # TODO: this can create a problem if there is a non raw file returned first
+        ### filter with JMESPath expressions ###
+        page_iterator = self.s3_manager.paginator.paginate(
+            Bucket=self.input_bucket_name,
+            Prefix=prefix,
+            Delimiter="/",
+        )
+        # page_iterator = page_iterator.search("Contents[?Size < `2200`][]")
+        page_iterator = page_iterator.search(
+            expression="Contents[?contains(Key, '.raw')] "
+        )
+        for res in page_iterator:
+            if "Key" in res:
+                return res["Key"]
+        return None
+        # else raise exception?
+
+        # DSJ0604-D20060406-T050022.bot 2kB == 2152 'Size'
+
+    def get_files_under_size(
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+    ):
+        # THIS isn't used, just playing with JMES paths spec
+        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"
+        ### filter with JMESPath expressions ###
+        page_iterator = self.s3_manager.paginator.paginate(
+            Bucket=self.input_bucket_name,
+            Prefix=prefix,
+            Delimiter="/",
+        )
+        page_iterator = page_iterator.search("Contents[?Size < `2200`][]")
+        all_files = []
+        for page in page_iterator:
+            if "Contents" in page.keys():
+                all_files.extend([i["Key"] for i in page["Contents"]])
+        return [i for i in all_files if i.endswith(".raw")]
+
     #################################################################
     def get_raw_files_csv(
         self,
@@ -85,9 +152,7 @@ class IndexManager:
         sensor_name,
     ):
         raw_files = self.get_raw_files(
-            ship_name=ship_name,
-            cruise_name=cruise_name,
-            sensor_name=sensor_name
+            ship_name=ship_name, cruise_name=cruise_name, sensor_name=sensor_name
         )
         files_list = [
             {
@@ -102,15 +167,34 @@ class IndexManager:
         df.to_csv(f"{ship_name}_{cruise_name}.csv", index=False, header=False, sep=" ")
         print("done")
 
-    #################################################################
-    def get_subset_ek60_prefix( # TODO: is this used?
+    def get_raw_files_list(
         self,
-        df: pd.DataFrame
-    ) -> pd.DataFrame:
+        ship_name,
+        cruise_name,
+        sensor_name,
+    ):
+        # gets all raw files in cruise and returns a list of dicts
+        raw_files = self.get_raw_files(
+            ship_name=ship_name, cruise_name=cruise_name, sensor_name=sensor_name
+        )
+        files_list = [
+            {
+                "ship_name": ship_name,
+                "cruise_name": cruise_name,
+                "sensor_name": sensor_name,
+                "file_name": os.path.basename(raw_file),
+            }
+            for raw_file in raw_files
+        ]
+        return files_list
+
+    #################################################################
+    @staticmethod
+    def get_subset_ek60_prefix(df: pd.DataFrame) -> pd.DataFrame:  # TODO: is this used?
         # Returns all objects with 'EK60' in prefix of file path
-        # Note that this can include 'EK80' data that are false-positives
+        # Note that this can include 'EK80' dataset that are false-positives
         # in dataframe with ['key', 'filename', 'ship', 'cruise', 'sensor', 'size', 'date', 'datagram']
-        print("getting subset of ek60 data by prefix")
+        print("getting subset of ek60 dataset by prefix")
         objects = []
         for row in df.itertuples():
             row_split = row[1].split(os.sep)
@@ -150,10 +234,7 @@ class IndexManager:
         return pd.DataFrame(objects)
 
     #################################################################
-    def scan_datagram(
-        self,
-        select_key: str
-    ) -> list:
+    def scan_datagram(self, select_key: str) -> list:
         # Reads the first 8 bytes of S3 file. Used to determine if ek60 or ek80
         # Note: uses boto3 session instead of boto3 client: https://github.com/boto/boto3/issues/801
         # select_key = 'data/raw/Albatross_Iv/AL0403/EK60/L0005-D20040302-T200108-EK60.raw'
@@ -165,20 +246,21 @@ class IndexManager:
             obj.get(Range="bytes=3-7")["Body"].read().decode().strip("\x00")
         )
         # return [{'KEY': select_key, 'DATAGRAM': first_datagram}]
-        ### EK60 data are denoted by 'CON0' ###
+        ### EK60 dataset are denoted by 'CON0' ###
         return first_datagram
 
     #################################################################
     def get_subset_datagrams(
-        self,
-        df: pd.DataFrame
-    ) -> list:
+        self, df: pd.DataFrame
+    ) -> list:  # TODO: is this getting used
         print("getting subset of datagrams")
-        select_keys = list(
-            df[["KEY", "CRUISE"]].drop_duplicates(subset="CRUISE")["KEY"].values
+        select_keys = (
+            df[["KEY", "CRUISE"]]
+            .drop_duplicates(subset="CRUISE")["KEY"]
+            .values.tolist()
         )
         all_datagrams = []
-        with ThreadPoolExecutor(max_workers=self.max_pool_connections) as executor:
+        with ThreadPoolExecutor(max_workers=MAX_POOL_CONNECTIONS) as executor:
             futures = [
                 executor.submit(self.scan_datagram, select_key)
                 for select_key in select_keys
@@ -190,11 +272,8 @@ class IndexManager:
         return all_datagrams
 
     #################################################################
-    def get_ek60_objects(
-        self,
-        df: pd.DataFrame,
-        subset_datagrams: list
-    ) -> pd.DataFrame:
+    @staticmethod
+    def get_ek60_objects(df: pd.DataFrame, subset_datagrams: list) -> pd.DataFrame:
         # for each key write datagram value to all other files in same cruise
         for subset_datagram in subset_datagrams:
             if subset_datagram["DATAGRAM"] == "CON0":
@@ -210,11 +289,11 @@ class IndexManager:
     def get_calibration_information(
         self,
     ) -> pd.DataFrame:
-        # Calibration data generated by data manager currently located here:
+        # Calibration dataset generated by dataset manager currently located here:
         #      https://noaa-wcsd-pds-index.s3.amazonaws.com/calibrated_crusies.csv
         # Note: Data are either:
-        #      [1] Calibrated w/ calibration data
-        #      [2] Calibrated w/o calibration data
+        #      [1] Calibrated w/ calibration dataset
+        #      [2] Calibrated w/o calibration dataset
         #      [3] uncalibrated
         response = self.s3_manager.get_object(
             bucket_name=self.calibration_bucket, key_name=self.calibration_key
@@ -261,3 +340,42 @@ class IndexManager:
     #     end_time = datetime.now()  # used for benchmarking
     #     print(start_time)
     #     print(end_time)
+
+    # TODO: wip
+    # def build_merkle_tree(self):
+    #     G = nx.DiGraph()
+    #     # https://noaa-wcsd-pds.s3.amazonaws.com/index.html#data/raw/Henry_B._Bigelow/HB0707/
+    #     ship_name = "Henry_B._Bigelow"
+    #     cruise_name = "HB0707"
+    #     # cruise_name = "HB0805"
+    #     prefix = f"data/raw/{ship_name}/{cruise_name}/"
+    #     # prefix = f"data/raw/{ship_name}/"
+    #     page_iterator = self.s3_manager.paginator.paginate(
+    #         Bucket=self.input_bucket_name,
+    #         Prefix=prefix,
+    #     )
+    #     for page in page_iterator:
+    #         for contents in page["Contents"]:
+    #             obj_key = contents["Key"]
+    #             # https://datatracker.ietf.org/doc/html/rfc7232#section-2.3
+    #             obj_etag = contents["ETag"].split('"')[1]  # properties
+    #             obj_size = contents["Size"]
+    #             basename = os.path.basename(obj_key)
+    #             G.add_node(
+    #                 node_for_adding=basename, ETag=obj_etag, Size=obj_size, Key=obj_key
+    #             )  # TODO: add parent hash
+    #             split_path = os.path.normpath(obj_key).split(os.path.sep)
+    #             # split_path: ['dataset', 'raw', 'Henry_B._Bigelow', 'HB0707', 'EK60', 'D20070712-T004447.raw']
+    #             for previous, current in zip(split_path, split_path[1:]):
+    #                 if not G.has_edge(previous, current):
+    #                     G.add_edge(previous, current)
+    #     # print(G)
+    #     etag_set = frozenset(
+    #         [k for j, k in list(G.nodes.data("ETag")) if k is not None]
+    #     )
+    #     new_hash = sha256(str(etag_set.__hash__()).encode("utf-8")).hexdigest()
+    #     total_size = [k for j, k in list(G.nodes.data("Size")) if k is not None]
+    #     print(np.sum(total_size))  # 22.24 Terabytes in Henry_B._Bigelow cruises
+    #     print(" ")
+    #     print(new_hash)
+    #     return new_hash