water-column-sonar-processing 25.3.2__py3-none-any.whl → 25.8.0__py3-none-any.whl

water_column_sonar_processing/index/index_manager.py

@@ -28,7 +28,7 @@ class IndexManager:
     #################################################################
     def list_ships(
         self,
-        prefix="data/raw/",
+        prefix="dataset/raw/",
     ):
         page_iterator = self.s3_manager.paginator.paginate(
             Bucket=self.input_bucket_name, Prefix=prefix, Delimiter="/"
@@ -44,7 +44,7 @@ class IndexManager:
     #################################################################
     def list_cruises(
         self,
-        ship_prefixes,  # e.g. 'data/raw/Alaska_Knight/'
+        ship_prefixes,  # e.g. 'dataset/raw/Alaska_Knight/'
     ):
         cruises = []
         for ship_prefix in ship_prefixes:
@@ -81,7 +81,7 @@ class IndexManager:
         sensor_name,
     ):
         # Gets all raw files for a cruise under the given prefix
-        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
+        prefix = f"dataset/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
         page_iterator = self.s3_manager.paginator.paginate(
             Bucket=self.input_bucket_name, Prefix=prefix, Delimiter="/"
         )
@@ -99,7 +99,8 @@ class IndexManager:
     ):
         # Same as above but only needs to get the first raw file
         # because we are only interested in the first datagram of one file
-        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
+        # TODO: "dataset?"
+        prefix = f"dataset/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
         # page_iterator = self.s3_manager.paginator.paginate(
         #     Bucket=self.input_bucket_name,
         #     Prefix=prefix,
@@ -130,7 +131,7 @@ class IndexManager:
         sensor_name,
     ):
         # THIS isn't used, just playing with JMES paths spec
-        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"
+        prefix = f"dataset/raw/{ship_name}/{cruise_name}/{sensor_name}/"
         ### filter with JMESPath expressions ###
         page_iterator = self.s3_manager.paginator.paginate(
             Bucket=self.input_bucket_name,
@@ -193,9 +194,9 @@ class IndexManager:
         self, df: pd.DataFrame
     ) -> pd.DataFrame:  # TODO: is this used?
         # Returns all objects with 'EK60' in prefix of file path
-        # Note that this can include 'EK80' data that are false-positives
+        # Note that this can include 'EK80' dataset that are false-positives
         # in dataframe with ['key', 'filename', 'ship', 'cruise', 'sensor', 'size', 'date', 'datagram']
-        print("getting subset of ek60 data by prefix")
+        print("getting subset of ek60 dataset by prefix")
         objects = []
         for row in df.itertuples():
             row_split = row[1].split(os.sep)
@@ -211,8 +212,8 @@ class IndexManager:
                         re.search("[D](\\d{8})", filename) is not None
                         and re.search("[T](\\d{6})", filename) is not None
                     ):
-                        # Parse date if possible e.g.: 'data/raw/Henry_B._Bigelow/HB1006/EK60/HBB-D20100723-T025105.raw'
-                        # and 'data/raw/Henry_B._Bigelow/HB1802/EK60/D20180513-T150250.raw'
+                        # Parse date if possible e.g.: 'dataset/raw/Henry_B._Bigelow/HB1006/EK60/HBB-D20100723-T025105.raw'
+                        # and 'dataset/raw/Henry_B._Bigelow/HB1802/EK60/D20180513-T150250.raw'
                         date_substring = re.search("[D](\\d{8})", filename).group(1)
                         time_substring = re.search("[T](\\d{6})", filename).group(1)
                         date_string = datetime.strptime(
@@ -238,7 +239,7 @@ class IndexManager:
     def scan_datagram(self, select_key: str) -> list:
         # Reads the first 8 bytes of S3 file. Used to determine if ek60 or ek80
         # Note: uses boto3 session instead of boto3 client: https://github.com/boto/boto3/issues/801
-        # select_key = 'data/raw/Albatross_Iv/AL0403/EK60/L0005-D20040302-T200108-EK60.raw'
+        # select_key = 'dataset/raw/Albatross_Iv/AL0403/EK60/L0005-D20040302-T200108-EK60.raw'
         s3_resource = self.s3_manager.s3_resource
         obj = s3_resource.Object(
             bucket_name=self.input_bucket_name, key=select_key
@@ -247,7 +248,7 @@ class IndexManager:
             obj.get(Range="bytes=3-7")["Body"].read().decode().strip("\x00")
         )
         # return [{'KEY': select_key, 'DATAGRAM': first_datagram}]
-        ### EK60 data are denoted by 'CON0' ###
+        ### EK60 dataset are denoted by 'CON0' ###
         return first_datagram
 
     #################################################################
@@ -291,11 +292,11 @@ class IndexManager:
     def get_calibration_information(
         self,
     ) -> pd.DataFrame:
-        # Calibration data generated by data manager currently located here:
+        # Calibration dataset generated by dataset manager currently located here:
         #      https://noaa-wcsd-pds-index.s3.amazonaws.com/calibrated_crusies.csv
         # Note: Data are either:
-        #      [1] Calibrated w/ calibration data
-        #      [2] Calibrated w/o calibration data
+        #      [1] Calibrated w/ calibration dataset
+        #      [2] Calibrated w/o calibration dataset
         #      [3] uncalibrated
         response = self.s3_manager.get_object(
             bucket_name=self.calibration_bucket, key_name=self.calibration_key
@@ -350,8 +351,8 @@ class IndexManager:
         ship_name = "Henry_B._Bigelow"
         cruise_name = "HB0707"
         # cruise_name = "HB0805"
-        prefix = f"data/raw/{ship_name}/{cruise_name}/"
-        # prefix = f"data/raw/{ship_name}/"
+        prefix = f"dataset/raw/{ship_name}/{cruise_name}/"
+        # prefix = f"dataset/raw/{ship_name}/"
         page_iterator = self.s3_manager.paginator.paginate(
             Bucket=self.input_bucket_name,
             Prefix=prefix,
@@ -367,7 +368,7 @@ class IndexManager:
                     node_for_adding=basename, ETag=obj_etag, Size=obj_size, Key=obj_key
                 )  # TODO: add parent hash
                 split_path = os.path.normpath(obj_key).split(os.path.sep)
-                # split_path: ['data', 'raw', 'Henry_B._Bigelow', 'HB0707', 'EK60', 'D20070712-T004447.raw']
+                # split_path: ['dataset', 'raw', 'Henry_B._Bigelow', 'HB0707', 'EK60', 'D20070712-T004447.raw']
                 for previous, current in zip(split_path, split_path[1:]):
                     if not G.has_edge(previous, current):
                         G.add_edge(previous, current)