water-column-sonar-processing 0.0.1__py3-none-any.whl → 25.11.1__py3-none-any.whl

model/index/index.py

@@ -1,228 +0,0 @@
-import os
-import re
-import pandas as pd
-from datetime import datetime
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures import as_completed
-from ..aws.s3_manager import S3Manager
-
-
-class IndexManager:
-
-    def __init__(
-            self,
-            input_bucket_name,
-            calibration_bucket,
-            calibration_key
-    ):
-        self.input_bucket_name = input_bucket_name
-        self.calibration_bucket = calibration_bucket
-        self.calibration_key = calibration_key
-        self.s3_manager = S3Manager()
-
-    #################################################################
-    def list_ships(
-            self,
-            prefix='data/raw/',
-    ):
-        # s3_client = self.s3_manager.s3_client
-        page_iterator = self.s3_manager.paginator.paginate(Bucket=self.input_bucket_name, Prefix=prefix, Delimiter="/")
-        # common_prefixes = s3_client.list_objects(Bucket=self.input_bucket_name, Prefix=prefix, Delimiter='/')
-        # print(common_prefixes)
-        ships = []
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                ships.extend([k['Prefix'] for k in page['CommonPrefixes']])
-        return ships  # ~76 ships
-
-    #################################################################
-    def list_cruises(
-            self,
-            ship_prefixes,  # e.g. 'data/raw/Alaska_Knight/'
-    ):
-        cruises = []
-        for ship_prefix in ship_prefixes:
-            page_iterator = self.s3_manager.paginator.paginate(Bucket=self.input_bucket_name, Prefix=ship_prefix, Delimiter="/")
-            for page in page_iterator:
-                cruises.extend([k['Prefix'] for k in page['CommonPrefixes']])
-        return cruises  # ~1204 cruises
-
-    #################################################################
-    def list_ek60_cruises(
-            self,
-            cruise_prefixes,
-    ):
-        cruise_sensors = []  # includes all sensor types
-        for cruise_prefix in cruise_prefixes:
-            page_iterator = self.s3_manager.paginator.paginate(Bucket=self.input_bucket_name, Prefix=cruise_prefix, Delimiter="/")
-            for page in page_iterator:
-                cruise_sensors.extend([k['Prefix'] for k in page['CommonPrefixes']])
-        # Note: these are "EK60" by prefix. They still need to be verified by scanning the datagram.
-        return [i for i in cruise_sensors if '/EK60/' in i]  # ~447 different cruises
-
-    #################################################################
-    def get_raw_files(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-    ):
-        prefix = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/"  # Note no forward slash at beginning
-        page_iterator = self.s3_manager.paginator.paginate(Bucket=self.input_bucket_name, Prefix=prefix, Delimiter="/")
-        all_files = []
-        for page in page_iterator:
-            if 'Contents' in page.keys():
-                all_files.extend([i['Key'] for i in page['Contents']])
-        return [i for i in all_files if i.endswith('.raw')]
-
-    def get_raw_files_csv(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-    ):
-        raw_files = self.get_raw_files(ship_name=ship_name, cruise_name=cruise_name, sensor_name=sensor_name)
-        files_list = [
-            {
-                'ship_name': ship_name,
-                'cruise_name': cruise_name,
-                'sensor_name': sensor_name,
-                'file_name': os.path.basename(raw_file)
-            } for raw_file in raw_files
-        ]
-        df = pd.DataFrame(files_list)
-        df.to_csv(f'{ship_name}_{cruise_name}.csv', index=False, header=False, sep=' ')
-        print('done')
-
-
-    #################################################################
-    def get_subset_ek60_prefix(
-            self,
-            df: pd.DataFrame
-    ) -> pd.DataFrame:
-        # Returns all objects with 'EK60' in prefix of file path
-        # Note that this can include 'EK80' data that are false-positives
-        # in dataframe with ['key', 'filename', 'ship', 'cruise', 'sensor', 'size', 'date', 'datagram']
-        print("getting subset of ek60 data by prefix")
-        objects = []
-        for row in df.itertuples():
-            row_split = row[1].split(os.sep)
-            if len(row_split) == 6:
-                filename = os.path.basename(row[1])  # 'EX1608_EK60-D20161205-T040300.raw'
-                if filename.endswith(".raw"):
-                    ship_name, cruise_name, sensor_name = row_split[2:5]  # 'Okeanos_Explorer', 'EX1608', 'EK60'
-                    if re.search("[D](\d{8})", filename) is not None and re.search("[T](\d{6})", filename) is not None:
-                        # Parse date if possible e.g.: 'data/raw/Henry_B._Bigelow/HB1006/EK60/HBB-D20100723-T025105.raw'
-                        # and 'data/raw/Henry_B._Bigelow/HB1802/EK60/D20180513-T150250.raw'
-                        date_substring = re.search("[D](\d{8})", filename).group(1)
-                        time_substring = re.search("[T](\d{6})", filename).group(1)
-                        date_string = datetime.strptime(f'{date_substring}{time_substring}', '%Y%m%d%H%M%S')
-                    else:  # otherwise use current date
-                        date_string = f"{datetime.utcnow().isoformat()[:19]}Z"
-                    objects.append(
-                        {
-                            'KEY': row[1],
-                            'FILENAME': filename,
-                            'SHIP': ship_name,
-                            'CRUISE': cruise_name,
-                            'SENSOR': sensor_name,
-                            'SIZE': row[2],
-                            'DATE': date_string,
-                            'DATAGRAM': None
-                        }
-                    )
-        return pd.DataFrame(objects)
-
-    #################################################################
-    def scan_datagram(
-            self,
-            select_key: str
-    ) -> list:
-        # Reads the first 8 bytes of S3 file. Used to determine if ek60 or ek80
-        # Note: uses boto3 session instead of boto3 client: https://github.com/boto/boto3/issues/801
-        # select_key = 'data/raw/Albatross_Iv/AL0403/EK60/L0005-D20040302-T200108-EK60.raw'
-        s3_resource = self.s3_manager.s3_resource
-        obj = s3_resource.Object(bucket_name=self.input_bucket_name, key=select_key)  # XML0
-        first_datagram = obj.get(Range='bytes=3-7')['Body'].read().decode().strip('\x00')
-        # return [{'KEY': select_key, 'DATAGRAM': first_datagram}]
-        ### EK60 data are denoted by 'CON0' ###
-        return first_datagram
-
-    #################################################################
-    def get_subset_datagrams(
-            self,
-            df: pd.DataFrame
-    ) -> list:
-        print("getting subset of datagrams")
-        select_keys = list(df[['KEY', 'CRUISE']].drop_duplicates(subset='CRUISE')['KEY'].values)
-        all_datagrams = []
-        with ThreadPoolExecutor(max_workers=self.max_pool_connections) as executor:
-            futures = [executor.submit(self.scan_datagram, select_key) for select_key in select_keys]
-            for future in as_completed(futures):
-                result = future.result()
-                if result:
-                    all_datagrams.extend(result)
-        return all_datagrams
-
-    #################################################################
-    def get_ek60_objects(
-            self,
-            df: pd.DataFrame,
-            subset_datagrams: list
-    ) -> pd.DataFrame:
-        # for each key write datagram value to all other files in same cruise
-        for subset_datagram in subset_datagrams:
-            if subset_datagram['DATAGRAM'] == 'CON0':
-                select_cruise = df.loc[df['KEY'] == subset_datagram['KEY']]['CRUISE'].iloc[0]
-                df.loc[df['CRUISE'] == select_cruise, ['DATAGRAM']] = subset_datagram['DATAGRAM']
-        return df.loc[df['DATAGRAM'] == 'CON0']
-
-    #################################################################
-    def get_calibration_information(  # tested
-            self,
-    ) -> pd.DataFrame:
-        # Calibration data generated by data manager currently located here:
-        #      https://noaa-wcsd-pds-index.s3.amazonaws.com/calibrated_crusies.csv
-        # Note: Data are either:
-        #      [1] Calibrated w/ calibration data
-        #      [2] Calibrated w/o calibration data
-        #      [3] uncalibrated
-        response = self.s3_manager.get_object(bucket_name=self.calibration_bucket, key_name=self.calibration_key)
-        calibration_statuses = pd.read_csv(response.get("Body"))
-        calibration_statuses['DATASET_NAME'] = calibration_statuses['DATASET_NAME'].apply(lambda x: x.split('_EK60')[0])
-        calibration_statuses['CAL_STATE'] = calibration_statuses['CAL_STATE'].apply(lambda x: x.find('Calibrated') >= 0)
-        return calibration_statuses
-
-    #################################################################
-    # def index(  # TODO: get rid of this?
-    #         self
-    # ):
-    #     start_time = datetime.now()  # used for benchmarking
-    #     # Get all object in public dataset bucket
-    #     all_objects = self.get_all_objects()
-    #     #
-    #     subset_ek60_by_prefix = self.get_subset_ek60_prefix(
-    #         df=all_objects[all_objects['Key'].str.contains('EK60')][['Key', 'Size']]
-    #     )
-    #     #
-    #     subset_datagrams = self.get_subset_datagrams(df=subset_ek60_by_prefix)
-    #     print("done getting subset of datagrams")
-    #     ek60_objects = self.get_ek60_objects(subset_ek60_by_prefix, subset_datagrams)
-    #     print("done getting ek60_objects")
-    #     print(start_time)
-    #     calibration_status = self.get_calibration_information(s3)
-    #     cruise_names = list(set(ek60_objects['CRUISE']))
-    #     cruise_names.sort()
-    #     for cruise_name in cruise_names:  # ~322 cruises
-    #         cruise_data = ek60_objects.groupby('CRUISE').get_group(cruise_name)
-    #         ship = cruise_data['SHIP'].iloc[0]
-    #         sensor = cruise_data['SENSOR'].iloc[0]
-    #         datagram = cruise_data['DATAGRAM'].iloc[0]
-    #         file_count = cruise_data.shape[0]
-    #         total_size = np.sum(cruise_data['SIZE'])
-    #         calibrated = cruise_name in calibration_status['DATASET_NAME'].unique()  # ~276 entries
-    #         start_date = np.min(cruise_data['DATE']).isoformat(timespec="seconds") + "Z"
-    #         end_date = np.max(cruise_data['DATE']).isoformat(timespec="seconds") + "Z"
-    #     end_time = datetime.now()  # used for benchmarking
-    #     print(start_time)
-    #     print(end_time)

model/model.py

@@ -1,138 +0,0 @@
-import os
-import json
-import numpy as np
-from src.model.aws.s3_manager import S3Manager
-from src.model.aws.sns_manager import SNSManager
-from src.model.aws.dynamodb_manager import DynamoDBManager
-
-
-###########################################################
-class Model:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        self.input_bucket_name = os.environ['INPUT_BUCKET_NAME']
-        self.output_bucket_name = os.environ['OUTPUT_BUCKET_NAME']
-        self.table_name = os.environ['TABLE_NAME']
-        self.topic_arn = os.environ['TOPIC_ARN']
-        # self.output_bucket_access_key = ?
-        # self.output_bucket_secret_access_key = ?
-
-    def execute(self):
-        input_s3_manager = S3Manager()  # TODO: Need to allow passing in of credentials when writing to protected bucket
-        output_s3_manager = S3Manager()
-        # TODO: s3fs?
-        sns_manager = SNSManager()
-        ddb_manager = DynamoDBManager()
-
-        # [1 of 5] Update Pipeline Status in DynamoDB
-        #self.dynamodb.update_ status ()
-
-        # [2 of 5] Download Object From Input Bucket
-        return_value = input_s3_manager.get(
-            bucket_name=self.input_bucket_name,
-            key='the_input_key'
-        )
-        print(return_value)
-
-        # [3 of 5] Update Entry in DynamoDB
-        ship_name = 'David_Starr_Jordan'  # TODO: get this from input sns message
-        cruise_name = 'DS0604'
-        sensor_name = 'EK60'
-        file_name = "DSJ0604-D20060406-T113407.raw"
-
-        test_channels = [
-            "GPT  38 kHz 009072055a7f 2 ES38B",
-            "GPT  70 kHz 00907203400a 3 ES70-7C",
-            "GPT 120 kHz 009072034d52 1 ES120-7",
-            "GPT 200 kHz 0090720564e4 4 ES200-7C"
-        ]
-        test_frequencies = [38_000, 70_000, 120_000, 200_000]
-        ddb_manager.update_item(
-            table_name=self.table_name,
-            key={
-                'FILE_NAME': {'S': file_name},  # Partition Key
-                'CRUISE_NAME': {'S': cruise_name},  # Sort Key
-            },
-            expression_attribute_names={
-                '#CH': 'CHANNELS',
-                '#ET': 'END_TIME',
-                '#ED': 'ERROR_DETAIL',
-                '#FR': 'FREQUENCIES',
-                '#MA': 'MAX_ECHO_RANGE',
-                '#MI': 'MIN_ECHO_RANGE',
-                '#ND': 'NUM_PING_TIME_DROPNA',
-                '#PS': 'PIPELINE_STATUS',  # testing this updated
-                '#PT': 'PIPELINE_TIME',  # testing this updated
-                '#SE': 'SENSOR_NAME',
-                '#SH': 'SHIP_NAME',
-                '#ST': 'START_TIME',
-                '#ZB': 'ZARR_BUCKET',
-                '#ZP': 'ZARR_PATH',
-            },
-            expression_attribute_values={
-                ':ch': {'L': [{'S': i} for i in test_channels]},
-                ':et': {'S': '2006-04-06T13:35:28.688Z'},
-                ':ed': {'S': ''},
-                ':fr': {'L': [{'N': str(i)} for i in test_frequencies]},
-                ':ma': {'N': str(np.round(499.7653, 4))},
-                ':mi': {'N': str(np.round(0.25, 4))},
-                ':nd': {'N': str(2458)},
-                ':ps': {'S': 'SUCCESS_AGGREGATOR'},
-                ':pt': {'S': '2023-10-02T08:54:43Z'},
-                ':se': {'S': sensor_name},
-                ':sh': {'S': ship_name},
-                ':st': {'S': '2006-04-06T11:34:07.288Z'},
-                ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-                ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr'},
-            },
-            update_expression=(
-                'SET '
-                '#CH = :ch, '
-                '#ET = :et, '
-                '#ED = :ed, '
-                '#FR = :fr, '
-                '#MA = :ma, '
-                '#MI = :mi, '
-                '#ND = :nd, '
-                '#PS = :ps, '
-                '#PT = :pt, '
-                '#SE = :se, '
-                '#SH = :sh, '
-                '#ST = :st, '
-                '#ZB = :zb, '
-                '#ZP = :zp'
-            )
-        )
-
-        # [4 of 5] Write Object to Output Bucket
-        output_s3_manager.put(
-            bucket_name=self.output_bucket_name,
-            key='123',
-            body='456'
-        )
-
-        # [_ of _] Read file-level Zarr store from bucket, Create GeoJSON, Write to bucket
-        # [_ of _] Create empty cruise-level Zarr store
-        # [_ of _] Resample and write to cruise-level Zarr Store
-
-        # [5 of 5] Publish Done Message
-        success_message = {
-            "default": {
-                "shipName": ship_name,
-                "cruiseName": cruise_name,
-                "sensorName": sensor_name,
-                "fileName": file_name,
-            }
-        }
-        sns_manager.publish(
-            topic_arn=self.topic_arn,
-            message=json.dumps(success_message),
-        )
-        print("done...")
-
-    #######################################################
-
-###########################################################
-###########################################################

model/utility/constants.py

@@ -1,56 +0,0 @@
-from enum import Enum, Flag, unique
-
-
-@unique
-class Constants(Flag):
-    TILE_SIZE = 512
-
-
-class Coordinates(Enum):
-    """
-    Should try to specify
-        dtype
-        units
-        long_name — most readable description of variable
-        standard_name — name in lowercase and snake_case
-    """
-    PROJECT_NAME = 'echofish'
-
-    DEPTH = 'depth'
-    DEPTH_DTYPE = 'float32'
-    DEPTH_UNITS = 'm'  # TODO: Pint? <https://pint.readthedocs.io/en/stable/>
-    DEPTH_LONG_NAME = 'Depth below surface'
-    DEPTH_STANDARD_NAME = 'depth'
-
-    TIME = 'time'
-    TIME_DTYPE = 'float64'
-    # Note: units and calendar are used downstream by Xarray
-    TIME_UNITS = 'seconds since 1970-01-01 00:00:00'
-    TIME_LONG_NAME = 'Timestamp of each ping'
-    TIME_STANDARD_NAME = 'time'
-    TIME_CALENDAR = 'proleptic_gregorian'
-    # TODO: create test for reading out timestamps in Xarray
-
-    FREQUENCY = 'frequency'
-    FREQUENCY_DTYPE = 'int'
-    FREQUENCY_UNITS = 'Hz'
-    FREQUENCY_LONG_NAME = 'Transducer frequency'
-    FREQUENCY_STANDARD_NAME = 'sound_frequency'
-
-    LATITUDE = 'latitude'
-    LATITUDE_DTYPE = 'float32'
-    LATITUDE_UNITS = 'degrees_north'
-    LATITUDE_LONG_NAME = 'Latitude'
-    LATITUDE_STANDARD_NAME = 'latitude'
-
-    LONGITUDE = 'longitude'
-    LONGITUDE_DTYPE = 'float32'
-    LONGITUDE_UNITS = 'degrees_east'
-    LONGITUDE_LONG_NAME = 'Longitude'
-    LONGITUDE_STANDARD_NAME = 'longitude'
-
-    SV = 'Sv'
-    SV_DTYPE = 'float32'  # TODO: experiment with dtype of int
-    SV_UNITS = 'dB'
-    SV_LONG_NAME = 'Volume backscattering strength (Sv re 1 m-1)'
-    SV_STANDARD_NAME = 'volume_backscattering_strength'

model/utility/timestamp.py

@@ -1,12 +0,0 @@
-from datetime import datetime
-
-
-###########################################################
-class Timestamp:
-    @staticmethod
-    def get_timestamp():
-        # return timestamp in form:
-        #   PIPELINE_TIME = '2024-03-29T19:36:52.433Z'
-        return f'{datetime.utcnow().isoformat()[:23]}Z'
-
-###########################################################

model/zarr/bar.py

@@ -1,28 +0,0 @@
-import requests
-
-
-# class Bar(object):
-#
-#     def biz(self):
-#         pass
-
-
-# class Bar(object):
-#
-#     def sync(self, id, query_first):
-#         if query_first:
-#             requests.get('/remote/api/{id}'.format(id=id))
-#
-#         requests.put(
-#             '/remote/other/api/{id}'.format(id=id),
-#             data=123  # current_data()
-#         )
-
-# class Bar(object):
-#     def biz(self, url, method, data, headers):
-#         pass
-
-class Bar(object):
-
-    def biz(self):
-        return 1

model/zarr/foo.py

@@ -1,11 +0,0 @@
-from src.model.zarr.bar import Bar
-
-
-# def foo():
-#     Bar().biz()
-
-# def foo(url, method='GET', data=None, headers=None):
-#     Bar().biz(url, method, data=data, headers=headers)
-
-def foo():
-    return Bar().biz()