water-column-sonar-processing 25.1.1__py3-none-any.whl → 25.1.3__py3-none-any.whl

water_column_sonar_processing/aws/dynamodb_manager.py

@@ -13,16 +13,16 @@ class DynamoDBManager:
             # endpoint_url
     ):
         # self.endpoint_url = endpoint_url
-        self.__dynamodb_session = boto3.Session(
+        self.dynamodb_session = boto3.Session(
             aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
             aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
             region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
-        self.__dynamodb_resource = self.__dynamodb_session.resource(
+        self.dynamodb_resource = self.dynamodb_session.resource(
             service_name="dynamodb",
             # endpoint_url=self.endpoint_url
         )
-        self.__dynamodb_client = self.__dynamodb_session.client(
+        self.dynamodb_client = self.dynamodb_session.client(
             service_name="dynamodb",
             # endpoint_url=self.endpoint_url
         )
@@ -46,7 +46,7 @@ class DynamoDBManager:
         self,
         table_name,
     ):
-        self.__dynamodb_client.create_table(
+        self.dynamodb_client.create_table(
             TableName=table_name,
             KeySchema=[
                 {
@@ -69,7 +69,7 @@ class DynamoDBManager:
             # }
         )
         # TODO: after creating status is 'CREATING', wait until 'ACTIVE'
-        response = self.__dynamodb_client.describe_table(TableName=table_name)
+        response = self.dynamodb_client.describe_table(TableName=table_name)
         print(response) # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/describe_table.html
         # sleep then response['Table']['TableStatus'] == 'ACTIVE'
 
@@ -80,7 +80,7 @@ class DynamoDBManager:
     #         table_name,
     #         key
     # ):
-    #     response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
+    #     response = self.dynamodb_client.get_item(TableName=table_name, Key=key)
     #     item = None
     #     if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
     #         if "Item" in response:
@@ -96,7 +96,7 @@ class DynamoDBManager:
         """
         Gets a single row from the db.
         """
-        table = self.__dynamodb_resource.Table(table_name)
+        table = self.dynamodb_resource.Table(table_name)
         response = table.get_item(Key=key)
         # TODO:
         # if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
@@ -113,7 +113,7 @@ class DynamoDBManager:
         update_expression,
     ): # TODO: convert to boolean
         try:
-            response = self.__dynamodb_client.update_item(
+            response = self.dynamodb_client.update_item(
                 TableName=table_name,
                 Key=key,
                 ExpressionAttributeNames=expression_attribute_names,
@@ -140,33 +140,53 @@ class DynamoDBManager:
         To be used to initialize a cruise, deletes all entries associated with that cruise
         in the database.
         """
-        expression_attribute_values = {
-            ":cr": {"S": cruise_name},
-            ":se": {"S": sensor_name},
-            ":sh": {"S": ship_name},
-        }
-
-        filter_expression = (
-            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
-        )
-        response = self.__dynamodb_client.scan(
+        filter_expression = "CRUISE_NAME = :cr"
+        response = self.dynamodb_client.scan(
             TableName=table_name,
-            Select="ALL_ATTRIBUTES",
-            ExpressionAttributeValues=expression_attribute_values,
+            # Limit=1000,
+            Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
+            # ExclusiveStartKey=where to pick up
+            #ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+            # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
             FilterExpression=filter_expression,
+            # ExpressionAttributeNames={
+            #     '#SH': 'SHIP_NAME',
+            #     '#CR': 'CRUISE_NAME',
+            #     '#FN': 'FILE_NAME',
+            # },
+            ExpressionAttributeValues={ # criteria
+                ':cr': {
+                    'S': cruise_name,
+                },
+            },
+            ConsistentRead=True
+            # ExclusiveStartKey=response["LastEvaluatedKey"],
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
-        if len(response["Items"]) == 0:
+
+        if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
             return pd.DataFrame() # If no results, return empty dataframe
 
         data = response["Items"]
 
-        while "LastEvaluatedKey" in response:
-            response = self.__dynamodb_client.scan(
+        while response.get('LastEvaluatedKey'): #"LastEvaluatedKey" in response:
+            response = self.dynamodb_client.scan(
                 TableName=table_name,
-                Select="ALL_ATTRIBUTES",
-                ExpressionAttributeValues=expression_attribute_values,
+                ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+                Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
                 FilterExpression=filter_expression,
+                #ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+                # ExpressionAttributeNames={ # would need to specify all cols in df
+                #     '#SH': 'SHIP_NAME',
+                #     '#CR': 'CRUISE_NAME',
+                #     '#FN': 'FILE_NAME',
+                # },
+                ExpressionAttributeValues={  # criteria
+                    ':cr': {
+                        'S': cruise_name,
+                    },
+                },
+                ConsistentRead=True,
                 ExclusiveStartKey=response["LastEvaluatedKey"],
             )
             data.extend(response["Items"])
@@ -176,6 +196,66 @@ class DynamoDBManager:
 
         return df.sort_values(by="START_TIME", ignore_index=True)
 
+    #####################################################################
+    # def get_cruise_list(
+    #     self,
+    #     table_name,
+    # ) -> list:
+    #     """
+    #     Experimental, gets all cruise names as list
+    #     """
+    #     filter_expression = "CRUISE_NAME = :cr"
+    #     response = self.dynamodb_client.scan(
+    #         TableName=table_name,
+    #         Select='SPECIFIC_ATTRIBUTES',
+    #         #ReturnConsumedCapacity='INDEXES' | 'TOTAL' | 'NONE', ...not sure
+    #         # ProjectionExpression='#SH, #CR, #FN', # what to specifically return — from expression_attribute_names
+    #         FilterExpression=filter_expression,
+    #         # ExpressionAttributeNames={
+    #         #     '#SH': 'SHIP_NAME',
+    #         #     '#CR': 'CRUISE_NAME',
+    #         #     '#FN': 'FILE_NAME',
+    #         # },
+    #         # ExpressionAttributeValues={ # criteria
+    #         #     ':cr': {
+    #         #         'S': cruise_name,
+    #         #     },
+    #         # },
+    #     )
+    #     # Note: table.scan() has 1 MB limit on results so pagination is used
+    #
+    #     if len(response["Items"]) == 0 and "LastEvaluatedKey" not in response:
+    #         return pd.DataFrame() # If no results, return empty dataframe
+    #
+    #     data = response["Items"]
+    #
+    #     while response.get('LastEvaluatedKey'): #"LastEvaluatedKey" in response:
+    #         response = self.dynamodb_client.scan(
+    #             TableName=table_name,
+    #             ### Either 'Select' or 'ExpressionAttributeNames'/'ProjectionExpression'
+    #             Select='ALL_ATTRIBUTES', # or 'SPECIFIC_ATTRIBUTES',
+    #             FilterExpression=filter_expression,
+    #             #ProjectionExpression='#SH, #CR, #FN',  # what to specifically return — from expression_attribute_names
+    #             # ExpressionAttributeNames={ # would need to specify all cols in df
+    #             #     '#SH': 'SHIP_NAME',
+    #             #     '#CR': 'CRUISE_NAME',
+    #             #     '#FN': 'FILE_NAME',
+    #             # },
+    #             ExpressionAttributeValues={  # criteria
+    #                 ':cr': {
+    #                     'S': cruise_name,
+    #                 },
+    #             },
+    #             ConsistentRead=True,
+    #             ExclusiveStartKey=response["LastEvaluatedKey"],
+    #         )
+    #         data.extend(response["Items"])
+    #
+    #     deserializer = self.type_deserializer
+    #     df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
+    #
+    #     return df.sort_values(by="START_TIME", ignore_index=True)
+
     #####################################################################
     # TODO: WIP
     def delete_item(
@@ -187,7 +267,7 @@ class DynamoDBManager:
         """
         Finds all rows associated with a cruise and deletes them.
         """
-        response = self.__dynamodb_client.delete_item(
+        response = self.dynamodb_client.delete_item(
             Key={
                 "CRUISE_NAME": {
                     "S": cruise_name
@@ -212,7 +292,7 @@ class DynamoDBManager:
         """
         Get a description of the table. Used to verify that records were added/removed.
         """
-        response = self.__dynamodb_client.describe_table(TableName=table_name)
+        response = self.dynamodb_client.describe_table(TableName=table_name)
         print(response)
         return response
 
@@ -230,7 +310,7 @@ class DynamoDBManager:
     #     print(f"Updating processing status to {pipeline_status}.")
     #     if error_message:
     #         print(f"Error message: {error_message}")
-    #         self.__dynamo.update_item(
+    #         self.dynamo.update_item(
     #             table_name=self.__table_name,
     #             key={
     #                 'FILE_NAME': {'S': file_name},  # Partition Key
@@ -255,7 +335,7 @@ class DynamoDBManager:
     #             }
     #         )
     #     else:
-    #         self.__dynamo.update_item(
+    #         self.dynamo.update_item(
     #             table_name=self.__table_name,
     #             key={
     #                 'FILE_NAME': {'S': file_name},  # Partition Key

water_column_sonar_processing/cruise/resample_regrid.py

@@ -1,5 +1,4 @@
 import gc
-import os
 from pathlib import Path
 
 import numcodecs
@@ -311,6 +310,12 @@ class ResampleRegrid:
                     output_zarr_store.bottom[
                         start_ping_time_index:end_ping_time_index
                     ] = detected_seafloor_depths
+                #
+                #
+                #
+                # TODO: write the time variable last so that I can parse that as check
+                #
+                #
                 #########################################################################
                 #########################################################################
         except Exception as err:

water_column_sonar_processing/model/zarr_manager.py

@@ -250,7 +250,7 @@ class ZarrManager:
         #
         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
         root.attrs["processing_software_version"] = (
-            "25.1.1"  # TODO: get programmatically, echopype>utils>prov.py
+            "25.1.3"  # TODO: get programmatically, echopype>utils>prov.py
         )
         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
         #

water_column_sonar_processing/utility/constants.py

@@ -3,7 +3,7 @@ from enum import Enum, Flag, unique
 
 @unique
 class Constants(Flag):
-    TILE_SIZE = 256 # TODO: add tile size to metadata?
+    TILE_SIZE = 1024 # TODO: add tile size to metadata?
 
     # Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
     # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data

{water_column_sonar_processing-25.1.1.dist-info → water_column_sonar_processing-25.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: water_column_sonar_processing
-Version: 25.1.1
+Version: 25.1.3
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -11,15 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiobotocore==2.15.2
-Requires-Dist: boto3==1.35.36
-Requires-Dist: botocore==1.35.36
+Requires-Dist: aiobotocore==2.19.0
+Requires-Dist: boto3==1.36.3
+Requires-Dist: botocore==1.36.3
 Requires-Dist: echopype==0.9.0
 Requires-Dist: fiona==1.10.1
 Requires-Dist: geopandas==1.0.1
 Requires-Dist: mock==5.1.0
-Requires-Dist: moto[all]==5.0.21
-Requires-Dist: moto[server]==5.0.21
+Requires-Dist: moto[all]==5.0.27
+Requires-Dist: moto[server]==5.0.27
 Requires-Dist: numcodecs==0.13.1
 Requires-Dist: numpy==1.26.4
 Requires-Dist: pandas==2.2.3
@@ -120,10 +120,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 # Tag a Release
 Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
 ```commandline
-git tag -a v25.01.01 -m "Releasing version v25.01.01"
-```
-
-```commandline
+git tag -a v25.1.2 -m "Releasing version v25.1.2"
 git push origin --tags
 ```
 

{water_column_sonar_processing-25.1.1.dist-info → water_column_sonar_processing-25.1.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
 water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
 water_column_sonar_processing/process.py,sha256=-yQtK3rnZq6lGAr3q02zLDe1NuMH9c0PiUOxKzG_r18,5386
 water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
-water_column_sonar_processing/aws/dynamodb_manager.py,sha256=LQ3eh7Zf1fBLG-RKovod9KbQwhE-0Qdq1JPk4Ro5bdo,10252
+water_column_sonar_processing/aws/dynamodb_manager.py,sha256=htP4Y2rmOSFtdzUFrgK14Bn-UXAFG22Ow-dDrR2alSw,13949
 water_column_sonar_processing/aws/s3_manager.py,sha256=-PCiW7YF31nGIPa1oVOVTzjTSExAAkT_IyNNnvWv2HU,16214
 water_column_sonar_processing/aws/s3fs_manager.py,sha256=Vo-DXj6vgb8t1l4LdtNu7JCtq_RfFsnl33RuGeBUXhk,2561
 water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
@@ -9,7 +9,7 @@ water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQ
 water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
 water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=ZsFQTDA0gXfQHlxDsXBGD1qQ0ipmx4kS81DcY6ml5Ew,7767
 water_column_sonar_processing/cruise/datatree_manager.py,sha256=Qy4dZCW8_q31lbjxbMsx3JtBS4BvQT17_2P0QD1RQcY,639
-water_column_sonar_processing/cruise/resample_regrid.py,sha256=gz_uP-mBD4JSBRBr69ZvsfmXX4yyBdRG9-P1z3If43E,14246
+water_column_sonar_processing/cruise/resample_regrid.py,sha256=986oZm6G8kngAmTg1GTTyxdx8Zy9fXAUGlK533GcrZ8,14413
 water_column_sonar_processing/geometry/__init__.py,sha256=GIzzc-_7pwEwbOkGpc4i_fmjWI5ymllXqzdHq_d3Rio,299
 water_column_sonar_processing/geometry/elevation_manager.py,sha256=eq9w691WJknPwWYkvO3giKTPleIxCVc2tMGR0e8ZRxQ,4267
 water_column_sonar_processing/geometry/geometry_manager.py,sha256=nz5T1vCDWHYIfQ853EqKYHDetTul7jRWS3y8Evep8QU,10855
@@ -18,17 +18,17 @@ water_column_sonar_processing/geometry/pmtile_generation.py,sha256=7Lm08Jr6YaM4n
 water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
 water_column_sonar_processing/index/index_manager.py,sha256=qsS6rKObJlFXKyzRuT1bk2_qW1YagW-Fg_AkQ1U_KRs,14213
 water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
-water_column_sonar_processing/model/zarr_manager.py,sha256=TkYP-FwQkW2I3IcpVQFzrmHQjD25mtN8epGjTDn5jWI,15505
+water_column_sonar_processing/model/zarr_manager.py,sha256=UT3GSEQ6-_kd62dDZf9AOvueR8jEiUPbzoFcpiwy8IY,15505
 water_column_sonar_processing/processing/__init__.py,sha256=tdpSfwnY6lbAS_yBTu4aG0SjPgCKqh6LAFvIj_t3j3U,168
 water_column_sonar_processing/processing/batch_downloader.py,sha256=qXoruHdbgzAolmroK6eRn9bWgeHFgaVQLwhJ6X5oHRE,6299
 water_column_sonar_processing/processing/raw_to_zarr.py,sha256=Sn0_zBT7yYP6abbSTlQBPA6iZSBxeVqPYYSgoroiBEU,17599
 water_column_sonar_processing/utility/__init__.py,sha256=yDObMOL0_OxKWet5wffK2-XVJgoE9iwiY2q04GZrtBQ,234
 water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
-water_column_sonar_processing/utility/constants.py,sha256=AD6RlDrJRVN1GYwRvo7cunLhrdC0F8CyOlbkB_GxL-s,2180
+water_column_sonar_processing/utility/constants.py,sha256=nAThVjJGpFp8kMdv3I1g6I1hVvOBJBDbGCTEEYewB4o,2181
 water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
 water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
-water_column_sonar_processing-25.1.1.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
-water_column_sonar_processing-25.1.1.dist-info/METADATA,sha256=M_ZpXvB7bE70nptx6DMHvlsUr0VoGWzmBaXrxzrmrWo,5474
-water_column_sonar_processing-25.1.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-water_column_sonar_processing-25.1.1.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
-water_column_sonar_processing-25.1.1.dist-info/RECORD,,
+water_column_sonar_processing-25.1.3.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
+water_column_sonar_processing-25.1.3.dist-info/METADATA,sha256=R0wRzg_ywppwzCzyY1MOyqRtFrPl6lyq1ltldgisbe4,5448
+water_column_sonar_processing-25.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+water_column_sonar_processing-25.1.3.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
+water_column_sonar_processing-25.1.3.dist-info/RECORD,,