water-column-sonar-processing 0.0.10__tar.gz → 0.0.11__tar.gz

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.10
+Version: 0.0.11
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "water_column_sonar_processing"
-version = "0.0.10"
+version = "0.0.11"
 authors = [
   { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
 ]

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/tests/test_raw_to_zarr.py

@@ -115,17 +115,18 @@ def test_raw_to_zarr(raw_to_zarr_test_path):
     # file_name = "D20070711-T182032.raw"
     #file_name = "D20070720-T224031.raw" # 84 KB
     raw_file_name = "D20070724-T042400.raw"  # 1 MB use this for testing
-    bottom_file_name = f"{Path(raw_file_name).stem}.bot"
+    # bottom_file_name = f"{Path(raw_file_name).stem}.bot"
 
     # TODO: move this into the raw_to_zarr function
-    s3_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
-    s3_bottom_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
-    s3_manager.download_file(bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name)
-    s3_manager.download_file(bucket_name=input_bucket_name, key=s3_bottom_file_path, file_name=bottom_file_name)
+    # s3_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
+    # s3_bottom_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
+    # s3_manager.download_file(bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name)
+    # s3_manager.download_file(bucket_name=input_bucket_name, key=s3_bottom_file_path, file_name=bottom_file_name)
 
     raw_to_zarr = RawToZarr()
     raw_to_zarr.raw_to_zarr(
         table_name=table_name,
+        input_bucket_name=input_bucket_name,
         output_bucket_name=output_bucket_name,
         ship_name=ship_name,
         cruise_name=cruise_name,
@@ -136,7 +137,7 @@ def test_raw_to_zarr(raw_to_zarr_test_path):
     # TODO: test if zarr store is accessible in the s3 bucket
     number_of_files = s3_manager.list_objects(bucket_name=output_bucket_name, prefix=f"level_1/{ship_name}/{cruise_name}/{sensor_name}/")
     # Ensure that all the files were uploaded properly
-    assert len(number_of_files) == 69
+    assert len(number_of_files) == 72
 
     # TODO: check the dynamodb dataframe to see if info is updated there
     # ---Verify Data is Populated in Table--- #

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/water_column_sonar_processing/aws/dynamodb_manager.py

@@ -111,17 +111,21 @@ class DynamoDBManager:
         expression_attribute_names,
         expression_attribute_values,
         update_expression,
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression,
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        assert response['ConsumedCapacity']['TableName'] == table_name
-        assert status_code == 200, "Problem, unable to update dynamodb table."
+    ): # TODO: convert to boolean
+        try:
+            response = self.__dynamodb_client.update_item(
+                TableName=table_name,
+                Key=key,
+                ExpressionAttributeNames=expression_attribute_names,
+                ExpressionAttributeValues=expression_attribute_values,
+                UpdateExpression=update_expression,
+            )
+            status_code = response["ResponseMetadata"]["HTTPStatusCode"]
+            print(f"HTTPStatusCode: {status_code}")
+            # assert status_code == 200, "Problem, unable to update dynamodb table."
+            # assert response['ConsumedCapacity']['TableName'] == table_name
+        except Exception as err:
+            print(f"Problem was encountered while updating item: {err}")
 
     #####################################################################
     # TODO: change to "get_cruise_as_df"

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/water_column_sonar_processing/processing/raw_to_zarr.py

@@ -9,7 +9,7 @@ from pathlib import Path # , PurePath
 
 from water_column_sonar_processing.aws import DynamoDBManager, S3Manager
 from water_column_sonar_processing.geometry import GeometryManager
-from water_column_sonar_processing.utility import Cleaner
+from water_column_sonar_processing.utility import Cleaner, PipelineStatus
 
 TEMPDIR = "/tmp"
 
@@ -53,10 +53,6 @@ class RawToZarr:
     ):
         print('Writing Zarr information to DynamoDB table.')
         dynamodb_manager = DynamoDBManager()
-
-        # The problem is that these values were never populated
-        # and so when the query looks for values that aren't there
-        # they fail
         dynamodb_manager.update_item(
             table_name=table_name,
             key={
@@ -87,7 +83,8 @@ class RawToZarr:
                 ":ma": {"N": str(np.round(max_echo_range, 4))},
                 ":mi": {"N": str(np.round(min_echo_range, 4))},
                 ":nd": {"N": str(num_ping_time_dropna)},
-                ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
+                # ":ps": {"S": "PROCESSING_RESAMPLE_AND_WRITE_TO_ZARR_STORE"},
+                ":ps": {"S": PipelineStatus.LEVEL_1_PROCESSING.name},
                 ":pt": {"S": datetime.now().isoformat(timespec="seconds") + "Z"},
                 ":se": {"S": sensor_name},
                 ":sh": {"S": ship_name},
@@ -113,6 +110,7 @@ class RawToZarr:
                 "#ZP = :zp"
             ),
         )
+        print('Done writing Zarr information to DynamoDB table.')
 
     ############################################################################
     ############################################################################
@@ -143,6 +141,7 @@ class RawToZarr:
     def raw_to_zarr(
             self,
             table_name,
+            input_bucket_name,
             output_bucket_name,
             ship_name,
             cruise_name,
@@ -157,6 +156,14 @@ class RawToZarr:
         geometry_manager = GeometryManager()
         cleaner = Cleaner()
         cleaner.delete_local_files(file_types=["*.zarr", "*.json"]) # TODO: include bot and raw?
+
+        s3_manager = S3Manager()
+        s3_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{raw_file_name}"
+        bottom_file_name = f"{Path(raw_file_name).stem}.bot"
+        s3_bottom_file_path = f"data/raw/{ship_name}/{cruise_name}/{sensor_name}/{bottom_file_name}"
+        s3_manager.download_file(bucket_name=input_bucket_name, key=s3_file_path, file_name=raw_file_name)
+        s3_manager.download_file(bucket_name=input_bucket_name, key=s3_bottom_file_path, file_name=bottom_file_name)
+
         try:
             gc.collect()
             print('Opening raw file with echopype.')
@@ -172,7 +179,12 @@ class RawToZarr:
             )
             print('Compute volume backscattering strength (Sv) from raw data.')
             ds_sv = ep.calibrate.compute_Sv(echodata)
-            print('Done computing volume backscattering strength (Sv) from raw data.')
+            print('Done computing volume backscatter strength (Sv) from raw data.')
+            # Note: detected_seafloor_depth is located at echodata.vendor.detected_seafloor_depth
+            # but is not written out with ds_sv
+            if "detected_seafloor_depth" in list(echodata.vendor.variables):
+                ds_sv["detected_seafloor_depth"] = echodata.vendor.detected_seafloor_depth
+            #
             frequencies = echodata.environment.frequency_nominal.values
             #################################################################
             # Get GPS coordinates
@@ -191,12 +203,9 @@ class RawToZarr:
             # TODO: this var name is supposed to represent minimum resolution of depth measurements
             # TODO revert this so that smaller diffs can be used
             # The most minimum the resolution can be is as small as 0.25 meters
-            min_echo_range = np.maximum(
-                0.25,
-                np.nanmin(np.diff(ds_sv.echo_range.values))
-            )
+            min_echo_range = np.round(np.nanmin(np.diff(ds_sv.echo_range.values)), 2)
             max_echo_range = float(np.nanmax(ds_sv.echo_range))
-            #
+            # This is the number of missing values found throughout the lat/lon
             num_ping_time_dropna = lat[~np.isnan(lat)].shape[0]  # symmetric to lon
             #
             start_time = np.datetime_as_string(ds_sv.ping_time.values[0], unit='ms') + "Z"
@@ -248,11 +257,12 @@ class RawToZarr:
             #######################################################################
             # TODO: verify count of objects matches, publish message, update status
             #######################################################################
-            print('here')
+            print('Finished raw-to-zarr conversion.')
         except Exception as err:
             print(f'Exception encountered creating local Zarr store with echopype: {err}')
             raise RuntimeError(f"Problem creating local Zarr store, {err}")
         finally:
+            print("Finally.")
             cleaner.delete_local_files(file_types=["*.raw", "*.bot", "*.zarr", "*.json"])
         print('Done creating local zarr store.')
 

{water_column_sonar_processing-0.0.10 → water_column_sonar_processing-0.0.11}/water_column_sonar_processing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.10
+Version: 0.0.11
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing