water-column-sonar-processing 0.0.1__py3-none-any.whl → 25.11.1__py3-none-any.whl

{model → water_column_sonar_processing}/utility/pipeline_status.py

@@ -16,13 +16,17 @@ class PipelineStatus(Flag):
     PROCESSING_RAW_PROCESSOR = auto()
     FAILURE_RAW_PROCESSOR = auto()
     SUCCESS_RAW_PROCESSOR = auto()
-    RAW_PROCESSOR = PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
+    RAW_PROCESSOR = (
+        PROCESSING_RAW_PROCESSOR | FAILURE_RAW_PROCESSOR | SUCCESS_RAW_PROCESSOR
+    )
     #
     # RAW_AGGREGATOR --> AGGREGATOR (Scatter-Gather EIP)
     PROCESSING_RAW_AGGREGATOR = auto()
     FAILURE_RAW_AGGREGATOR = auto()
     SUCCESS_RAW_AGGREGATOR = auto()
-    RAW_AGGREGATOR = PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
+    RAW_AGGREGATOR = (
+        PROCESSING_RAW_AGGREGATOR | FAILURE_RAW_AGGREGATOR | SUCCESS_RAW_AGGREGATOR
+    )
     #
     LEVEL_1_PROCESSING = RAW_SPLITTER | RAW_PROCESSOR | RAW_AGGREGATOR
     #
@@ -35,27 +39,43 @@ class PipelineStatus(Flag):
     PROCESSING_CRUISE_INITIALIZER = auto()
     FAILURE_CRUISE_INITIALIZER = auto()
     SUCCESS_CRUISE_INITIALIZER = auto()
-    CRUISE_INITIALIZER = PROCESSING_CRUISE_INITIALIZER | FAILURE_CRUISE_INITIALIZER | SUCCESS_CRUISE_INITIALIZER
+    CRUISE_INITIALIZER = (
+        PROCESSING_CRUISE_INITIALIZER
+        | FAILURE_CRUISE_INITIALIZER
+        | SUCCESS_CRUISE_INITIALIZER
+    )
     #
     # CRUISE_SPLITTER --> SPLITTER
     PROCESSING_CRUISE_SPLITTER = auto()
     FAILURE_CRUISE_SPLITTER = auto()
     SUCCESS_CRUISE_SPLITTER = auto()
-    CRUISE_SPLITTER = PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
+    CRUISE_SPLITTER = (
+        PROCESSING_CRUISE_SPLITTER | FAILURE_CRUISE_SPLITTER | SUCCESS_CRUISE_SPLITTER
+    )
     #
     # CRUISE_PROCESSOR --> PROCESSOR <-- Note: these need to run sequentially now
     PROCESSING_CRUISE_PROCESSOR = auto()
     FAILURE_CRUISE_PROCESSOR = auto()
     SUCCESS_CRUISE_PROCESSOR = auto()
-    CRUISE_PROCESSOR = PROCESSING_CRUISE_PROCESSOR | FAILURE_CRUISE_PROCESSOR | SUCCESS_CRUISE_PROCESSOR
+    CRUISE_PROCESSOR = (
+        PROCESSING_CRUISE_PROCESSOR
+        | FAILURE_CRUISE_PROCESSOR
+        | SUCCESS_CRUISE_PROCESSOR
+    )
     #
     # CRUISE_AGGREGATOR --> AGGREGATOR
     PROCESSING_CRUISE_AGGREGATOR = auto()
     FAILURE_CRUISE_AGGREGATOR = auto()
     SUCCESS_CRUISE_AGGREGATOR = auto()
-    CRUISE_AGGREGATOR = PROCESSING_CRUISE_AGGREGATOR | FAILURE_CRUISE_AGGREGATOR | SUCCESS_CRUISE_AGGREGATOR
+    CRUISE_AGGREGATOR = (
+        PROCESSING_CRUISE_AGGREGATOR
+        | FAILURE_CRUISE_AGGREGATOR
+        | SUCCESS_CRUISE_AGGREGATOR
+    )
     #
-    LEVEL_2_PROCESSING = CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
+    LEVEL_2_PROCESSING = (
+        CRUISE_INITIALIZER | CRUISE_SPLITTER | CRUISE_PROCESSOR | CRUISE_AGGREGATOR
+    )
     #
     # --------------------------------------------------- #
     # --- Level 3 Data ---------------------------------- #
@@ -64,35 +84,38 @@ class PipelineStatus(Flag):
     PROCESSING_TILE_PROCESSOR = auto()
     FAILURE_TILE_PROCESSOR = auto()
     SUCCESS_TILE_PROCESSOR = auto()
-    TILE_PROCESSOR = PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
+    TILE_PROCESSOR = (
+        PROCESSING_TILE_PROCESSOR | FAILURE_TILE_PROCESSOR | SUCCESS_TILE_PROCESSOR
+    )
     #
     # GEOHASH_PROCESSOR
     PROCESSING_GEOHASH_PROCESSOR = auto()
     FAILURE_GEOHASH_PROCESSOR = auto()
     SUCCESS_GEOHASH_PROCESSOR = auto()
-    GEOHASH_PROCESSOR = PROCESSING_GEOHASH_PROCESSOR | FAILURE_GEOHASH_PROCESSOR | SUCCESS_GEOHASH_PROCESSOR
+    GEOHASH_PROCESSOR = (
+        PROCESSING_GEOHASH_PROCESSOR
+        | FAILURE_GEOHASH_PROCESSOR
+        | SUCCESS_GEOHASH_PROCESSOR
+    )
     #
     LEVEL_3_PROCESSING = TILE_PROCESSOR | GEOHASH_PROCESSOR
     # --------------------------------------------------- #
     # --------------------------------------------------- #
 
+
 # Status.PROCESSING_RAW_AGGREGATOR in Status.LEVEL_1_PROCESSING
 # Status.LEVEL_1_PROCESSING.value < Status.LEVEL_2_PROCESSING.value
 
 # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudformation-stack.html
-"""
-CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
-ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
-DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
-UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
-UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
-UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
-REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
-IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
-
-failure - noun - 
-failed - verb - "verbs should be avoided"
-
-success - noun
 
-"""
+# CREATE_IN_PROGRESS | CREATE_FAILED | CREATE_COMPLETE |
+# ROLLBACK_IN_PROGRESS | ROLLBACK_FAILED | ROLLBACK_COMPLETE |
+# DELETE_IN_PROGRESS | DELETE_FAILED | DELETE_COMPLETE |
+# UPDATE_IN_PROGRESS | UPDATE_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_COMPLETE |
+# UPDATE_FAILED | UPDATE_ROLLBACK_IN_PROGRESS | UPDATE_ROLLBACK_FAILED |
+# UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS | UPDATE_ROLLBACK_COMPLETE |
+# REVIEW_IN_PROGRESS | IMPORT_IN_PROGRESS | IMPORT_COMPLETE |
+# IMPORT_ROLLBACK_IN_PROGRESS | IMPORT_ROLLBACK_FAILED | IMPORT_ROLLBACK_COMPLETE
+# failure - noun -
+# failed - verb - "verbs should be avoided"
+# success - noun

water_column_sonar_processing/utility/timestamp.py

@@ -0,0 +1,12 @@
+import datetime
+
+
+###########################################################
+class Timestamp:
+    @staticmethod
+    def get_timestamp():
+        # return timestamp in form: '2024-03-29T19:36:52.433Z'
+        return f"{datetime.datetime.now(datetime.UTC).isoformat()[:23]}Z"
+
+
+###########################################################

water_column_sonar_processing-25.11.1.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: water-column-sonar-processing
+Version: 25.11.1
+Summary: Processing tool for water column sonar data.
+Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
+Maintainer-email: Rudy Klucik <rudy.klucik@noaa.gov>
+License-Expression: MIT
+Project-URL: Homepage, https://echo.fish
+Project-URL: Repository, https://github.com/CI-CMG/water-column-sonar-processing
+Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
+Keywords: ocean,sonar,water column,zarr
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bandit==1.8.6
+Requires-Dist: black>=25.11.0
+Requires-Dist: boto3==1.40.49
+Requires-Dist: boto3-stubs==1.40.49
+Requires-Dist: botocore==1.40.49
+Requires-Dist: build>=1.3.0
+Requires-Dist: echopype==0.11.0
+Requires-Dist: fiona==1.10.1
+Requires-Dist: geopandas==1.1.1
+Requires-Dist: importlib
+Requires-Dist: mock==5.2.0
+Requires-Dist: moto[all]==5.1.16
+Requires-Dist: moto[server]==5.1.16
+Requires-Dist: multidict==6.7.0
+Requires-Dist: netcdf4==1.7.3
+Requires-Dist: numpy==2.3.4
+Requires-Dist: pandas==2.3.3
+Requires-Dist: pandas-stubs==2.3.2.250926
+Requires-Dist: pooch==1.8.2
+Requires-Dist: pyarrow==22.0.0
+Requires-Dist: pykalman==0.10.2
+Requires-Dist: python-dotenv==1.2.1
+Requires-Dist: requests==2.32.5
+Requires-Dist: s3fs==2025.10.0
+Requires-Dist: scipy==1.16.3
+Requires-Dist: setuptools==80.9.0
+Requires-Dist: shapely==2.1.2
+Requires-Dist: xarray==2025.10.1
+Requires-Dist: zarr==3.1.3
+Dynamic: license-file
+
+# Water Column Sonar Processing
+
+Processing tool for converting Level_0 water column sonar data to Level_1 and Level_2 derived data sets as well as
+generating geospatial information.
+
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/CI-CMG/water-column-sonar-processing/test_action.yaml)
+![PyPI - Implementation](https://img.shields.io/pypi/v/water-column-sonar-processing) ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/CI-CMG/water-column-sonar-processing) ![GitHub repo size](https://img.shields.io/github/repo-size/CI-CMG/water-column-sonar-processing)
+
+# Setting up the Python Environment
+
+> Python 3.12.12
+
+# Installing Dependencies
+
+```
+source .venv/bin/activate
+# or ".venv\Scripts\activate" in windows
+
+uv pip install --upgrade pip
+
+uv sync --all-groups
+
+uv run pre-commit install
+```
+
+# Pytest
+
+```
+uv run pytest --cache-clear tests # -W ignore::DeprecationWarning
+```
+
+or
+> pytest --cache-clear --cov=src tests/ --cov-report=xml
+
+# Instructions
+
+Following this tutorial:
+https://packaging.python.org/en/latest/tutorials/packaging-projects/
+
+# Pre Commit Hook
+
+see here for installation: https://pre-commit.com/
+https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
+
+```
+uv run pre-commit install --allow-missing-config
+# or
+uv run pre-commit install
+```
+
+# Black
+
+https://ljvmiranda921.github.io/notebook/2018/06/21/precommits-using-black-and-flake8/
+
+```
+Settings > Black
+Execution mode: Package
+Python Interpreter: .../.venv/bin/python
+Use Black Formatter: X On Code reformat, X On Save
+```
+
+# Linting
+
+Ruff
+https://plugins.jetbrains.com/plugin/20574-ruff
+
+# Colab Test
+
+https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ
+
+# Test Coverage
+
+TODO
+
+# Tag a Release
+
+Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
+
+```commandline
+git tag -a v25.11.1 -m "Releasing v25.11.1"
+git push origin --tags
+```
+
+# To Publish To PROD
+
+```
+uv build --no-sources
+uv publish
+```
+
+# TODO:
+
+add https://pypi.org/project/setuptools-scm/
+for extracting the version
+
+# Security scanning
+
+> bandit -r water_column_sonar_processing/
+
+# Data Debugging
+
+Experimental Plotting in Xarray (hvPlot):
+https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
+
+HB0707 Zoomable Cruise:
+https://hb0707.s3.us-east-1.amazonaws.com/index.html
+
+# UV Debugging
+
+```
+uv pip install --upgrade pip
+uv sync --all-groups
+uv run pre-commit install
+uv lock --check
+uv lock
+uv sync --all-groups
+uv run pytest --cache-clear tests
+```
+
+# Fixing S3FS Problems
+
+```commandline
+To enable/disa asyncio for the debugger, follow the steps:
+Open PyCharm
+Use Shift + Shift (Search Everywhere)
+In the popup type: Registry and press Enter
+Find "Registry" in the list of results and click on it.
+In the new popup find python.debug.asyncio.repl line and check the respective checkbox
+Press Close.
+Restart the IDE.
+The asyncio support will be enabled in the debugger.
+```
+
+# Fixing windows/wsl/ubuntu/mac git compatability
+
+> git config --global core.filemode false
+> git config --global core.autocrlf true

water_column_sonar_processing-25.11.1.dist-info/RECORD

@@ -0,0 +1,34 @@
+water_column_sonar_processing/__init__.py,sha256=fvRK4uFo_A0l7w_T4yckvDqJ3wMUq4JB3VVPXqWfewE,226
+water_column_sonar_processing/process.py,sha256=eatdHHc7_mPapTjxsOyD24OlEWeqj56MmVTMPjxF2QM,5290
+water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
+water_column_sonar_processing/aws/dynamodb_manager.py,sha256=EwSc4UTVQTR6tXsrx2eNdbhvexxfrz3GkkvO97F_9fM,14017
+water_column_sonar_processing/aws/s3_manager.py,sha256=ZuShUfg4hx7XUWRiSywHJ3m0Szl699Bd8AfPualbuhs,16299
+water_column_sonar_processing/aws/s3fs_manager.py,sha256=vl_NbekHoAMv81wtx3QaNOFmv0erxAtOyD5t_RLUqCA,2546
+water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
+water_column_sonar_processing/aws/sqs_manager.py,sha256=j_DGOKble3i-DlHT_uGxCFEmHVbYhFrpbhXdJKLtBSo,1600
+water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
+water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=IRPsEEAZ5C1XKlQYxNuYbPd7yrvN2aO_RqGBkRLKo90,7987
+water_column_sonar_processing/cruise/datatree_manager.py,sha256=ed_lY04MyaXKADl7EaqHyEZ8JQMiyIAXVhvtE7YnWcg,662
+water_column_sonar_processing/cruise/resample_regrid.py,sha256=wsO1_RwBlxRJ2QeAmvSBNN6Q5sOtS6WI8ADuEAd6rQU,15197
+water_column_sonar_processing/geometry/__init__.py,sha256=KNhgfKimxuMtyZwMQlIc_OTtEkYXNXl1WyvfNJThlWY,306
+water_column_sonar_processing/geometry/elevation_manager.py,sha256=Wi7xl6LFsxqUPGbrquqtPEKnOMpl3dMLJtEb7DR5o1c,4249
+water_column_sonar_processing/geometry/geometry_manager.py,sha256=dmPdFjeGXfh-bhdRSNVr1pBjnSlcjFUGwXIwXmq4Fe0,10986
+water_column_sonar_processing/geometry/line_simplification.py,sha256=adnFKOYby6Z37Io-6UZR-9QPxhWGC4IU1EtNpphQQ-w,9294
+water_column_sonar_processing/geometry/pmtile_generation.py,sha256=kZPG5oskCqsVyCr97c9nlhgNBkLtu_-gOlB2GKvtQhA,9462
+water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
+water_column_sonar_processing/index/index_manager.py,sha256=lW4FAczAih3e1nwlYhwL_-idLx2cLYRgTEwR2KgMQ2Q,16124
+water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
+water_column_sonar_processing/model/zarr_manager.py,sha256=PNgvxEJ3efidqNqTX2TLaX6YVESKOHnJz7mLpzShruQ,32971
+water_column_sonar_processing/processing/__init__.py,sha256=TDnA4_Nyb8MnU8HaixaCzEdTbFkXsUXA0ekvxtYm1-U,150
+water_column_sonar_processing/processing/raw_to_netcdf.py,sha256=3UdglrWXFLxa_jHkhvC9P9NUkp-iKmp6MvnXFnXi4ns,13098
+water_column_sonar_processing/processing/raw_to_zarr.py,sha256=EfB_JWiatlAQKc17jDR1GD6k7txlKpQPFI4HknlFuoA,18826
+water_column_sonar_processing/utility/__init__.py,sha256=5gV8sJcnQGrV2AecwkBFZljOC-fQwLuDX70DxX59x2o,289
+water_column_sonar_processing/utility/cleaner.py,sha256=q9xx4NeVhaxQgGCCYyZv29PxvMHfD9txbnMahOccF8U,607
+water_column_sonar_processing/utility/constants.py,sha256=R0dF68O4I8mlOyZbI1nJuU8_cDMCcHC9fu10rMPtTpA,3390
+water_column_sonar_processing/utility/pipeline_status.py,sha256=xbl6-4ePq1Krfo18Mgr0jzWz9C_g2Kbey_QZNJuwdkI,4406
+water_column_sonar_processing/utility/timestamp.py,sha256=rn8SDbGYjRvanDUOrvKpNjp3-AK6-KlU2NaCk6Ok8rc,337
+water_column_sonar_processing-25.11.1.dist-info/licenses/LICENSE,sha256=TosqaZpJgYvhgXIyYBti-ggJaO8rxRg3FtThY08s9Aw,1110
+water_column_sonar_processing-25.11.1.dist-info/METADATA,sha256=ytZEScINaRFxfYJibro9GHeNfCtgUm8xiwe9nCe1ico,4718
+water_column_sonar_processing-25.11.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+water_column_sonar_processing-25.11.1.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
+water_column_sonar_processing-25.11.1.dist-info/RECORD,,

{water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-25.11.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{water_column_sonar_processing-0.0.1.dist-info → water_column_sonar_processing-25.11.1.dist-info/licenses}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Cooperative Institutes, Coastal and Marine Geophysics
+Copyright (c) 2025 Cooperative Institutes, Coastal and Marine Geophysics
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

water_column_sonar_processing-25.11.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+water_column_sonar_processing

model/aws/dynamodb_manager.py

@@ -1,149 +0,0 @@
-import os
-import boto3
-import pandas as pd
-from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
-
-
-#########################################################################
-class DynamoDBManager:
-    #####################################################################
-    def __init__(self):
-        self.__dynamodb_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=os.environ.get("AWS_REGION", default="us-east-1")
-        )
-        self.__dynamodb_resource = self.__dynamodb_session.resource(
-            service_name="dynamodb",
-        )
-        self.__dynamodb_client = self.__dynamodb_session.client(
-            service_name="dynamodb",
-        )
-        self.type_serializer = TypeSerializer()  # https://stackoverflow.com/a/46738251
-        self.type_deserializer = TypeDeserializer()
-
-    #####################################################################
-    ### defined in raw-to-zarr, not used
-    # def put_item(
-    #         self,
-    #         table_name,
-    #         item
-    # ):
-    #     response = boto3.Session().client(service_name='dynamodb').put_item(TableName=table_name, Item=item)
-    #     status_code = response['ResponseMetadata']['HTTPStatusCode']
-    #     assert (status_code == 200), "Problem, unable to update dynamodb table."
-
-    #####################################################################
-    def create_table(
-            self,
-            table_name,
-            key_schema,
-            attribute_definitions,
-    ):
-        self.__dynamodb_client.create_table(
-            AttributeDefinitions=attribute_definitions,
-            TableName=table_name,
-            KeySchema=key_schema,
-            BillingMode="PAY_PER_REQUEST",  # "PROVISIONED",
-            # ProvisionedThroughput={
-            #     'ReadCapacityUnits': 1_000,
-            #     'WriteCapacityUnits': 1_000
-            # }
-        )
-
-    #####################################################################
-    def get_item(
-            self,
-            table_name,
-            key
-    ):
-        response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
-        item = None
-        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
-            if 'Item' in response:
-                item = response['Item']
-        return item
-
-    #####################################################################
-    def update_item(
-            self,
-            table_name,
-            key,
-            expression_attribute_names,
-            expression_attribute_values,
-            update_expression
-    ):
-        response = self.__dynamodb_client.update_item(
-            TableName=table_name,
-            Key=key,
-            ExpressionAttributeNames=expression_attribute_names,
-            ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression
-        )
-        status_code = response['ResponseMetadata']['HTTPStatusCode']
-        # TODO: change to exception
-        assert (status_code == 200), "Problem, unable to update dynamodb table."
-
-    #####################################################################
-    def get_table_as_df(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
-    ):
-        expression_attribute_values = {
-            ':cr': {'S': cruise_name},
-            ':se': {'S': sensor_name},
-            ':sh': {'S': ship_name},
-        }
-
-        filter_expression = 'CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh'
-        response = self.__dynamodb_client.scan(
-            TableName=table_name,
-            Select='ALL_ATTRIBUTES',
-            ExpressionAttributeValues=expression_attribute_values,
-            FilterExpression=filter_expression,
-        )
-        # Note: table.scan() has 1 MB limit on results so pagination is used
-        data = response['Items']
-
-        while 'LastEvaluatedKey' in response:
-            response = self.__dynamodb_client.scan(
-                TableName=table_name,
-                Select='ALL_ATTRIBUTES',
-                ExpressionAttributeValues=expression_attribute_values,
-                FilterExpression=filter_expression,
-                ExclusiveStartKey=response['LastEvaluatedKey']
-            )
-            data.extend(response['Items'])
-
-        deserializer = self.type_deserializer
-        df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
-
-        return df.sort_values(by='START_TIME', ignore_index=True)
-
-    #####################################################################
-    # is this used?
-    def get_table_item(
-            self,
-            table_name,
-            key,
-    ):
-        # a bit more high level, uses resource to get table item
-        table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(
-            Key=key
-        )
-        return response
-
-    #####################################################################
-    # TODO: add helper method to delete the data
-    def delete_cruise(
-            self,
-            table_name,
-            cruise_name,
-    ):
-        pass
-
-#########################################################################