water-column-sonar-processing 0.0.4__tar.gz → 0.0.6__tar.gz

{water_column_sonar_processing-0.0.4/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.4
+Version: 0.0.6
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
@@ -11,6 +11,26 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aiobotocore~=2.9.0
+Requires-Dist: aiohttp~=3.10.10
+Requires-Dist: boto3==1.33.13
+Requires-Dist: botocore~=1.33.13
+Requires-Dist: echopype==0.9.0
+Requires-Dist: geopandas==1.0.1
+Requires-Dist: mock~=5.1.0
+Requires-Dist: moto~=5.0.18
+Requires-Dist: numcodecs==0.13.1
+Requires-Dist: numpy==1.26.4
+Requires-Dist: pandas==2.2.3
+Requires-Dist: pytest~=8.3.3
+Requires-Dist: python-dotenv==1.0.0
+Requires-Dist: requests==2.32.3
+Requires-Dist: s3fs==2024.10.0
+Requires-Dist: scipy==1.14.1
+Requires-Dist: shapely==2.0.3
+Requires-Dist: typing-extensions==4.10.0
+Requires-Dist: xarray==2022.12.0
+Requires-Dist: zarr==2.18.3
 
 # Water Column Sonar Processing
 Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
@@ -72,6 +92,8 @@ https://packaging.python.org/en/latest/tutorials/packaging-projects/
 # To Publish To TEST
 ```commandline
 python -m build
+# python -m build --sdist
+# python -m build --wheel
 python -m twine upload --repository testpypi dist/*
 pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
 python
@@ -87,7 +109,15 @@ python -m build
 python -m twine upload --repository pypi dist/*
 ```
 
+# Pre Commit Hook
+https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
+```
+pre-commit install --allow-missing-config
+```
 
 # Linting
 Ruff
 https://plugins.jetbrains.com/plugin/20574-ruff
+
+# Colab Test
+https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.6}/README.md

@@ -58,6 +58,8 @@ https://packaging.python.org/en/latest/tutorials/packaging-projects/
 # To Publish To TEST
 ```commandline
 python -m build
+# python -m build --sdist
+# python -m build --wheel
 python -m twine upload --repository testpypi dist/*
 pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
 python
@@ -73,7 +75,15 @@ python -m build
 python -m twine upload --repository pypi dist/*
 ```
 
+# Pre Commit Hook
+https://dev.to/rafaelherik/using-trufflehog-and-pre-commit-hook-to-prevent-secret-exposure-edo
+```
+pre-commit install --allow-missing-config
+```
 
 # Linting
 Ruff
-https://plugins.jetbrains.com/plugin/20574-ruff
+https://plugins.jetbrains.com/plugin/20574-ruff
+
+# Colab Test
+https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scrollTo=AayVyvpBdfIZ

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.6}/pyproject.toml

@@ -1,10 +1,13 @@
 [build-system]
-requires = ["setuptools", "wheel"]
+requires = [
+    "setuptools>=61.0",
+    "wheel >= 0.29.0",
+]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "water_column_sonar_processing"
-version = "0.0.4"
+version = "0.0.6"
 authors = [
   { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
 ]
@@ -16,6 +19,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
+dynamic = ["dependencies"]
 
 [project.urls]
 Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
@@ -27,7 +31,13 @@ Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "-ra -q"
+#addopts = "-ra -q"
 testpaths = [
     "tests",
-]
+]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies = {dev = { file = ["requirements-dev.txt"] }}
+
+# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/

water_column_sonar_processing-0.0.6/requirements.txt

@@ -0,0 +1,27 @@
+# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
+# defined for Python 3.12
+# Note: be careful with conversions for pandas >=2.0.0, timestamps will have a lot of problems
+
+aiobotocore~=2.9.0
+aiohttp~=3.10.10
+boto3==1.33.13
+botocore~=1.33.13
+echopype==0.9.0
+#fiona==1.10.1
+geopandas==1.0.1
+mock~=5.1.0
+moto~=5.0.18
+numcodecs==0.13.1
+numpy==1.26.4
+pandas==2.2.3
+pytest~=8.3.3
+python-dotenv==1.0.0
+requests==2.32.3
+#s3fs==2023.12.2
+s3fs==2024.10.0
+scipy==1.14.1
+shapely==2.0.3
+typing-extensions==4.10.0
+xarray==2022.12.0
+#zarr==2.16.1
+zarr==2.18.3

water_column_sonar_processing-0.0.6/src/water_column_sonar_processing/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import absolute_import
+
+from . import aws, cruise, geometry, index, model, utility, process
+from .model import ZarrManager
+from .process import Process
+
+__all__ = [
+    "aws",
+    "cruise",
+    "geometry",
+    "index",
+    "model",
+    "utility",
+    "process",
+    "Process",
+]

water_column_sonar_processing-0.0.6/src/water_column_sonar_processing/aws/__init__.py

@@ -0,0 +1,7 @@
+from .dynamodb_manager import DynamoDBManager
+from .s3_manager import S3Manager
+from .s3fs_manager import S3FSManager
+from .sns_manager import SNSManager
+from .sqs_manager import SQSManager
+
+__all__ = ["DynamoDBManager", "S3Manager", "S3FSManager", "SNSManager", "SQSManager"]

{water_column_sonar_processing-0.0.4/src/aws_manager → water_column_sonar_processing-0.0.6/src/water_column_sonar_processing/aws}/dynamodb_manager.py

@@ -1,7 +1,8 @@
 import os
+
 import boto3
 import pandas as pd
-from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
+from boto3.dynamodb.types import TypeDeserializer, TypeSerializer
 
 
 #########################################################################
@@ -9,9 +10,9 @@ class DynamoDBManager:
     #####################################################################
     def __init__(self):
         self.__dynamodb_session = boto3.Session(
-            aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
-            aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
-            region_name=os.environ.get("AWS_REGION", default="us-east-1")
+            aws_access_key_id=os.environ.get("ACCESS_KEY_ID"),
+            aws_secret_access_key=os.environ.get("SECRET_ACCESS_KEY"),
+            region_name=os.environ.get("AWS_REGION", default="us-east-1"),
         )
         self.__dynamodb_resource = self.__dynamodb_session.resource(
             service_name="dynamodb",
@@ -23,7 +24,7 @@ class DynamoDBManager:
         self.type_deserializer = TypeDeserializer()
 
     #####################################################################
-    ### defined in raw-to-zarr_manager, not used
+    ### defined in raw-to-model, not used
     # def put_item(
     #         self,
     #         table_name,
@@ -35,10 +36,10 @@ class DynamoDBManager:
 
     #####################################################################
     def create_table(
-            self,
-            table_name,
-            key_schema,
-            attribute_definitions,
+        self,
+        table_name,
+        key_schema,
+        attribute_definitions,
     ):
         self.__dynamodb_client.create_table(
             AttributeDefinitions=attribute_definitions,
@@ -52,98 +53,118 @@ class DynamoDBManager:
         )
 
     #####################################################################
-    def get_item(
-            self,
-            table_name,
-            key
+    def create_water_column_sonar_table(
+        self,
+        table_name,
     ):
+        self.create_table(
+            table_name=table_name,
+            key_schema=[
+                {
+                    "AttributeName": "FILE_NAME",
+                    "KeyType": "HASH",
+                },
+                {
+                    "AttributeName": "CRUISE_NAME",
+                    "KeyType": "RANGE",
+                },
+            ],
+            attribute_definitions=[
+                {"AttributeName": "FILE_NAME", "AttributeType": "S"},
+                {"AttributeName": "CRUISE_NAME", "AttributeType": "S"},
+            ],
+        )
+
+    #####################################################################
+    def get_item(self, table_name, key):
         response = self.__dynamodb_client.get_item(TableName=table_name, Key=key)
         item = None
-        if response['ResponseMetadata']['HTTPStatusCode'] == 200:
-            if 'Item' in response:
-                item = response['Item']
+        if response["ResponseMetadata"]["HTTPStatusCode"] == 200:
+            if "Item" in response:
+                item = response["Item"]
         return item
 
     #####################################################################
     def update_item(
-            self,
-            table_name,
-            key,
-            expression_attribute_names,
-            expression_attribute_values,
-            update_expression
+        self,
+        table_name,
+        key,
+        expression_attribute_names,
+        expression_attribute_values,
+        update_expression,
     ):
         response = self.__dynamodb_client.update_item(
             TableName=table_name,
             Key=key,
             ExpressionAttributeNames=expression_attribute_names,
             ExpressionAttributeValues=expression_attribute_values,
-            UpdateExpression=update_expression
+            UpdateExpression=update_expression,
         )
-        status_code = response['ResponseMetadata']['HTTPStatusCode']
+        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
         # TODO: change to exception
-        assert (status_code == 200), "Problem, unable to update dynamodb table."
+        assert status_code == 200, "Problem, unable to update dynamodb table."
 
     #####################################################################
     def get_table_as_df(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        table_name,
     ):
         expression_attribute_values = {
-            ':cr': {'S': cruise_name},
-            ':se': {'S': sensor_name},
-            ':sh': {'S': ship_name},
+            ":cr": {"S": cruise_name},
+            ":se": {"S": sensor_name},
+            ":sh": {"S": ship_name},
         }
 
-        filter_expression = 'CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh'
+        filter_expression = (
+            "CRUISE_NAME = :cr and SENSOR_NAME = :se and SHIP_NAME = :sh"
+        )
         response = self.__dynamodb_client.scan(
             TableName=table_name,
-            Select='ALL_ATTRIBUTES',
+            Select="ALL_ATTRIBUTES",
             ExpressionAttributeValues=expression_attribute_values,
             FilterExpression=filter_expression,
         )
         # Note: table.scan() has 1 MB limit on results so pagination is used
-        data = response['Items']
+        data = response["Items"]
 
-        while 'LastEvaluatedKey' in response:
+        while "LastEvaluatedKey" in response:
             response = self.__dynamodb_client.scan(
                 TableName=table_name,
-                Select='ALL_ATTRIBUTES',
+                Select="ALL_ATTRIBUTES",
                 ExpressionAttributeValues=expression_attribute_values,
                 FilterExpression=filter_expression,
-                ExclusiveStartKey=response['LastEvaluatedKey']
+                ExclusiveStartKey=response["LastEvaluatedKey"],
             )
-            data.extend(response['Items'])
+            data.extend(response["Items"])
 
         deserializer = self.type_deserializer
         df = pd.DataFrame([deserializer.deserialize({"M": i}) for i in data])
 
-        return df.sort_values(by='START_TIME', ignore_index=True)
+        return df.sort_values(by="START_TIME", ignore_index=True)
 
     #####################################################################
     # is this used?
     def get_table_item(
-            self,
-            table_name,
-            key,
+        self,
+        table_name,
+        key,
     ):
         # a bit more high level, uses resource to get table item
         table = self.__dynamodb_resource.Table(table_name)
-        response = table.get_item(
-            Key=key
-        )
+        response = table.get_item(Key=key)
         return response
 
     #####################################################################
     # TODO: add helper method to delete the data
     def delete_cruise(
-            self,
-            table_name,
-            cruise_name,
+        self,
+        table_name,
+        cruise_name,
     ):
         pass
 
+
 #########################################################################