water-column-sonar-processing 25.1.3__tar.gz → 25.1.4__tar.gz

water_column_sonar_processing-25.1.4/.github/workflows/test_action.yaml

@@ -0,0 +1,46 @@
+name: Testing
+
+on: [push]
+
+jobs:
+#  build:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out
+#        uses: actions/checkout@v4
+#      - name: Set up Python
+#        uses: actions/setup-python@v5
+#        with:
+#          # Semantic version range syntax or exact version of a Python version
+#          python-version: '3.10'
+#          # Optional - x64 or x86 architecture, defaults to x64
+##          architecture: 'x64'
+#          cache: 'pip'
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install -r requirements_dev.txt
+#      - name: Run the tests
+#        run: python -m pytest
+
+  build:
+    name: python
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.25"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --all-extras --dev
+
+      - name: Run tests
+        run: uv run pytest tests

water_column_sonar_processing-25.1.4/.python-version

@@ -0,0 +1 @@
+3.10.12

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.2
 Name: water_column_sonar_processing
-Version: 25.1.3
-Summary: A processing tool for water column sonar data.
+Version: 25.1.4
+Summary: Processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
 Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiobotocore==2.19.0
@@ -34,6 +34,19 @@ Requires-Dist: typing-extensions==4.10.0
 Requires-Dist: xarray==2024.10.0
 Requires-Dist: xbatcher==0.4.0
 Requires-Dist: zarr==2.18.3
+Provides-Extra: dev
+Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pyinstaller; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: flake8==7.1.1; extra == "dev"
+Requires-Dist: pooch==1.8.2; extra == "dev"
+Requires-Dist: pytest~=8.3.3; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
 
 # Water Column Sonar Processing
 Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
@@ -80,14 +93,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
      3. Set interpreter 
 
 # Installing Dependencies
-
-  1. Add dependencies with versions to requirements.txt
-  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
+```
+uv pip install --upgrade pip
+#uv pip install -r requirements_dev.txt
+uv pip install -r pyproject.toml --extra dev
+```
 
 
 # Pytest
 ```commandline
-pytest --disable-warnings
+uv run pytest tests
+#pytest --disable-warnings
 ```
 or
 > pytest --cache-clear --cov=src tests/ --cov-report=xml
@@ -120,7 +136,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 # Tag a Release
 Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
 ```commandline
-git tag -a v25.1.2 -m "Releasing version v25.1.2"
+git tag -a v25.1.4 -m "Releasing version v25.1.4"
 git push origin --tags
 ```
 

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/README.md

@@ -43,14 +43,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
      3. Set interpreter 
 
 # Installing Dependencies
-
-  1. Add dependencies with versions to requirements.txt
-  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
+```
+uv pip install --upgrade pip
+#uv pip install -r requirements_dev.txt
+uv pip install -r pyproject.toml --extra dev
+```
 
 
 # Pytest
 ```commandline
-pytest --disable-warnings
+uv run pytest tests
+#pytest --disable-warnings
 ```
 or
 > pytest --cache-clear --cov=src tests/ --cov-report=xml
@@ -83,7 +86,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 # Tag a Release
 Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
 ```commandline
-git tag -a v25.1.2 -m "Releasing version v25.1.2"
+git tag -a v25.1.4 -m "Releasing version v25.1.4"
 git push origin --tags
 ```
 
@@ -105,4 +108,4 @@ Experimental Plotting in Xarray (hvPlot):
 https://colab.research.google.com/drive/18vrI9LAip4xRGEX6EvnuVFp35RAiVYwU#scrollTo=q9_j9p2yXsLV
 
 HB0707 Cruise zoomable:
-https://hb0707.s3.us-east-1.amazonaws.com/index.html
+https://hb0707.s3.us-east-1.amazonaws.com/index.html

water_column_sonar_processing-25.1.4/pyproject.toml

@@ -0,0 +1,73 @@
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel >= 0.29.0",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "water_column_sonar_processing"
+version = "25.1.4"
+authors = [
+  { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
+]
+description = "Processing tool for water column sonar data."
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    "aiobotocore==2.19.0",
+    "boto3==1.36.3",
+    "botocore==1.36.3",
+    "echopype==0.9.0",
+    "fiona==1.10.1",
+    "geopandas==1.0.1",
+    "mock==5.1.0",
+    "moto[all]==5.0.27",
+    "moto[server]==5.0.27",
+    "numcodecs==0.13.1",
+    "numpy==1.26.4",
+    "pandas==2.2.3",
+    "pyarrow==18.1.0",
+    "python-dotenv==1.0.1",
+    "requests==2.32.3",
+    "s3fs==2024.2.0",
+    "scipy==1.14.1",
+    "setuptools",
+    "shapely==2.0.3",
+    "typing-extensions==4.10.0",
+    "xarray==2024.10.0",
+    "xbatcher==0.4.0",
+    "zarr==2.18.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "bandit[toml]==1.8.0",
+    "build",
+    "pre-commit",
+    "pyinstaller",
+    "twine",
+    "flake8==7.1.1",
+    "pooch==1.8.2",
+    "pytest~=8.3.3",
+    "tqdm",
+    "bandit"
+]
+test = [
+    "pytest-cov",
+]
+
+[project.urls]
+Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
+Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+[tool.pre-commit-hooks.bandit]
+exclude = ["*/tests/*"]

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/aws/s3fs_manager.py

@@ -16,7 +16,7 @@ class S3FSManager:
         # self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3fs = s3fs.S3FileSystem(
-            asynchronous=False,
+            # asynchronous=False,
             endpoint_url=endpoint_url,
             key=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             secret=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/cruise/resample_regrid.py

@@ -197,9 +197,9 @@ class ResampleRegrid:
                 #  df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
 
                 # Get index from all cruise files. Note: should be based on which are included in cruise.
-                index = cruise_df.index[
+                index = int(cruise_df.index[
                     cruise_df["FILE_NAME"] == f"{file_name_stem}.raw"
-                ][0]
+                ][0])
 
                 # get input store
                 input_xr_zarr_store = zarr_manager.open_s3_zarr_store_with_xarray(
@@ -226,18 +226,20 @@ class ResampleRegrid:
                 min_echo_range = np.nanmin(np.float32(cruise_df["MIN_ECHO_RANGE"]))
                 max_echo_range = np.nanmax(np.float32(cruise_df["MAX_ECHO_RANGE"]))
 
-                print(
-                    "Creating empty ndarray for Sv data."
-                )  # Note: cruise_zarr dimensions are (depth, time, frequency)
+                print("Creating empty ndarray for Sv data.")  # Note: cruise dims (depth, time, frequency)
+                output_zarr_store_shape = output_zarr_store.Sv.shape
+                end_ping_time_index - start_ping_time_index
+                output_zarr_store_height = output_zarr_store_shape[0]
+                output_zarr_store_width = end_ping_time_index - start_ping_time_index
+                output_zarr_store_depth = output_zarr_store_shape[2]
                 cruise_sv_subset = np.empty(
-                    shape=output_zarr_store.Sv[
-                        :, start_ping_time_index:end_ping_time_index, :
-                    ].shape
+                    shape=(output_zarr_store_height, output_zarr_store_width, output_zarr_store_depth)
                 )
                 cruise_sv_subset[:, :, :] = np.nan
 
                 all_cruise_depth_values = zarr_manager.get_depth_values(
-                    min_echo_range=min_echo_range, max_echo_range=max_echo_range
+                    min_echo_range=min_echo_range,
+                    max_echo_range=max_echo_range
                 ) # (5262,) and
 
                 print(" ".join(list(input_xr_zarr_store.Sv.dims)))
@@ -281,16 +283,6 @@ class ResampleRegrid:
                 #########################################################################
                 # write Sv values to cruise-level-model-store
                 output_zarr_store.Sv[:, start_ping_time_index:end_ping_time_index, :] = regrid_resample.values
-
-                #########################################################################
-                # [5] write subset of latitude/longitude
-                output_zarr_store.latitude[
-                    start_ping_time_index:end_ping_time_index
-                ] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
-                output_zarr_store.longitude[
-                    start_ping_time_index:end_ping_time_index
-                ] = geospatial.dropna()["longitude"].values
-
                 #########################################################################
                 # TODO: add the "detected_seafloor_depth/" to the
                 #  L2 cruise dataarrays
@@ -311,11 +303,14 @@ class ResampleRegrid:
                         start_ping_time_index:end_ping_time_index
                     ] = detected_seafloor_depths
                 #
-                #
-                #
-                # TODO: write the time variable last so that I can parse that as check
-                #
-                #
+                #########################################################################
+                # [5] write subset of latitude/longitude
+                output_zarr_store.latitude[
+                    start_ping_time_index:end_ping_time_index
+                ] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
+                output_zarr_store.longitude[
+                    start_ping_time_index:end_ping_time_index
+                ] = geospatial.dropna()["longitude"].values
                 #########################################################################
                 #########################################################################
         except Exception as err:

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/model/zarr_manager.py

@@ -2,6 +2,7 @@ import numcodecs
 import numpy as np
 import xarray as xr
 import zarr
+import importlib.metadata
 from numcodecs import Blosc
 
 from water_column_sonar_processing.aws import S3FSManager
@@ -249,9 +250,9 @@ class ZarrManager:
         root.attrs["sensor_name"] = sensor_name
         #
         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-        root.attrs["processing_software_version"] = (
-            "25.1.3"  # TODO: get programmatically, echopype>utils>prov.py
-        )
+
+        current_project_version = importlib.metadata.version('water_column_sonar_processing')
+        root.attrs["processing_software_version"] = current_project_version
         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
         #
         root.attrs["calibration_status"] = calibration_status
@@ -290,7 +291,7 @@ class ZarrManager:
         # zarr_synchronizer: Union[str, None] = None, # TODO:
         output_bucket_name: str,
         endpoint_url=None,
-    ):
+    ) -> zarr.hierarchy.Group:
         # Mounts a Zarr store using pythons Zarr implementation. The mounted store
         #  will have read/write privileges so that store can be updated.
         print("Opening L2 Zarr store with Zarr for writing.")
@@ -316,18 +317,21 @@ class ZarrManager:
         input_bucket_name: str,
         endpoint_url=None,
     ) -> xr.Dataset:
-        print("Opening L1 Zarr store in S3 with Xarray.")
+        print("Opening L1 Zarr store in S3 with Xarray.") # TODO: Is this only used for reading from?
         try:
             zarr_path = f"s3://{input_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
             s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
             store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
-            ds = xr.open_zarr(
-                store=store_s3_map, consolidated=None
-            )  # synchronizer=SYNCHRONIZER
+            ds = xr.open_dataset(
+                filename_or_obj=store_s3_map,
+                engine="zarr",
+                chunks={}
+            )
         except Exception as err:
             print("Problem opening Zarr store in S3 as Xarray.")
             raise err
-        print("Done opening Zarr store in S3 as Xarray.")
+        finally:
+            print("Exiting opening Zarr store in S3 as Xarray.")
         return ds
 
     def open_l2_zarr_store_with_xarray(

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing/utility/constants.py

@@ -3,7 +3,7 @@ from enum import Enum, Flag, unique
 
 @unique
 class Constants(Flag):
-    TILE_SIZE = 1024 # TODO: add tile size to metadata?
+    TILE_SIZE = 1024
 
     # Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
     # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.2
 Name: water_column_sonar_processing
-Version: 25.1.3
-Summary: A processing tool for water column sonar data.
+Version: 25.1.4
+Summary: Processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
 Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiobotocore==2.19.0
@@ -34,6 +34,19 @@ Requires-Dist: typing-extensions==4.10.0
 Requires-Dist: xarray==2024.10.0
 Requires-Dist: xbatcher==0.4.0
 Requires-Dist: zarr==2.18.3
+Provides-Extra: dev
+Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pyinstaller; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: flake8==7.1.1; extra == "dev"
+Requires-Dist: pooch==1.8.2; extra == "dev"
+Requires-Dist: pytest~=8.3.3; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
 
 # Water Column Sonar Processing
 Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
@@ -80,14 +93,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
      3. Set interpreter 
 
 # Installing Dependencies
-
-  1. Add dependencies with versions to requirements.txt
-  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
+```
+uv pip install --upgrade pip
+#uv pip install -r requirements_dev.txt
+uv pip install -r pyproject.toml --extra dev
+```
 
 
 # Pytest
 ```commandline
-pytest --disable-warnings
+uv run pytest tests
+#pytest --disable-warnings
 ```
 or
 > pytest --cache-clear --cov=src tests/ --cov-report=xml
@@ -120,7 +136,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 # Tag a Release
 Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
 ```commandline
-git tag -a v25.1.2 -m "Releasing version v25.1.2"
+git tag -a v25.1.4 -m "Releasing version v25.1.4"
 git push origin --tags
 ```
 

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/SOURCES.txt

@@ -5,15 +5,11 @@
 LICENSE
 README.md
 pyproject.toml
-pytest.ini
-requirements.txt
-requirements_dev.txt
 .github/workflows/test_action.yaml
 open-science-data-federation/ml/autoencoder_example.py
 open-science-data-federation/osdf_examples/foo.ipynb
 open-science-data-federation/osdf_examples/sonar_ai.ipynb
 tests/conftest.py
-tests/test_process.py
 tests/test_resources/index/calibrated_cruises.csv
 tests/test_resources/raw_to_zarr/D20070724-T042400.bot
 tests/test_resources/raw_to_zarr/D20070724-T042400.idx

{water_column_sonar_processing-25.1.3 → water_column_sonar_processing-25.1.4}/water_column_sonar_processing.egg-info/requires.txt

@@ -21,3 +21,18 @@ typing-extensions==4.10.0
 xarray==2024.10.0
 xbatcher==0.4.0
 zarr==2.18.3
+
+[dev]
+bandit[toml]==1.8.0
+build
+pre-commit
+pyinstaller
+twine
+flake8==7.1.1
+pooch==1.8.2
+pytest~=8.3.3
+tqdm
+bandit
+
+[test]
+pytest-cov

water_column_sonar_processing-25.1.3/.github/workflows/test_action.yaml

@@ -1,24 +0,0 @@
-name: Python package
-
-on: [push]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out
-        uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          # Semantic version range syntax or exact version of a Python version
-          python-version: '3.10'
-          # Optional - x64 or x86 architecture, defaults to x64
-#          architecture: 'x64'
-          cache: 'pip'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements_dev.txt
-      - name: Run the tests
-        run: python -m pytest

water_column_sonar_processing-25.1.3/.python-version

@@ -1,2 +0,0 @@
-3.10.12
-water-column-sonar-processing

water_column_sonar_processing-25.1.3/pyproject.toml

@@ -1,43 +0,0 @@
-[build-system]
-requires = [
-    "setuptools>=61.0",
-    #"setuptools_scm[toml] >= 4, <6",
-    "wheel >= 0.29.0",
-]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "water_column_sonar_processing"
-version = "25.1.3"
-authors = [
-  { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
-]
-description = "A processing tool for water column sonar data."
-readme = "README.md"
-#requires-python = ">=3.10"
-requires-python = ">=3.8"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-]
-dynamic = ["dependencies"]
-
-[project.urls]
-Homepage = "https://github.com/CI-CMG/water-column-sonar-processing"
-Issues = "https://github.com/CI-CMG/water-column-sonar-processing/issues"
-
-[tool.setuptools.dynamic]
-dependencies = {file = ["requirements.txt"]}
-optional-dependencies = {dev = { file = ["requirements_dev.txt"] }}
-
-#[tool.setuptools_scm]
-#fallback_version = "unknown"
-#local_scheme = "node-and-date"
-#write_to = "_water_column_sonar_processing_version.py"
-#write_to_template = 'version = "{version}"'
-
-[tool.bandit]
-exclude_dirs = ["tests"]
-[tool.pre-commit-hooks.bandit]
-exclude = ["*/tests/*"]

water_column_sonar_processing-25.1.3/pytest.ini

@@ -1,13 +0,0 @@
-# test directory
-#[pytest]
-#testpaths = src/water_column_sonar_processing/tests
-#cache_dir = .cache
-#markers =
-#    unit: marks tests as unit tests
-#    integration: marks tests as integration tests
-[pytest]
-addopts = "-p no:warnings"
-#testpaths = "tests"
-#testpaths=src/water_column_sonar_processing/tests
-cache_dir=.cache
-pythonpath="."

water_column_sonar_processing-25.1.3/requirements.txt

@@ -1,32 +0,0 @@
-# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html
-# defined for Python 3.12
-# Note: be careful with conversions for pandas >=2.0.0, timestamps will have a lot of problems
-
-aiobotocore==2.19.0
-boto3==1.36.3
-botocore==1.36.3
-echopype==0.9.0
-fiona==1.10.1
-# Alternative to geopandas: pyogrio
-geopandas==1.0.1
-mock==5.1.0
-moto[all]==5.0.27
-moto[server]==5.0.27
-numcodecs==0.13.1
-numpy==1.26.4
-pandas==2.2.3
-pyarrow==18.1.0
-python-dotenv==1.0.1
-requests==2.32.3
-#s3fs==2024.3.1
-#s3fs==2024.3.0 # does not work
-s3fs==2024.2.0 # works ...something between 2024.2 and 2024.3 creates the problem
-scipy==1.14.1
-#setuptools==75.6.0
-setuptools
-shapely==2.0.3
-typing-extensions==4.10.0
-xarray==2024.10.0
-#  xbatcher[tensorflow]
-xbatcher==0.4.0
-zarr==2.18.3

water_column_sonar_processing-25.1.3/requirements_dev.txt

@@ -1,14 +0,0 @@
--r requirements.txt
-
-bandit[toml]==1.8.0
-build
-pre-commit
-pyinstaller
-twine
-
-flake8==7.1.1
-pooch==1.8.2
-pytest~=8.3.3
-pytest-cov==6.0.0
-tqdm
-bandit

water_column_sonar_processing-25.1.3/tests/test_process.py

@@ -1,472 +0,0 @@
-# import json
-# import os
-# import pytest
-# import numpy as np
-# from dotenv import find_dotenv, load_dotenv
-# from moto import mock_aws
-#
-# from water_column_sonar_processing.aws import DynamoDBManager
-# from water_column_sonar_processing.aws import S3Manager
-# from water_column_sonar_processing.aws import SNSManager
-# from water_column_sonar_processing.aws import SQSManager
-# from water_column_sonar_processing.process import Process
-#
-#
-# #######################################################
-# def setup_module():
-#     print("setup")
-#     env_file = find_dotenv(".env-test")
-#     load_dotenv(dotenv_path=env_file, override=True)
-#
-#
-# def teardown_module():
-#     print("teardown")
-#
-#
-# #######################################################
-# # TODO: Delete this?
-# @mock_aws
-# @pytest.mark.skip(reason="no way of currently testing this")
-# def test_model_happy_path():
-#     test_input_bucket_name = os.environ["INPUT_BUCKET_NAME"]
-#
-#     test_output_bucket_name = os.environ["OUTPUT_BUCKET_NAME"]
-#
-#     test_table_name = os.environ["TABLE_NAME"]
-#
-#     test_topic_arn = os.environ["TOPIC_ARN"]
-#     test_topic_name = test_topic_arn.split(":")[-1]
-#
-#     # [1 of 3] Create DynamoDB table
-#     ddbm = DynamoDBManager()
-#     ddbm.create_water_column_sonar_table(table_name=test_table_name)
-#     ###################################################
-#     # tests data 0 - David_Starr_Jordan - DS0604
-#     # tests data 1 - Okeanos_Explorer - EX1404L2
-#     # tests data 2 - Henry_B._Bigelow - HB0707
-#     # tests data 3 - Miller_Freeman - MF0710
-#     ###################################################
-#     # tests data 0 - David_Starr_Jordan - DS0604
-#     test_channels = [
-#         "GPT  38 kHz 009072055a7f 2 ES38B",
-#         "GPT  70 kHz 00907203400a 3 ES70-7C",
-#         "GPT 120 kHz 009072034d52 1 ES120-7",
-#         "GPT 200 kHz 0090720564e4 4 ES200-7C",
-#     ]
-#     test_frequencies = [38_000, 70_000, 120_000, 200_000]
-#     # Create the first and third tests example files for the same cruise
-#     ddbm.update_item(
-#         table_name=test_table_name,
-#         key={
-#             "FILE_NAME": {"S": "DSJ0604-D20060406-T035914.raw"},  # Partition Key
-#             "CRUISE_NAME": {"S": "DS0604"},  # Sort Key
-#         },
-#         expression_attribute_names={
-#             "#CH": "CHANNELS",
-#             "#ET": "END_TIME",
-#             "#ED": "ERROR_DETAIL",
-#             "#FR": "FREQUENCIES",
-#             "#MA": "MAX_ECHO_RANGE",
-#             "#MI": "MIN_ECHO_RANGE",
-#             "#ND": "NUM_PING_TIME_DROPNA",
-#             "#PS": "PIPELINE_STATUS",  # testing this updated
-#             "#PT": "PIPELINE_TIME",  # testing this updated
-#             "#SE": "SENSOR_NAME",
-#             "#SH": "SHIP_NAME",
-#             "#ST": "START_TIME",
-#             "#ZB": "ZARR_BUCKET",
-#             "#ZP": "ZARR_PATH",
-#         },
-#         expression_attribute_values={
-#             ":ch": {"L": [{"S": i} for i in test_channels]},
-#             ":et": {"S": "2006-04-06T03:59:15.587Z"},
-#             ":ed": {"S": ""},
-#             ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
-#             ":ma": {"N": str(np.round(499.5721, 4))},
-#             ":mi": {"N": str(np.round(0.25, 4))},
-#             ":nd": {"N": str(1)},
-#             ":ps": {"S": "SUCCESS_AGGREGATOR"},
-#             ":pt": {"S": "2023-10-02T08:54:41Z"},
-#             ":se": {"S": "EK60"},
-#             ":sh": {"S": "David_Starr_Jordan"},
-#             ":st": {"S": "2006-04-06T03:59:14.115Z"},
-#             ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
-#             ":zp": {
-#                 "S": "level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T035914.model"
-#             },
-#         },
-#         update_expression=(
-#             "SET "
-#             "#CH = :ch, "
-#             "#ET = :et, "
-#             "#ED = :ed, "
-#             "#FR = :fr, "
-#             "#MA = :ma, "
-#             "#MI = :mi, "
-#             "#ND = :nd, "
-#             "#PS = :ps, "
-#             "#PT = :pt, "
-#             "#SE = :se, "
-#             "#SH = :sh, "
-#             "#ST = :st, "
-#             "#ZB = :zb, "
-#             "#ZP = :zp"
-#         ),
-#     )
-#     ddbm.update_item(
-#         table_name=test_table_name,
-#         key={
-#             "FILE_NAME": {"S": "DSJ0604-D20060406-T133530.raw"},  # Partition Key
-#             "CRUISE_NAME": {"S": "DS0604"},  # Sort Key
-#         },
-#         expression_attribute_names={
-#             "#CH": "CHANNELS",
-#             "#ET": "END_TIME",
-#             "#ED": "ERROR_DETAIL",
-#             "#FR": "FREQUENCIES",
-#             "#MA": "MAX_ECHO_RANGE",
-#             "#MI": "MIN_ECHO_RANGE",
-#             "#ND": "NUM_PING_TIME_DROPNA",
-#             "#PS": "PIPELINE_STATUS",  # testing this updated
-#             "#PT": "PIPELINE_TIME",  # testing this updated
-#             "#SE": "SENSOR_NAME",
-#             "#SH": "SHIP_NAME",
-#             "#ST": "START_TIME",
-#             "#ZB": "ZARR_BUCKET",
-#             "#ZP": "ZARR_PATH",
-#         },
-#         expression_attribute_values={
-#             ":ch": {"L": [{"S": i} for i in test_channels]},
-#             ":et": {"S": "2006-04-06T15:16:51.945Z"},
-#             ":ed": {"S": ""},
-#             ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
-#             ":ma": {"N": str(np.round(499.7653, 4))},
-#             ":mi": {"N": str(np.round(0.25, 4))},
-#             ":nd": {"N": str(2467)},
-#             ":ps": {"S": "SUCCESS_AGGREGATOR"},
-#             ":pt": {"S": "2023-10-02T08:54:43Z"},
-#             ":se": {"S": "EK60"},
-#             ":sh": {"S": "David_Starr_Jordan"},
-#             ":st": {"S": "2006-04-06T13:35:30.701Z"},
-#             ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
-#             ":zp": {
-#                 "S": "level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T133530.model"
-#             },
-#         },
-#         update_expression=(
-#             "SET "
-#             "#CH = :ch, "
-#             "#ET = :et, "
-#             "#ED = :ed, "
-#             "#FR = :fr, "
-#             "#MA = :ma, "
-#             "#MI = :mi, "
-#             "#ND = :nd, "
-#             "#PS = :ps, "
-#             "#PT = :pt, "
-#             "#SE = :se, "
-#             "#SH = :sh, "
-#             "#ST = :st, "
-#             "#ZB = :zb, "
-#             "#ZP = :zp"
-#         ),
-#     )
-#     ###################################################
-#     # tests data 1 - Okeanos_Explorer - EX1404L2
-#     test_channels = ["GPT  18 kHz 009072066c0e 1-1 ES18-11"]
-#     test_frequencies = [18_000]
-#     ddbm.update_item(
-#         table_name=test_table_name,
-#         key={
-#             "FILE_NAME": {"S": "EX1404L2_EK60_-D20140908-T173907.raw"},  # Partition Key
-#             "CRUISE_NAME": {"S": "EX1404L2"},  # Sort Key
-#         },
-#         expression_attribute_names={
-#             "#CH": "CHANNELS",
-#             "#ET": "END_TIME",
-#             "#ED": "ERROR_DETAIL",
-#             "#FR": "FREQUENCIES",
-#             "#MA": "MAX_ECHO_RANGE",
-#             "#MI": "MIN_ECHO_RANGE",
-#             "#ND": "NUM_PING_TIME_DROPNA",
-#             "#PS": "PIPELINE_STATUS",  # testing this updated
-#             "#PT": "PIPELINE_TIME",  # testing this updated
-#             "#SE": "SENSOR_NAME",
-#             "#SH": "SHIP_NAME",
-#             "#ST": "START_TIME",
-#             "#ZB": "ZARR_BUCKET",
-#             "#ZP": "ZARR_PATH",
-#         },
-#         expression_attribute_values={
-#             ":ch": {"L": [{"S": i} for i in test_channels]},
-#             ":et": {"S": "2014-09-08T17:56:49.024Z"},
-#             ":ed": {"S": ""},
-#             ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
-#             ":ma": {"N": str(np.round(2499.7573, 4))},
-#             ":mi": {"N": str(np.round(0.25, 4))},
-#             ":nd": {"N": str(324)},
-#             ":ps": {"S": "SUCCESS_AGGREGATOR"},
-#             ":pt": {"S": "2023-10-02T18:19:44Z"},
-#             ":se": {"S": "EK60"},
-#             ":sh": {"S": "Okeanos_Explorer"},
-#             ":st": {"S": "2014-09-08T17:39:07.660Z"},
-#             ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
-#             ":zp": {
-#                 "S": "level_1/Okeanos_Explorer/EX1404L2/EK60/EX1404L2_EK60_-D20140908-T173907.model"
-#             },
-#         },
-#         update_expression=(
-#             "SET "
-#             "#CH = :ch, "
-#             "#ET = :et, "
-#             "#ED = :ed, "
-#             "#FR = :fr, "
-#             "#MA = :ma, "
-#             "#MI = :mi, "
-#             "#ND = :nd, "
-#             "#PS = :ps, "
-#             "#PT = :pt, "
-#             "#SE = :se, "
-#             "#SH = :sh, "
-#             "#ST = :st, "
-#             "#ZB = :zb, "
-#             "#ZP = :zp"
-#         ),
-#     )
-#     ###################################################
-#     # tests data 2 - Henry_B._Bigelow - HB0707
-#     test_channels = [
-#         "GPT  18 kHz 009072056b0e 2 ES18-11",
-#         "GPT  38 kHz 0090720346bc 1 ES38B",
-#         "GPT 120 kHz 0090720580f1 3 ES120-7C",
-#         "GPT 200 kHz 009072034261 4 ES200-7C",
-#     ]
-#     test_frequencies = [18_000, 38_000, 120_000, 200_000]
-#     ddbm.update_item(
-#         table_name=test_table_name,
-#         key={
-#             "FILE_NAME": {"S": "D20070712-T061745.raw"},  # Partition Key
-#             "CRUISE_NAME": {"S": "HB0707"},  # Sort Key
-#         },
-#         expression_attribute_names={
-#             "#CH": "CHANNELS",
-#             "#ET": "END_TIME",
-#             "#ED": "ERROR_DETAIL",
-#             "#FR": "FREQUENCIES",
-#             "#MA": "MAX_ECHO_RANGE",
-#             "#MI": "MIN_ECHO_RANGE",
-#             "#ND": "NUM_PING_TIME_DROPNA",
-#             "#PS": "PIPELINE_STATUS",  # testing this updated
-#             "#PT": "PIPELINE_TIME",  # testing this updated
-#             "#SE": "SENSOR_NAME",
-#             "#SH": "SHIP_NAME",
-#             "#ST": "START_TIME",
-#             "#ZB": "ZARR_BUCKET",
-#             "#ZP": "ZARR_PATH",
-#         },
-#         expression_attribute_values={
-#             ":ch": {"L": [{"S": i} for i in test_channels]},
-#             ":et": {"S": "2007-07-12T10:05:02.579Z"},
-#             ":ed": {"S": ""},
-#             ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
-#             ":ma": {"N": str(np.round(249.792, 4))},
-#             ":mi": {"N": str(np.round(0.25, 4))},
-#             ":nd": {"N": str(9733)},
-#             ":ps": {"S": "SUCCESS_AGGREGATOR"},
-#             ":pt": {"S": "2023-10-01T20:13:58Z"},
-#             ":se": {"S": "EK60"},
-#             ":sh": {"S": "Henry_B._Bigelow"},
-#             ":st": {"S": "2007-07-12T06:17:45.579Z"},
-#             ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
-#             ":zp": {
-#                 "S": "level_1/Henry_B._Bigelow/HB0707/EK60/D20070712-T061745.model"
-#             },
-#         },
-#         update_expression=(
-#             "SET "
-#             "#CH = :ch, "
-#             "#ET = :et, "
-#             "#ED = :ed, "
-#             "#FR = :fr, "
-#             "#MA = :ma, "
-#             "#MI = :mi, "
-#             "#ND = :nd, "
-#             "#PS = :ps, "
-#             "#PT = :pt, "
-#             "#SE = :se, "
-#             "#SH = :sh, "
-#             "#ST = :st, "
-#             "#ZB = :zb, "
-#             "#ZP = :zp"
-#         ),
-#     )
-#     ###################################################
-#     # tests data 3 - Miller_Freeman - MF0710
-#     test_channels = [
-#         "GPT  18 kHz 009072034d55 3 ES18-11",
-#         "GPT  38 kHz 009072016e01 4 ES38B",
-#         "GPT 120 kHz 009072016a73 1 ES120-7C",
-#         "GPT 200 kHz 009072033fcc 2 ES200-7C",
-#     ]
-#     test_frequencies = [18_000, 38_000, 120_000, 200_000]
-#     ddbm.update_item(
-#         table_name=test_table_name,
-#         key={
-#             "FILE_NAME": {"S": "HAKE2007-D20070708-T200449.raw"},  # Partition Key
-#             "CRUISE_NAME": {"S": "MF0710"},  # Sort Key
-#         },
-#         expression_attribute_names={
-#             "#CH": "CHANNELS",
-#             "#ET": "END_TIME",
-#             "#ED": "ERROR_DETAIL",
-#             "#FR": "FREQUENCIES",
-#             "#MA": "MAX_ECHO_RANGE",
-#             "#MI": "MIN_ECHO_RANGE",
-#             "#ND": "NUM_PING_TIME_DROPNA",
-#             "#PS": "PIPELINE_STATUS",  # testing this updated
-#             "#PT": "PIPELINE_TIME",  # testing this updated
-#             "#SE": "SENSOR_NAME",
-#             "#SH": "SHIP_NAME",
-#             "#ST": "START_TIME",
-#             "#ZB": "ZARR_BUCKET",
-#             "#ZP": "ZARR_PATH",
-#         },
-#         expression_attribute_values={
-#             ":ch": {"L": [{"S": i} for i in test_channels]},
-#             ":et": {"S": "2007-07-08T20:44:55.598Z"},
-#             ":ed": {"S": ""},
-#             ":fr": {"L": [{"N": str(i)} for i in test_frequencies]},
-#             ":ma": {"N": str(np.round(749.7416, 4))},
-#             ":mi": {"N": str(np.round(0.25, 4))},
-#             ":nd": {"N": str(801)},
-#             ":ps": {"S": "SUCCESS_AGGREGATOR"},
-#             ":pt": {"S": "2023-10-02T08:41:50Z"},
-#             ":se": {"S": "EK60"},
-#             ":sh": {"S": "Miller_Freeman"},
-#             ":st": {"S": "2007-07-08T20:04:49.552Z"},
-#             ":zb": {"S": "r2d2-dev-echofish2-118234403147-echofish-dev-output"},
-#             ":zp": {
-#                 "S": "level_1/Miller_Freeman/MF0710/EK60/HAKE2007-D20070708-T200449.model"
-#             },
-#         },
-#         update_expression=(
-#             "SET "
-#             "#CH = :ch, "
-#             "#ET = :et, "
-#             "#ED = :ed, "
-#             "#FR = :fr, "
-#             "#MA = :ma, "
-#             "#MI = :mi, "
-#             "#ND = :nd, "
-#             "#PS = :ps, "
-#             "#PT = :pt, "
-#             "#SE = :se, "
-#             "#SH = :sh, "
-#             "#ST = :st, "
-#             "#ZB = :zb, "
-#             "#ZP = :zp"
-#         ),
-#     )
-#     ###################################################
-#
-#     # [2 of 3 - Part I] Create S3 bucket
-#     input_s3m = S3Manager()
-#     input_s3m.create_bucket(bucket_name=test_input_bucket_name)
-#     output_s3m = S3Manager()  # TODO: requires different credentials
-#     output_s3m.create_bucket(bucket_name=test_output_bucket_name)
-#     # TODO: create two buckets with two sets of credentials required
-#     all_buckets = input_s3m.list_buckets()
-#     print(all_buckets)
-#
-#     # [2 of 3 - Part II] Add Object to Input Bucket
-#     input_s3m.put(
-#         bucket_name=test_input_bucket_name, key="the_input_key", body="the_input_body"
-#     )
-#
-#     # [3 of 3] Set up SNS and SQS
-#     snsm = SNSManager()
-#     sqsm = SQSManager()
-#
-#     sqs_queue_name = "test-queue"
-#     create_queue_response = sqsm.create_queue(queue_name=sqs_queue_name)
-#     print(create_queue_response["QueueUrl"])
-#     assert create_queue_response["ResponseMetadata"]["HTTPStatusCode"] == 200
-#
-#     create_topic_response = snsm.create_topic(topic_name=test_topic_name)
-#     sns_topic_arn = create_topic_response["TopicArn"]
-#     sqs_queue = sqsm.get_queue_by_name(queue_name=sqs_queue_name)
-#     sqs_queue_arn = sqs_queue.attributes["QueueArn"]
-#     snsm.subscribe(topic_arn=sns_topic_arn, endpoint=sqs_queue_arn)
-#     ###troubleshooting
-#     # snsm.list_topics()
-#     # snsm.publish(
-#     #     topic_arn=sns_topic_arn,
-#     #     message=json.dumps("abc"),
-#     #     # MessageStructure='json'
-#     # )
-#     ###### end setup ######
-#
-#     #############################################################
-#     model_instance = Process()
-#     # run the src
-#     model_instance.execute()
-#     #############################################################
-#
-#     # tests all the outcomes
-#     # (1) file is in bucket
-#     # (2) sns messages are in queue
-#     # (3) dynamodb was updated
-#
-#     # [1 of 3] Check that file is in the Output Bucket
-#     # TODO: change to writing file to s3 bucket using s3fs
-#     s3_object = input_s3m.get(bucket_name=test_input_bucket_name, key="the_input_key")
-#     body = s3_object.get()["Body"].read().decode("utf-8")
-#     assert body == "the_input_body"
-#
-#     # [2 of 3] Validate SNS Message was Dispatched
-#     sqs_msgs = sqs_queue.receive_messages(
-#         AttributeNames=["All"],
-#         MessageAttributeNames=["All"],
-#         VisibilityTimeout=15,
-#         WaitTimeSeconds=20,
-#         MaxNumberOfMessages=10,
-#     )
-#     assert len(sqs_msgs) == 1
-#     test_success_message = {
-#         "default": {
-#             "shipName": "David_Starr_Jordan",
-#             "cruiseName": "DS0604",
-#             "sensorName": "EK60",
-#             "fileName": "DSJ0604-D20060406-T113407.raw",
-#         }
-#     }
-#     assert json.loads(sqs_msgs[0].body)["Message"] == json.dumps(test_success_message)
-#
-#     # [3 of 3] Check that DynamoDB has been updated
-#     # TODO: get the table as a dataframe
-#     df = ddbm.get_table_as_df(
-#         table_name=test_table_name,
-#         ship_name="David_Starr_Jordan",
-#         cruise_name="DS0604",
-#         sensor_name="EK60",
-#     )
-#
-#     # 2 files were processed previously, creating new total of 3
-#     assert df.shape[0] == 3
-#
-#     # 16 columns of data are captured
-#     assert df.shape[1] == 16
-#
-#     # check that new file name is included
-#     assert "DSJ0604-D20060406-T113407.raw" in list(df["FILE_NAME"])
-#
-#     # make sure that other filenames aren't included
-#     assert "HAKE2007-D20070708-T200449.raw" not in list(df["FILE_NAME"])
-#
-#     # assert df[PIPELINE_STATUS'] == __?__
-#
-#
-# # def test_model_file_already_exists(self):
-# #     pass
-#
-# #######################################################