water-column-sonar-processing 0.0.4__tar.gz → 0.0.5__tar.gz

{water_column_sonar_processing-0.0.4/src/water_column_sonar_processing.egg-info → water_column_sonar_processing-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.4
+Version: 0.0.5
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "water_column_sonar_processing"
-version = "0.0.4"
+version = "0.0.5"
 authors = [
   { name="Rudy Klucik", email="rudy.klucik@noaa.gov" },
 ]

{water_column_sonar_processing-0.0.4/src/aws_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/aws}/dynamodb_manager.py

@@ -23,7 +23,7 @@ class DynamoDBManager:
         self.type_deserializer = TypeDeserializer()
 
     #####################################################################
-    ### defined in raw-to-zarr_manager, not used
+    ### defined in raw-to-model, not used
     # def put_item(
     #         self,
     #         table_name,

{water_column_sonar_processing-0.0.4/src/aws_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/aws}/s3_manager.py

@@ -139,12 +139,12 @@ class S3Manager:
         return all_uploads
 
     #####################################################################
-    def upload_zarr_files_to_bucket(  # noaa-wcsd-zarr_manager-pds
+    def upload_zarr_files_to_bucket(  # noaa-wcsd-model-pds
             self,
             local_directory,
             remote_directory,
     ):
-        # Right now this is just for uploading a zarr_manager store to s3
+        # Right now this is just for uploading a model store to s3
         print('Uploading files to output bucket.')
         store_name = os.path.basename(local_directory)
         all_files = []
@@ -162,8 +162,8 @@ class S3Manager:
         return all_uploads
 
     #####################################################################
-    # used: raw-to-zarr_manager
-    def list_objects(  # noaa-wcsd-pds and noaa-wcsd-zarr_manager-pds
+    # used: raw-to-model
+    def list_objects(  # noaa-wcsd-pds and noaa-wcsd-model-pds
             self,
             bucket_name,
             prefix
@@ -179,7 +179,7 @@ class S3Manager:
                 keys.extend([k['Key'] for k in page['Contents']])
         return keys
 
-    def list_nodd_objects(  # These are used by the geometry_manager for uploading data
+    def list_nodd_objects(  # These are used by the geometry for uploading data
             self,
             prefix,
     ):
@@ -250,8 +250,8 @@ class S3Manager:
         return raw_files
 
     #####################################################################
-    def get_object(  # TODO: Move this to index_manager.py
-                     # noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
+    def get_object(  # TODO: Move this to index.py
+                     # noaa-wcsd-pds or noaa-wcsd-model-pds
             self,
             bucket_name,
             key_name,
@@ -272,9 +272,9 @@ class S3Manager:
         return response
 
     #####################################################################
-    # used raw-to-zarr_manager
+    # used raw-to-model
     def download_file(  # TODO: change to download_object
-                        # noaa-wcsd-pds or noaa-wcsd-zarr_manager-pds
+                        # noaa-wcsd-pds or noaa-wcsd-model-pds
             self,
             bucket_name,
             key,
@@ -289,7 +289,7 @@ class S3Manager:
 
     #####################################################################
     # not used
-    # def delete_nodd_object(  # noaa-wcsd-zarr_manager-pds
+    # def delete_nodd_object(  # noaa-wcsd-model-pds
     #         self,
     #         bucket_name,
     #         key
@@ -319,7 +319,7 @@ class S3Manager:
 
     #####################################################################
     # not used TODO: remove
-    def put(  # noaa-wcsd-zarr_manager-pds
+    def put(  # noaa-wcsd-model-pds
             self,
             bucket_name,
             key,

{water_column_sonar_processing-0.0.4/src → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing}/cruise/create_empty_zarr_store.py

@@ -2,17 +2,17 @@ import os
 import numcodecs
 import numpy as np
 
-from utility.cleaner import Cleaner
-from aws_manager.dynamodb_manager import DynamoDBManager
-from aws_manager.s3_manager import S3Manager
-from zarr_manager.zarr_manager import ZarrManager
+from water_column_sonar_processing.utility.cleaner import Cleaner
+from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.model.zarr_manager import ZarrManager
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
 
 TEMPDIR = "/tmp"
 
-# TODO: when ready switch to version 3 of zarr_manager spec
+# TODO: when ready switch to version 3 of model spec
 # ZARR_V3_EXPERIMENTAL_API = 1
 # creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
 
@@ -34,7 +34,7 @@ class CreateEmptyZarrStore:
             object_prefix: str,
             cruise_name: str,
     ) -> None:
-        print('uploading zarr_manager store to s3')
+        print('uploading model store to s3')
         s3_manager = S3Manager()
         #
         print('Starting upload with thread pool executor.')
@@ -43,8 +43,8 @@ class CreateEmptyZarrStore:
         for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr_manager"):
             for file in files:
                 local_path = os.path.join(subdir, file)
-                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr_manager/.zattrs'
-                s3_key = f'{object_prefix}/{cruise_name}.zarr_manager{local_path.split(f"{cruise_name}.zarr_manager")[-1]}'
+                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.model/.zattrs'
+                s3_key = f'{object_prefix}/{cruise_name}.model{local_path.split(f"{cruise_name}.model")[-1]}'
                 all_files.append([local_path, s3_key])
         #
         # print(all_files)
@@ -98,10 +98,10 @@ class CreateEmptyZarrStore:
             new_width = int(consolidated_zarr_width)
             print(f"new_width: {new_width}")
             #################################################################
-            store_name = f"{cruise_name}.zarr_manager"
+            store_name = f"{cruise_name}.model"
             print(store_name)
             ################################################################
-            # Delete existing zarr_manager store if it exists
+            # Delete existing model store if it exists
             s3_manager = S3Manager()
             zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
             child_objects = s3_manager.get_child_objects(
@@ -113,7 +113,7 @@ class CreateEmptyZarrStore:
                     objects=child_objects,
                 )
             ################################################################
-            # Create new zarr_manager store
+            # Create new model store
             zarr_manager = ZarrManager()
             new_height = len(zarr_manager.get_depth_values(
                 min_echo_range=cruise_min_echo_range,
@@ -157,11 +157,11 @@ class CreateEmptyZarrStore:
             # TODO: update enum in dynamodb
             #################################################################
         except Exception as err:
-            print(f"Problem trying to create new cruise zarr_manager store: {err}")
+            print(f"Problem trying to create new cruise model store: {err}")
         finally:
             cleaner = Cleaner()
             cleaner.delete_local_files()
-        print("Done creating cruise level zarr_manager store")
+        print("Done creating cruise level model store")
 
 
 ###########################################################

{water_column_sonar_processing-0.0.4/src → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing}/cruise/resample_regrid.py

@@ -6,16 +6,16 @@ import numpy as np
 import xarray as xr
 import pandas as pd
 
-from aws_manager.dynamodb_manager import DynamoDBManager
-from zarr_manager.zarr_manager import ZarrManager
-from geometry_manager.geometry_manager import GeometryManager
+from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
+from water_column_sonar_processing.model.zarr_manager import ZarrManager
+from water_column_sonar_processing.geometry.geometry_manager import GeometryManager
 
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
 
 
-# TODO: when ready switch to version 3 of zarr_manager spec
+# TODO: when ready switch to version 3 of model spec
 #  ZARR_V3_EXPERIMENTAL_API = 1
 #  creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
 
@@ -115,7 +115,7 @@ class ResampleRegrid:
     ) -> None:
         """
         The goal here is to interpolate the data against the depth values already populated
-        in the existing file level zarr_manager stores. We open the cruise-level store with zarr_manager for
+        in the existing file level model stores. We open the cruise-level store with model for
         read/write operations. We open the file-level store with Xarray to leverage tools for
         resampling and subsetting the data.
         """
@@ -124,7 +124,7 @@ class ResampleRegrid:
             zarr_manager = ZarrManager()
             # s3_manager = S3Manager()
             geo_manager = GeometryManager()
-            # get zarr_manager store
+            # get model store
             output_zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
                 ship_name=ship_name,
                 cruise_name=cruise_name,
@@ -157,7 +157,7 @@ class ResampleRegrid:
                 # TODO: filter rows by enum success, filter the dataframe just for enums >= LEVEL_1_PROCESSING
                 #  df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
 
-                # Get index_manager from all cruise files. Note: should be based on which are included in cruise.
+                # Get index from all cruise files. Note: should be based on which are included in cruise.
                 index = cruise_df.index[cruise_df['FILE_NAME'] == f"{file_name_stem}.raw"][0]
 
                 # get input store
@@ -169,7 +169,7 @@ class ResampleRegrid:
                 )
                 #########################################################################
                 # [3] Get needed indices
-                # Offset from start index_manager to insert new data. Note that missing values are excluded.
+                # Offset from start index to insert new data. Note that missing values are excluded.
                 ping_time_cumsum = np.insert(
                     np.cumsum(cruise_df['NUM_PING_TIME_DROPNA'].dropna().to_numpy(dtype=int)),
                     obj=0,
@@ -223,7 +223,7 @@ class ResampleRegrid:
                 print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
 
                 #########################################################################
-                # write Sv values to cruise-level-zarr_manager-store
+                # write Sv values to cruise-level-model-store
                 for channel in range(len(input_xr.channel.values)):  # doesn't like being written in one fell swoop :(
                     output_zarr_store.Sv[
                         :,

{water_column_sonar_processing-0.0.4/src/geometry_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/geometry}/geometry_manager.py

@@ -3,8 +3,8 @@ import numpy as np
 import geopandas
 import pandas as pd
 
-from utility.cleaner import Cleaner
-from aws_manager.s3_manager import S3Manager
+from water_column_sonar_processing.utility.cleaner import Cleaner
+from water_column_sonar_processing.aws.s3_manager import S3Manager
 
 """
 //  [Decimal / Places / Degrees	/ Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]

{water_column_sonar_processing-0.0.4/src/geometry_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/geometry}/pmtile_generation.py

@@ -23,7 +23,7 @@ class PMTileGeneration(object):
 
     #######################################################
     def generate_geojson_feature_collection(self):
-        # This was used to read from noaa-wcsd-zarr_manager-pds bucket geojson files and then to
+        # This was used to read from noaa-wcsd-model-pds bucket geojson files and then to
         # generate the geopandas dataframe which could be exported to another comprehensive
         # geojson file. That
         result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))

{water_column_sonar_processing-0.0.4/src/index_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/index}/index_manager.py

@@ -4,7 +4,7 @@ import pandas as pd
 from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
-from aws_manager.s3_manager import S3Manager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
 
 class IndexManager:
 
@@ -193,7 +193,7 @@ class IndexManager:
         return calibration_statuses
 
     #################################################################
-    # def index_manager(  # TODO: get rid of this?
+    # def index(  # TODO: get rid of this?
     #         self
     # ):
     #     start_time = datetime.now()  # used for benchmarking

{water_column_sonar_processing-0.0.4/src/zarr_manager → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/model}/zarr_manager.py

@@ -4,20 +4,18 @@ import numcodecs
 import numpy as np
 import xarray as xr
 from numcodecs import Blosc
-
-from utility.constants import Constants, Coordinates
-from utility.timestamp import Timestamp
-from aws_manager.s3fs_manager import S3FSManager
+from water_column_sonar_processing.utility.constants import Constants, Coordinates
+from water_column_sonar_processing.utility.timestamp import Timestamp
+from water_column_sonar_processing.aws.s3fs_manager import S3FSManager
 
 numcodecs.blosc.use_threads = False
 numcodecs.blosc.set_nthreads(1)
 
 
-# TODO: when ready switch to version 3 of zarr_manager spec
+# TODO: when ready switch to version 3 of model spec
 # ZARR_V3_EXPERIMENTAL_API = 1
 
 # creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
-
 class ZarrManager:
     #######################################################
     def __init__(
@@ -37,7 +35,7 @@ class ZarrManager:
             max_echo_range: float = 100.,  # maximum depth measured from whole cruise
     ):
         # Gets the set of depth values that will be used when resampling and
-        # regridding the data to a cruise level zarr_manager store.
+        # regridding the data to a cruise level model store.
         # Note: returned values do not start at zero.
         print('Getting depth values.')
         all_cruise_depth_values = np.linspace(
@@ -252,7 +250,7 @@ class ZarrManager:
             s3fs_manager = S3FSManager()
             root = f'{self.output_bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr'
             store = s3fs_manager.s3_map(s3_zarr_store_path=root)
-            # synchronizer = zarr_manager.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
+            # synchronizer = model.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
             cruise_zarr = zarr.open(store=store, mode="r+")
         except Exception as err:  # Failure
             print(f'Exception encountered opening Zarr store with Zarr.: {err}')
@@ -284,12 +282,12 @@ class ZarrManager:
 
     #######################################################
     # def create_process_synchronizer(self):
-    #     # TODO: explore aws_manager redis options
+    #     # TODO: explore aws redis options
     #     pass
 
     #######################################################
     # def verify_cruise_store_data(self):
-    #     # TODO: run a check on a finished zarr_manager store to ensure that
+    #     # TODO: run a check on a finished model store to ensure that
     #     #   none of the time, latitude, longitude, or depth values
     #     #   are NaN.
     #     pass

water_column_sonar_processing-0.0.4/src/model.py → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing/process.py

@@ -1,13 +1,13 @@
 import os
 import json
 import numpy as np
-from aws_manager.s3_manager import S3Manager
-from aws_manager.s3fs_manager import S3FSManager
-from aws_manager.sns_manager import SNSManager
-from aws_manager.dynamodb_manager import DynamoDBManager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.aws.s3fs_manager import S3FSManager
+from water_column_sonar_processing.aws.sns_manager import SNSManager
+from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
 
 ###########################################################
-class Model:
+class Process:
     #######################################################
     def __init__(
             self,
@@ -87,7 +87,7 @@ class Model:
                 ':sh': {'S': ship_name},
                 ':st': {'S': '2006-04-06T11:34:07.288Z'},
                 ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-                ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.zarr_manager'},
+                ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T113407.model'},
             },
             update_expression=(
                 'SET '

{water_column_sonar_processing-0.0.4/src → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing}/utility/cleaner.py

@@ -7,9 +7,9 @@ import shutil
 class Cleaner:
     @staticmethod
     def delete_local_files(
-            file_types=['*.raw*', '*.zarr_manager']  # '*.json'
+            file_types=['*.raw*', '*.model']  # '*.json'
     ):
-        print('Deleting all local raw and zarr_manager files')
+        print('Deleting all local raw and model files')
         for i in file_types:
             for j in glob.glob(i):
                 if os.path.isdir(j):

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5/src/water_column_sonar_processing.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: water_column_sonar_processing
-Version: 0.0.4
+Version: 0.0.5
 Summary: A processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing

water_column_sonar_processing-0.0.5/src/water_column_sonar_processing.egg-info/SOURCES.txt

@@ -0,0 +1,42 @@
+LICENSE
+README.md
+pyproject.toml
+src/water_column_sonar_processing/__init__.py
+src/water_column_sonar_processing/process.py
+src/water_column_sonar_processing.egg-info/PKG-INFO
+src/water_column_sonar_processing.egg-info/SOURCES.txt
+src/water_column_sonar_processing.egg-info/dependency_links.txt
+src/water_column_sonar_processing.egg-info/top_level.txt
+src/water_column_sonar_processing/aws/__init__.py
+src/water_column_sonar_processing/aws/dynamodb_manager.py
+src/water_column_sonar_processing/aws/s3_manager.py
+src/water_column_sonar_processing/aws/s3fs_manager.py
+src/water_column_sonar_processing/aws/sns_manager.py
+src/water_column_sonar_processing/aws/sqs_manager.py
+src/water_column_sonar_processing/cruise/__init__.py
+src/water_column_sonar_processing/cruise/create_empty_zarr_store.py
+src/water_column_sonar_processing/cruise/resample_regrid.py
+src/water_column_sonar_processing/geometry/__init__.py
+src/water_column_sonar_processing/geometry/geometry_manager.py
+src/water_column_sonar_processing/geometry/geometry_simplification.py
+src/water_column_sonar_processing/geometry/pmtile_generation.py
+src/water_column_sonar_processing/index/__init__.py
+src/water_column_sonar_processing/index/index_manager.py
+src/water_column_sonar_processing/model/__init__.py
+src/water_column_sonar_processing/model/zarr_manager.py
+src/water_column_sonar_processing/utility/__init__.py
+src/water_column_sonar_processing/utility/cleaner.py
+src/water_column_sonar_processing/utility/constants.py
+src/water_column_sonar_processing/utility/pipeline_status.py
+src/water_column_sonar_processing/utility/timestamp.py
+tests/test_create_empty_zarr_store.py
+tests/test_dynamodb_manager.py
+tests/test_geometry_manager.py
+tests/test_geometry_simplification.py
+tests/test_index.py
+tests/test_process.py
+tests/test_resample_regrid.py
+tests/test_s3_manager.py
+tests/test_s3fs_manager.py
+tests/test_sns_sqs_manager.py
+tests/test_zarr_manager.py

water_column_sonar_processing-0.0.5/src/water_column_sonar_processing.egg-info/top_level.txt

@@ -0,0 +1 @@
+water_column_sonar_processing

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_create_empty_zarr_store.py

@@ -1,12 +1,7 @@
 import os
-import pytest
 from moto import mock_aws
 from dotenv import load_dotenv, find_dotenv
-
-from cruise.create_empty_zarr_store import CreateEmptyZarrStore
-
-
-# from src.model.cruise.create_empty_zarr_store import CreateEmptyZarrStore
+from water_column_sonar_processing.cruise.create_empty_zarr_store import CreateEmptyZarrStore
 
 
 #######################################################
@@ -25,7 +20,6 @@ TEMPDIR = "/tmp"
 
 #######################################################
 @mock_aws
-# @pytest.mark.skip(reason="no way of currently testing this")
 def test_create_empty_zarr_store(tmp_path=TEMPDIR):
     # temporary_directory = str(tmp_path)
     temporary_directory = str(tmp_path)

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_dynamodb_manager.py

@@ -1,9 +1,9 @@
 import numpy as np
 from moto import mock_aws
 from dotenv import load_dotenv, find_dotenv
+from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
+from water_column_sonar_processing.utility.pipeline_status import PipelineStatus
 
-from aws_manager.dynamodb_manager import DynamoDBManager
-from utility.pipeline_status import PipelineStatus
 
 #######################################################
 def setup_module():
@@ -11,13 +11,10 @@ def setup_module():
     env_file = find_dotenv('.env-test')
     load_dotenv(dotenv_path=env_file, override=True)
 
-
 def teardown_module():
     print('teardown')
 
-
 #######################################################
-
 def test_serializer_deserializer():
     # ---Initialize--- #
     low_level_data = {
@@ -35,7 +32,7 @@ def test_serializer_deserializer():
                 "customer": {
                     "S": "TEST"
                 },
-                "index_manager": {
+                "index": {
                     "N": "1"
                 }
             }
@@ -58,7 +55,7 @@ def test_serializer_deserializer():
     assert python_data['ACTIVE']
     assert python_data['CRC'] == -1600155180
     assert python_data['ID']
-    assert python_data['params'] == {'customer': 'TEST', 'index_manager': 1}
+    assert python_data['params'] == {'customer': 'TEST', 'index': 1}
     assert python_data['THIS_STATUS'] == 10
     assert python_data['TYPE'] == 22
 
@@ -68,7 +65,6 @@ def test_serializer_deserializer():
 
     assert low_level_data == low_level_copy
 
-
 #######################################################
 @mock_aws
 def test_dynamodb_manager():

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_geometry_manager.py

@@ -1,17 +1,15 @@
 import numpy as np
 import echopype as ep
 from dotenv import load_dotenv, find_dotenv
+from water_column_sonar_processing.geometry.geometry_manager import GeometryManager
 
-from geometry_manager.geometry_manager import GeometryManager
 
 #######################################################
 def setup_module():
     print('setup')
     env_file = find_dotenv('.env-test')
-    # env_file = find_dotenv('.env-prod')
     load_dotenv(dotenv_path=env_file, override=True)
 
-
 def teardown_module():
     print('teardown')
 
@@ -29,7 +27,7 @@ def test_geometry_manager(tmp_path):
     sensor_name = 'EK60'
 
     s3_path = f"s3://{bucket_name}/data/raw/{ship_name}/{cruise_name}/{sensor_name}/{file_name}"
-    # s3_path = f"r2d2-testing-level-2-data/level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.zarr_manager"
+    # s3_path = f"r2d2-testing-level-2-data/level_2/Henry_B._Bigelow/HB0707/EK60/HB0707.model"
 
     print(s3_path)
 

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_geometry_simplification.py

@@ -1,9 +1,5 @@
-import pytest
-import unittest
 from dotenv import load_dotenv, find_dotenv
 
-# import os
-
 # s3fs.core.setup_logging("DEBUG")
 
 
@@ -33,7 +29,7 @@ def teardown_module():
 
 # @mock_s3
 def test_geometry_simplification():
-    # bucket_name = 'noaa-wcsd-zarr_manager-pds'
+    # bucket_name = 'noaa-wcsd-model-pds'
     pass
 
 #######################################################

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_index.py

@@ -1,9 +1,6 @@
 from dotenv import load_dotenv, find_dotenv
+from water_column_sonar_processing.index.index_manager import IndexManager
 
-from index_manager.index_manager import IndexManager
-
-
-# from src.model.index_manager.index_manager import IndexManager
 
 #######################################################
 def setup_module(module):
@@ -11,7 +8,6 @@ def setup_module(module):
     env_file = find_dotenv('.env-prod')
     load_dotenv(dotenv_path=env_file, override=True)
 
-
 def teardown_module(module):
     print('teardown')
 
@@ -19,7 +15,7 @@ def teardown_module(module):
 
 def test_get_calibration_information():
     input_bucket_name = 'noaa-wcsd-pds'
-    calibration_bucket = 'noaa-wcsd-pds-index_manager'
+    calibration_bucket = 'noaa-wcsd-pds-index'
     calibration_key = 'calibrated_cruises.csv'
 
     index_manager = IndexManager(
@@ -35,21 +31,21 @@ def test_get_calibration_information():
 # @mock_s3
 # def test_index_manager(tmp_path):
 #     input_bucket_name = 'noaa-wcsd-pds'
-#     calibration_bucket = 'noaa-wcsd-pds-index_manager'
+#     calibration_bucket = 'noaa-wcsd-pds-index'
 #     calibration_key = 'calibrated_crusies.csv'
 #
-#     index_manager = IndexManager(
+#     index = IndexManager(
 #         input_bucket_name,
 #         calibration_bucket,
 #         calibration_key
 #     )
 #
-#     all_ek60_data = index_manager.index_manager()
+#     all_ek60_data = index.index()
 #     print(all_ek60_data)
 
 def test_get_all_cruise_raw_files(tmp_path):
     input_bucket_name = 'noaa-wcsd-pds'
-    calibration_bucket = 'noaa-wcsd-pds-index_manager'
+    calibration_bucket = 'noaa-wcsd-pds-index'
     calibration_key = 'calibrated_crusies.csv'
 
     index_manager = IndexManager(
@@ -132,7 +128,7 @@ def test_get_all_cruise_raw_files(tmp_path):
     ]
 
     # TODO: for each verify ek60 datagram
-    # index_manager.get_raw_files_csv(
+    # index.get_raw_files_csv(
     #     ship_name='Henry_B._Bigelow',
     #     cruise_name='HB2206',
     #     sensor_name='EK60'
@@ -156,12 +152,12 @@ def test_get_all_cruise_raw_files(tmp_path):
             )
         else:
             print(f"{c_n} is not ek60")
-    # all_raw_files = index_manager.get_raw_files(ship_name='Bell_M._Shimada', cruise_name='SH1906', sensor_name='EK60')
+    # all_raw_files = index.get_raw_files(ship_name='Bell_M._Shimada', cruise_name='SH1906', sensor_name='EK60')
     # 'data/raw/Bell_M._Shimada/SH1204/EK60/'
-    # all_ek60_data = index_manager.index_manager()
+    # all_ek60_data = index.index()
     # print(all_ek60_data)
 
 #######################################################
 
 # TODO: for post analysis of coverage
-#  need to check each cruise has same number of files in noaa-wcsd-pds and noaa-wcsd-zarr_manager-pds buckets
+#  need to check each cruise has same number of files in noaa-wcsd-pds and noaa-wcsd-model-pds buckets

water_column_sonar_processing-0.0.4/tests/test_model.py → water_column_sonar_processing-0.0.5/tests/test_process.py

@@ -1,21 +1,14 @@
 import os
 import json
-# import pytest
-# import unittest
 import numpy as np
 from dotenv import load_dotenv, find_dotenv
 from moto import mock_aws
-# from aws_manager.dynamodb_manager import DynamoDBManager
-# from aws_manager.s3_manager import S3Manager
-# from aws_manager.sns_manager import SNSManager
-# from aws_manager.sqs_manager import SQSManager
-# from model import Model
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.aws.sns_manager import SNSManager
+from water_column_sonar_processing.aws.sqs_manager import SQSManager
+from water_column_sonar_processing.aws.dynamodb_manager import DynamoDBManager
+from water_column_sonar_processing.process import Process
 
-from aws_manager.s3_manager import S3Manager
-from aws_manager.sns_manager import SNSManager
-from aws_manager.sqs_manager import SQSManager
-from aws_manager.dynamodb_manager import DynamoDBManager
-from model import Model
 
 #######################################################
 def setup_module():
@@ -27,7 +20,6 @@ def setup_module():
 def teardown_module():
     print('teardown')
 
-
 #######################################################
 @mock_aws
 def test_model_happy_path():
@@ -116,7 +108,7 @@ def test_model_happy_path():
             ':sh': {'S': 'David_Starr_Jordan'},
             ':st': {'S': '2006-04-06T03:59:14.115Z'},
             ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-            ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T035914.zarr_manager'},
+            ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T035914.model'},
         },
         update_expression=(
             'SET '
@@ -172,7 +164,7 @@ def test_model_happy_path():
             ':sh': {'S': 'David_Starr_Jordan'},
             ':st': {'S': '2006-04-06T13:35:30.701Z'},
             ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-            ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T133530.zarr_manager'},
+            ':zp': {'S': 'level_1/David_Starr_Jordan/DS0604/EK60/DSJ0604-D20060406-T133530.model'},
         },
         update_expression=(
             'SET '
@@ -232,7 +224,7 @@ def test_model_happy_path():
             ':sh': {'S': 'Okeanos_Explorer'},
             ':st': {'S': '2014-09-08T17:39:07.660Z'},
             ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-            ':zp': {'S': 'level_1/Okeanos_Explorer/EX1404L2/EK60/EX1404L2_EK60_-D20140908-T173907.zarr_manager'},
+            ':zp': {'S': 'level_1/Okeanos_Explorer/EX1404L2/EK60/EX1404L2_EK60_-D20140908-T173907.model'},
         },
         update_expression=(
             'SET '
@@ -297,7 +289,7 @@ def test_model_happy_path():
             ':sh': {'S': 'Henry_B._Bigelow'},
             ':st': {'S': '2007-07-12T06:17:45.579Z'},
             ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-            ':zp': {'S': 'level_1/Henry_B._Bigelow/HB0707/EK60/D20070712-T061745.zarr_manager'},
+            ':zp': {'S': 'level_1/Henry_B._Bigelow/HB0707/EK60/D20070712-T061745.model'},
         },
         update_expression=(
             'SET '
@@ -362,7 +354,7 @@ def test_model_happy_path():
             ':sh': {'S': 'Miller_Freeman'},
             ':st': {'S': '2007-07-08T20:04:49.552Z'},
             ':zb': {'S': 'r2d2-dev-echofish2-118234403147-echofish-dev-output'},
-            ':zp': {'S': 'level_1/Miller_Freeman/MF0710/EK60/HAKE2007-D20070708-T200449.zarr_manager'},
+            ':zp': {'S': 'level_1/Miller_Freeman/MF0710/EK60/HAKE2007-D20070708-T200449.model'},
         },
         update_expression=(
             'SET '
@@ -424,7 +416,7 @@ def test_model_happy_path():
     ###### end setup ######
 
     #############################################################
-    model_instance = Model()
+    model_instance = Process()
     # run the src
     model_instance.execute()
     #############################################################

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_resample_regrid.py

@@ -1,18 +1,14 @@
 from dotenv import load_dotenv, find_dotenv
 import pytest
 from moto import mock_aws
-from cruise.resample_regrid import ResampleRegrid
+from water_column_sonar_processing.cruise.resample_regrid import ResampleRegrid
 
 #######################################################
 def setup_module():
     print('setup')
-
-    # env_file = find_dotenv('.env-test')
     env_file = find_dotenv('.env-prod')  # functional test
-
     load_dotenv(dotenv_path=env_file, override=True)
 
-
 def teardown_module():
     print('teardown')
 
@@ -23,18 +19,18 @@ def teardown_module():
 @mock_aws
 @pytest.mark.skip(reason="no way of currently testing this")
 def test_resample_regrid():
-    # Opens s3 input zarr_manager store as xr and writes data to output zarr_manager store
+    # Opens s3 input model store as xr and writes data to output model store
     resample_regrid = ResampleRegrid()
 
     # HB0706 - 53 files
-    # bucket_name = 'noaa-wcsd-zarr_manager-pds'
+    # bucket_name = 'noaa-wcsd-model-pds'
     ship_name = "Henry_B._Bigelow"
     cruise_name = "HB0706"
     sensor_name = "EK60"
-    # file_name = "D20070719-T232718.zarr_manager"  # first file
-    #file_name = "D20070720-T021024.zarr_manager"  # second file
-    #file_name = "D20070720-T224031.zarr_manager"  # third file, isn't in dynamodb
-    # "D20070719-T232718.zarr_manager"
+    # file_name = "D20070719-T232718.model"  # first file
+    #file_name = "D20070720-T021024.model"  # second file
+    #file_name = "D20070720-T224031.model"  # third file, isn't in dynamodb
+    # "D20070719-T232718.model"
     # file_name_stem = Path(file_name).stem  # TODO: remove
     table_name = "r2d2-dev-echofish-EchoFish-File-Info"
 

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_s3_manager.py

@@ -1,17 +1,11 @@
 import os
-
 from moto import mock_aws
 from dotenv import load_dotenv, find_dotenv
-
-from aws_manager.s3_manager import S3Manager, chunked
-
-# from aws_manager.s3_manager import S3Manager, chunked
-
+from water_column_sonar_processing.aws.s3_manager import S3Manager, chunked
 
 input_bucket_name = 'example_input_bucket'
 output_bucket_name = 'example_output_bucket'
 
-
 #######################################################
 def setup_module():
     print('setup')

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_s3fs_manager.py

@@ -3,12 +3,9 @@ import zarr
 import numpy as np
 import xarray as xr
 from dotenv import load_dotenv, find_dotenv
-# import moto
-
 from typing import Any  # Callable
 from moto import mock_aws
 from unittest.mock import MagicMock
-
 import aiobotocore.awsrequest
 import aiobotocore.endpoint
 import aiohttp
@@ -16,25 +13,8 @@ import aiohttp.client_reqrep
 import aiohttp.typedefs
 import botocore.awsrequest
 import botocore.model
-
-from aws_manager.s3_manager import S3Manager
-from aws_manager.s3fs_manager import S3FSManager
-
-
-# import pytest
-
-# # ### ATTEMPT TWO #@###
-# from collections.abc import Awaitable, Callable, Iterator
-# from dataclasses import dataclass
-# from typing import TypeVar
-# import aiobotocore
-# import aiobotocore.endpoint
-# import botocore
-# import botocore.retries.standard
-# from aws_manager.s3_manager import S3Manager
-# from aws_manager.s3fs_manager import S3FSManager
-# from aws_manager.s3_manager import S3Manager
-# from aws_manager.s3fs_manager import S3FSManager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.aws.s3fs_manager import S3FSManager
 
 
 #######################################################
@@ -45,11 +25,9 @@ def setup_module():
     # https://docs.getmoto.org/en/latest/docs/server_mode.html
     # free_port = 5000
 
-
 def teardown_module():
     print('teardown')
 
-
 #####################################################################
 # ### ATTEMPT ONE #@###
 class MockAWSResponse(aiobotocore.awsrequest.AioAWSResponse):
@@ -254,7 +232,7 @@ def test_add_file(tmp_path):
 
     # --- Create Local Zarr Store --- #
     temporary_directory = str(tmp_path)
-    zarr_path = f'{temporary_directory}/example.zarr_manager'
+    zarr_path = f'{temporary_directory}/example.model'
     ds = xr.Dataset(
         {
             "a": (("y", "x"), np.random.rand(6).reshape(2, 3)),
@@ -267,23 +245,23 @@ def test_add_file(tmp_path):
     # --- Upload to S3 --- #
     s3fs_manager = S3FSManager()
     # TODO: just copy from a to b
-    # foo = s3_manager.upload_files_to_bucket(local_directory=zarr_path, object_prefix='ship/cruise/sensor/example.zarr_manager', bucket_name=test_bucket_name)
-    # s3_manager.upload_file(zarr_path + '/.zmetadata', test_bucket_name, 'ship/cruise/sensor/example.zarr_manager/.zmetadata')
+    # foo = s3_manager.upload_files_to_bucket(local_directory=zarr_path, object_prefix='ship/cruise/sensor/example.model', bucket_name=test_bucket_name)
+    # s3_manager.upload_file(zarr_path + '/.zmetadata', test_bucket_name, 'ship/cruise/sensor/example.model/.zmetadata')
 
     # s3fs_manager.upload_data(
     #     bucket_name=test_bucket_name,
     #     file_path=zarr_path,
-    #     file_name='ship/cruise/sensor/example.zarr_manager'
+    #     file_name='ship/cruise/sensor/example.model'
     # )
 
-    # s3_object = s3_manager.get(bucket_name=test_bucket_name, key="ship/cruise/sensor/example.zarr_manager/.zmetadata")
+    # s3_object = s3_manager.get(bucket_name=test_bucket_name, key="ship/cruise/sensor/example.model/.zmetadata")
     # body = s3_object.get()["Body"].read().decode("utf-8")
     # print(body)
 
     ### The file is there, trying to copy with boto3, then mount with s3fs.... incompatible version of s3fs
 
-    # assert s3_manager.folder_exists_and_not_empty(test_bucket_name, "/example.zarr_manager")
-    # assert s3fs_manager.exists(f"{test_bucket_name}/ship/cruise/sensor/example.zarr_manager")
+    # assert s3_manager.folder_exists_and_not_empty(test_bucket_name, "/example.model")
+    # assert s3fs_manager.exists(f"{test_bucket_name}/ship/cruise/sensor/example.model")
     #
     #
     # TODO: need to upload the file!!
@@ -296,13 +274,13 @@ def test_add_file(tmp_path):
     # TODO: get this working with s3 client
     s3fs_manager.upload_data(bucket_name=test_bucket_name, file_path=zarr_path, prefix="ship/cruise/sensor")
 
-    found = s3_manager.list_objects(test_bucket_name, 'ship/cruise/sensor/example.zarr_manager')
+    found = s3_manager.list_objects(test_bucket_name, 'ship/cruise/sensor/example.model')
     print(found)
-    s3_object = s3_manager.get(bucket_name=test_bucket_name, key="ship/cruise/sensor/example.zarr_manager/.zgroup")
+    s3_object = s3_manager.get(bucket_name=test_bucket_name, key="ship/cruise/sensor/example.model/.zgroup")
     body = s3_object.get()["Body"].read().decode("utf-8")
     print(body)
 
-    s3_store = s3fs_manager.s3_map(s3_zarr_store_path=f"s3://{test_bucket_name}/ship/cruise/sensor/example.zarr_manager")
+    s3_store = s3fs_manager.s3_map(s3_zarr_store_path=f"s3://{test_bucket_name}/ship/cruise/sensor/example.model")
 
     # --- Test S3Map Opening Zarr store with Zarr for Writing --- #
     cruise_zarr = zarr.open(store=s3_store, mode="r+")  # , synchronizer=synchronizer) # TODO: test synchronizer

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_sns_sqs_manager.py

@@ -2,15 +2,8 @@ import os
 import json
 from dotenv import load_dotenv, find_dotenv
 from moto import mock_aws
-from aws_manager.sns_manager import SNSManager
-from aws_manager.sqs_manager import SQSManager
-
-
-# from aws_manager.sns_manager import SNSManager
-
-
-# from aws_manager.sns_manager import SNSManager
-# from aws_manager.sqs_manager import SQSManager
+from water_column_sonar_processing.aws.sns_manager import SNSManager
+from water_column_sonar_processing.aws.sqs_manager import SQSManager
 
 
 #######################################################

{water_column_sonar_processing-0.0.4 → water_column_sonar_processing-0.0.5}/tests/test_zarr_manager.py

@@ -1,44 +1,29 @@
-# import os
 import zarr
 import numcodecs
-# import numpy as np
-# import xarray as xr
-# from moto import mock_s3
 from moto import mock_aws
 from dotenv import load_dotenv, find_dotenv
-# import warnings
-
-# import gc
 import os
-# import time
-# import boto3
-# import shutil
 import numpy as np
-# import pandas as pd
 import xarray as xr
-
-from aws_manager.s3_manager import S3Manager
-from zarr_manager.zarr_manager import ZarrManager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.model.zarr_manager import ZarrManager
 
 
+TEMPDIR = "/tmp"
 #######################################################
-# def setup_module(module):
 def setup_module():
     print('setup')
     env_file = find_dotenv('.env-test')
     load_dotenv(dotenv_path=env_file, override=True)
 
-
 def teardown_module():
     print('teardown')
 
-TEMPDIR = "/tmp"
-
 #######################################################
 @mock_aws
 def test_zarr_manager(tmp_path=TEMPDIR):
-    # Tests creating zarr_manager store and opening with both xarray and
-    # zarr_manager libraries
+    # Tests creating model store and opening with both xarray and
+    # model libraries
 
     temporary_directory = str(tmp_path)
 
@@ -66,7 +51,7 @@ def test_zarr_manager(tmp_path=TEMPDIR):
     numcodecs.blosc.use_threads = False
     numcodecs.blosc.set_nthreads(1)
 
-    # synchronizer = zarr_manager.ProcessSynchronizer(f"/mnt/zarr_manager/{ship_name}_{cruise_name}.sync")
+    # synchronizer = model.ProcessSynchronizer(f"/mnt/model/{ship_name}_{cruise_name}.sync")
 
     cruise_zarr = zarr.open(store=f"{temporary_directory}/{cruise_name}.zarr", mode="r")  # synchronizer=synchronizer)
     print(cruise_zarr.info)
@@ -79,7 +64,7 @@ def test_zarr_manager(tmp_path=TEMPDIR):
     file_xr = xr.open_zarr(store=f"{temporary_directory}/{cruise_name}.zarr", consolidated=None)  # synchronizer=SYNCHRONIZER)
     print(file_xr)
 
-    # for newly initialized zarr_manager store all the timestamps will be 0 epoch time
+    # for newly initialized model store all the timestamps will be 0 epoch time
     assert file_xr.time.values[0] == np.datetime64('1970-01-01T00:00:00.000000000')
     assert str(file_xr.time.values[0].dtype) == 'datetime64[ns]'
 
@@ -110,14 +95,14 @@ def test_zarr_manager(tmp_path=TEMPDIR):
 def test_open_zarr_with_zarr_read_write(tmp_path):
     temporary_directory = str(tmp_path)
 
-    # TODO: open with zarr_manager python library and check format
+    # TODO: open with model python library and check format
     test_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
 
     # create a bucket
     s3_manager = S3Manager()
     s3_manager.create_bucket(bucket_name=test_bucket_name)
 
-    # initialize zarr_manager store
+    # initialize model store
     ship_name = "test_ship"
     cruise_name = "test_cruise"
     sensor_name = "test_sensor"
@@ -138,7 +123,7 @@ def test_open_zarr_with_zarr_read_write(tmp_path):
 
     # TODO: copy store to bucket
 
-    # TODO: open zarr_manager store with zarr_manager
+    # TODO: open model store with model
     pass
 
 
@@ -149,7 +134,7 @@ def test_open_zarr_with_xarray(tmp_path):
     #  [1] check timestamps are in proper format
     #  [2] check that lat/lons are formatted (need data)
     #  [3] check
-    # TODO: open with zarr_manager python library and check format
+    # TODO: open with model python library and check format
     ship_name = "Okeanos_Explorer"
     cruise_name = "EX1404L2"
     sensor_name = "EK60"
@@ -163,8 +148,8 @@ def test_open_zarr_with_xarray(tmp_path):
     s3_manager = S3Manager()
     s3_manager.create_bucket(bucket_name=bucket_name)
 
-    # initialize zarr_manager store
-    # zarr_name = f"{cruise_name}.zarr_manager"
+    # initialize model store
+    # zarr_name = f"{cruise_name}.model"
     min_echo_range = 0.50
     max_echo_range = 250.0
 
@@ -189,13 +174,13 @@ def test_open_zarr_with_xarray(tmp_path):
     )
 
     # copy store to bucket
-    # TODO: create function to get list of files in zarr_manager store
+    # TODO: create function to get list of files in model store
     # s3_manager.upload_files_to_bucket(
     #       local_path,  # TODO: change to path
     #       s3_path
     # )
 
-    # open zarr_manager store with zarr_manager
+    # open model store with model
 
     #assert root.Sv.shape == (501, 1201, 4)
 

water_column_sonar_processing-0.0.4/src/water_column_sonar_processing.egg-info/SOURCES.txt

@@ -1,42 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-src/__init__.py
-src/model.py
-src/aws_manager/__init__.py
-src/aws_manager/dynamodb_manager.py
-src/aws_manager/s3_manager.py
-src/aws_manager/s3fs_manager.py
-src/aws_manager/sns_manager.py
-src/aws_manager/sqs_manager.py
-src/cruise/__init__.py
-src/cruise/create_empty_zarr_store.py
-src/cruise/resample_regrid.py
-src/geometry_manager/__init__.py
-src/geometry_manager/geometry_manager.py
-src/geometry_manager/geometry_simplification.py
-src/geometry_manager/pmtile_generation.py
-src/index_manager/__init__.py
-src/index_manager/index_manager.py
-src/utility/__init__.py
-src/utility/cleaner.py
-src/utility/constants.py
-src/utility/pipeline_status.py
-src/utility/timestamp.py
-src/water_column_sonar_processing.egg-info/PKG-INFO
-src/water_column_sonar_processing.egg-info/SOURCES.txt
-src/water_column_sonar_processing.egg-info/dependency_links.txt
-src/water_column_sonar_processing.egg-info/top_level.txt
-src/zarr_manager/__init__.py
-src/zarr_manager/zarr_manager.py
-tests/test_create_empty_zarr_store.py
-tests/test_dynamodb_manager.py
-tests/test_geometry_manager.py
-tests/test_geometry_simplification.py
-tests/test_index.py
-tests/test_model.py
-tests/test_resample_regrid.py
-tests/test_s3_manager.py
-tests/test_s3fs_manager.py
-tests/test_sns_sqs_manager.py
-tests/test_zarr_manager.py

water_column_sonar_processing-0.0.4/src/water_column_sonar_processing.egg-info/top_level.txt

@@ -1,8 +0,0 @@
-__init__
-aws_manager
-cruise
-geometry_manager
-index_manager
-model
-utility
-zarr_manager