water-column-sonar-processing 0.0.1__py3-none-any.whl → 25.11.1__py3-none-any.whl

model/zarr/zarr_manager.py

@@ -1,298 +0,0 @@
-import os
-import zarr
-import numcodecs
-import numpy as np
-import xarray as xr
-from numcodecs import Blosc
-from ..aws.s3fs_manager import S3FSManager
-from ..utility.constants import Constants, Coordinates
-from ..utility.timestamp import Timestamp
-
-numcodecs.blosc.use_threads = False
-numcodecs.blosc.set_nthreads(1)
-
-
-# TODO: when ready switch to version 3 of zarr spec
-# ZARR_V3_EXPERIMENTAL_API = 1
-
-# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
-
-class ZarrManager:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        # TODO: revert to Blosc.BITSHUFFLE, troubleshooting misc error
-        self.__compressor = Blosc(cname="zstd", clevel=2)  # shuffle=Blosc.NOSHUFFLE
-        self.__overwrite = True
-        self.__num_threads = numcodecs.blosc.get_nthreads()
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-
-    #######################################################
-    @staticmethod
-    def get_depth_values(
-            min_echo_range: float = 1.,  # minimum depth measured (zero non-inclusive) from whole cruise
-            max_echo_range: float = 100.,  # maximum depth measured from whole cruise
-    ):
-        # Gets the set of depth values that will be used when resampling and
-        # regridding the data to a cruise level zarr store.
-        # Note: returned values do not start at zero.
-        print('Getting depth values.')
-        all_cruise_depth_values = np.linspace(
-            start=min_echo_range,
-            stop=max_echo_range,
-            num=int(max_echo_range / min_echo_range) + 1,
-            endpoint=True
-        )
-
-        print("Done getting depth values.")
-        return all_cruise_depth_values.round(decimals=2)
-
-    #######################################################
-    def create_zarr_store(
-            self,
-            path: str,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            frequencies: list,  # units in Hz
-            width: int,  # TODO: needs better name... "ping_time"
-            min_echo_range: float,  # smallest resolution in meters
-            max_echo_range: float,
-            calibration_status: bool = False  # Assume uncalibrated
-    ) -> str:
-        print(f'Creating local zarr store at {cruise_name}.zarr for ship {ship_name}')
-
-        # There should be no repeated frequencies
-        assert len(frequencies) == len(set(frequencies))
-        # TODO: eventually switch coordinate to "channel"
-
-        print(f"Debugging number of threads: {self.__num_threads}")
-
-        zarr_path = f"{path}/{cruise_name}.zarr"
-        store = zarr.DirectoryStore(path=zarr_path, normalize_keys=False)
-        root = zarr.group(store=store, overwrite=self.__overwrite, cache_attrs=True)
-
-        #####################################################################
-        # --- Coordinate: Time --- #
-        # https://zarr.readthedocs.io/en/stable/spec/v2.html#data-type-encoding
-        root.create_dataset(
-            name=Coordinates.TIME.value,
-            data=np.repeat(0., width),
-            shape=width,
-            chunks=(Constants.TILE_SIZE.value, ),  # TODO: the chunking scheme doesn't seem to be working here
-            dtype=np.dtype(Coordinates.TIME_DTYPE.value),
-            compressor=self.__compressor,
-            # fill_value=0.,
-            fill_value=np.nan,  # TODO: do i want nan's?
-            overwrite=self.__overwrite
-        )
-
-        root.time.attrs['_ARRAY_DIMENSIONS'] = [Coordinates.TIME.value]
-
-        root.time.attrs['calendar'] = Coordinates.TIME_CALENDAR.value
-        root.time.attrs['units'] = Coordinates.TIME_UNITS.value
-        root.time.attrs['long_name'] = Coordinates.TIME_LONG_NAME.value
-        root.time.attrs['standard_name'] = Coordinates.TIME_STANDARD_NAME.value
-
-        #####################################################################
-        # --- Coordinate: Depth --- #
-        depth_values = self.get_depth_values(
-            min_echo_range=min_echo_range,
-            max_echo_range=max_echo_range
-        )
-
-        root.create_dataset(
-            name=Coordinates.DEPTH.value,
-            # TODO: verify that these values are correct
-            data=depth_values,
-            shape=len(depth_values),
-            chunks=Constants.TILE_SIZE.value,
-            dtype=np.dtype(Coordinates.DEPTH_DTYPE.value),  # float16 == 2 significant digits would be ideal
-            compressor=self.__compressor,
-            # fill_value=np.nan,
-            overwrite=self.__overwrite
-        )
-        # TODO: change to exception
-        assert not np.any(np.isnan(depth_values))
-
-        root.depth.attrs['_ARRAY_DIMENSIONS'] = [Coordinates.DEPTH.value]
-
-        root.depth.attrs['long_name'] = Coordinates.DEPTH_LONG_NAME.value
-        root.depth.attrs['units'] = Coordinates.DEPTH_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Latitude --- #
-        root.create_dataset(
-            name=Coordinates.LATITUDE.value,
-            data=np.repeat(0., width),
-            shape=width,
-            chunks=Constants.TILE_SIZE.value,
-            dtype=np.dtype(Coordinates.LATITUDE_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=0.,
-            overwrite=self.__overwrite
-        )
-
-        root.latitude.attrs['_ARRAY_DIMENSIONS'] = [Coordinates.TIME.value]
-
-        root.latitude.attrs['long_name'] = Coordinates.LATITUDE_LONG_NAME.value
-        root.latitude.attrs['units'] = Coordinates.LATITUDE_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Longitude --- #
-        root.create_dataset(
-            name=Coordinates.LONGITUDE.value,
-            data=np.repeat(0., width),  # root.longitude[:] = np.nan
-            shape=width,
-            chunks=Constants.TILE_SIZE.value,
-            dtype=np.dtype(Coordinates.LONGITUDE_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=0.,
-            overwrite=self.__overwrite
-        )
-
-        root.longitude.attrs['_ARRAY_DIMENSIONS'] = [Coordinates.TIME.value]
-
-        root.longitude.attrs['long_name'] = Coordinates.LONGITUDE_LONG_NAME.value
-        root.longitude.attrs['units'] = Coordinates.LONGITUDE_UNITS.value
-
-        #####################################################################
-        # --- Coordinate: Frequency --- #
-        root.create_dataset(
-            name=Coordinates.FREQUENCY.value,
-            data=frequencies,
-            shape=len(frequencies),
-            chunks=1,
-            dtype=np.dtype(Coordinates.FREQUENCY_DTYPE.value),
-            compressor=self.__compressor,
-            fill_value=0.,
-            overwrite=self.__overwrite
-        )
-
-        # TODO: best coordinate would be channel with str type
-        root.frequency.attrs['_ARRAY_DIMENSIONS'] = [Coordinates.FREQUENCY.value]  # TODO: is this correct
-
-        root.frequency.attrs['long_name'] = Coordinates.FREQUENCY_LONG_NAME.value
-        root.frequency.attrs['standard_name'] = Coordinates.FREQUENCY_STANDARD_NAME.value
-        root.frequency.attrs['units'] = Coordinates.FREQUENCY_UNITS.value
-
-        #####################################################################
-        # --- Sv Data --- #
-        root.create_dataset(
-            name=Coordinates.SV.value,
-            shape=(len(depth_values), width, len(frequencies)),
-            chunks=(Constants.TILE_SIZE.value, Constants.TILE_SIZE.value, 1),
-            dtype=np.dtype(Coordinates.SV_DTYPE.value),  # TODO: try to experiment with 'float16'
-            compressor=self.__compressor,
-            fill_value=np.nan,
-            overwrite=self.__overwrite
-        )
-
-        root.Sv.attrs['_ARRAY_DIMENSIONS'] = [
-            Coordinates.DEPTH.value,
-            Coordinates.TIME.value,
-            Coordinates.FREQUENCY.value,
-        ]
-
-        root.Sv.attrs['long_name'] = Coordinates.SV_LONG_NAME.value
-        root.Sv.attrs['units'] = Coordinates.SV_UNITS.value
-        root.Sv.attrs['tile_size'] = Constants.TILE_SIZE.value
-
-        #####################################################################
-        # --- Metadata --- #
-        root.attrs["ship_name"] = ship_name
-        root.attrs["cruise_name"] = cruise_name
-        root.attrs["sensor_name"] = sensor_name
-        #
-        root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-        root.attrs["processing_software_version"] = "0.0.2"  # TODO: get programmatically
-        root.attrs["processing_software_time"] = Timestamp.get_timestamp()
-        #
-        root.attrs["calibration_status"] = calibration_status
-
-        zarr.consolidate_metadata(store)
-        #####################################################################
-        """
-        # zzz = zarr.open('https://echofish-dev-master-118234403147-echofish-zarr-store.s3.us-west-2.amazonaws.com/GU1002_resample.zarr')
-        # zzz.time[0] = 1274979445.423
-        # Initialize all to origin time, will be overwritten late
-        """
-        return zarr_path
-
-    ############################################################################
-    # def update_zarr_store(
-    #         self,
-    #         path: str,
-    #         ship_name: str,
-    #         cruise_name: str,  # TODO: just pass stem
-    #         sensor_name: str,
-    # ) -> None:
-    #     """
-    #     Opens an existing Zarr store living in a s3 bucket for the purpose
-    #     of updating just a subset of the cruise-level Zarr store associated
-    #     with a file-level Zarr store.
-    #     """
-    #     pass
-
-    ############################################################################
-    def open_s3_zarr_store_with_zarr(
-            self,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            # zarr_synchronizer: Union[str, None] = None,
-    ):
-        # Mounts a Zarr store using pythons Zarr implementation. The mounted store
-        #  will have read/write privileges so that store can be updated.
-        print('Opening Zarr store with Zarr.')
-        try:
-            s3fs_manager = S3FSManager()
-            root = f'{self.output_bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{cruise_name}.zarr'
-            store = s3fs_manager.s3_map(s3_zarr_store_path=root)
-            # synchronizer = zarr.ProcessSynchronizer(f"/tmp/{ship_name}_{cruise_name}.sync")
-            cruise_zarr = zarr.open(store=store, mode="r+")
-        except Exception as err:  # Failure
-            print(f'Exception encountered opening Zarr store with Zarr.: {err}')
-            raise
-        print('Done opening Zarr store with Zarr.')
-        return cruise_zarr
-
-    ############################################################################
-    def open_s3_zarr_store_with_xarray(
-            self,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            file_name_stem: str,
-    ) -> xr.Dataset:
-        print('Opening Zarr store in S3 as Xarray.')
-        try:
-            zarr_path = f"s3://{self.output_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
-            s3fs_manager = S3FSManager()
-            store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
-            ds = xr.open_zarr(store=store_s3_map, consolidated=None)  # synchronizer=SYNCHRONIZER
-        except Exception as err:
-            print('Problem opening Zarr store in S3 as Xarray.')
-            raise err
-        print("Done opening Zarr store in S3 as Xarray.")
-        return ds
-
-    ############################################################################
-
-    #######################################################
-    # def create_process_synchronizer(self):
-    #     # TODO: explore aws redis options
-    #     pass
-
-    #######################################################
-    # def verify_cruise_store_data(self):
-    #     # TODO: run a check on a finished zarr store to ensure that
-    #     #   none of the time, latitude, longitude, or depth values
-    #     #   are NaN.
-    #     pass
-
-    #######################################################
-
-###########################################################

water_column_sonar_processing-0.0.1.dist-info/METADATA

@@ -1,89 +0,0 @@
-Metadata-Version: 2.1
-Name: water-column-sonar-processing
-Version: 0.0.1
-Summary: A processing tool for water column sonar data.
-Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
-Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
-Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-
-# water-column-sonar-processing
-Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
-
-## Setting up the Python Environment
-> Python 3.10.12
-
-# MacOS Pyenv Installation Instructions
-  1. Install pyenv (https://github.com/pyenv/pyenv#set-up-your-shell-environment-for-pyenv)
-     1. ```brew update```
-     2. ```arch -arm64 brew install pyenv```
-     3. In ~/.bashrc add
-        1. ```export PYENV_ROOT="$HOME/.pyenv"```
-        2. ```export PATH="$PYENV_ROOT/bin:$PATH"```
-        3. ```eval "$(pyenv init -)"```
-     4. ```arch -arm64 brew install openssl readline sqlite3 xz zlib tcl-tk```
-  2. Install pyenv-virtualenv (https://github.com/pyenv/pyenv-virtualenv)
-     1. ```arch -arm64 brew install pyenv-virtualenv```
-     2. In ~/.bashrc add
-         1. ```eval "$(pyenv virtualenv-init -)"```
-  3. Open a new terminal
-  4. Install Python version
-     1. ```env PYTHON_CONFIGURE_OPTS="--enable-shared"```
-     2. ```env CONFIGURE_OPTS='--enable-optimizations' arch -arm64 pyenv install 3.10.12```
-  5. Create virtual env (to delete 'pyenv uninstall 3.10.12/water-column-sonar-processing')
-     1. ```pyenv virtualenv 3.10.12 water-column-sonar-processing```
-  6. Set local version of python (if not done already)
-     1. change directory to root of project
-     2. ```pyenv local 3.10.12 water-column-sonar-processing```
-     3. ```pyenv activate water-column-sonar-processing```
-
-## Setting up IntelliJ
-
-  1. Install the IntelliJ Python plugin
-  2. Set up pyenv
-     1. File -> Project Structure or CMD + ;
-     2. SDKs -> + -> Add Python SDK -> Virtual Environment
-     3. Select Existing Environment
-     4. Choose ~/.pyenv/versions/mocking_aws/bin/python
-  3. Set up Python Facet (not sure if this is required)
-     1. File -> Project Structure or CMD + ;
-     2. Facets -> + -> Python 
-     3. Set interpreter 
-
-## Installing Dependencies
-
-  1. Add dependencies with versions to requirements.txt
-  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
-
-
-## Pytest
-```commandline
-pytest --disable-warnings
-```
-
-## Instructions
-Following this tutorial:
-https://packaging.python.org/en/latest/tutorials/packaging-projects/
-
-## To Publish To TEST
-```commandline
-python -m build
-python -m twine upload --repository testpypi dist/*
-pytho -m pip install --index-url https://test.pypi.org/simple/ hello-pypi-rudy-klucik
-python
-```
-```python
-from water-column-sonar-processing import ZarrManager
-example.add_one(2)
-```
-
-## To Publish To PROD
-```commandline
-python -m build
-python -m twine upload --repository pypi dist/*
-```

water_column_sonar_processing-0.0.1.dist-info/RECORD

@@ -1,32 +0,0 @@
-__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/model.py,sha256=JjhqZM2t8evZrFjbFHBOdHH0tlG_twOsysRMoIh2Zdw,5099
-model/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/aws/dynamodb_manager.py,sha256=fppu4Kvl-22psu8MrF-NGKJwymwVf5CyvO7-Ye8zeFc,5338
-model/aws/s3_manager.py,sha256=_DcBOYRcuyRTcmlTQ1qzMCtE0GRcGU8LaZ-JXKGKaOo,13397
-model/aws/s3fs_manager.py,sha256=k4nW48E0reQwj1t4pe_ybK5kX0H1SHeG8urddHelGZ4,2463
-model/aws/sns_manager.py,sha256=5ZW0bi7BQHldxBbmAxN4tT1KVfDAF_eYPJwpa_W1C7Q,2202
-model/aws/sqs_manager.py,sha256=01y3LB5B_R0rmxB00SWLwL2rBUSDy_8OgjpZImEuxYQ,1694
-model/cruise/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/cruise/create_empty_zarr_store.py,sha256=gbQWvMsmmGPl359aakYF--x09OMOzlOmk8T1aIOe-Bs,6907
-model/cruise/resample_regrid.py,sha256=Nhb1yMRG7dCYEgsS4fPxtlJ_iYKMcdgcOHPy_Xxx9kw,11228
-model/geospatial/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/geospatial/geometry_manager.py,sha256=ele0bd8sDp2iyOjx9scZay8Iy3D-IhGO8L3DlPOonWs,9400
-model/geospatial/geometry_simplification.py,sha256=_TCrlxqWlhO05Od3ZB4Sfgob9p3tQr3r1vjseG0qMNQ,3071
-model/geospatial/pmtile_generation.py,sha256=ND9rqAH4ow6FkRr_6yFF0o0DBby8u0zRVj6cqci2cPw,2790
-model/index/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/index/index.py,sha256=s9GhEJHKF5urHWqjehHeral-lQlmTS1Zx1tvlQ7vcEM,10705
-model/utility/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/utility/cleaner.py,sha256=l4G3iRso1xOTGH8hBXRRVPEn1ZTOJJT9_9JA0yRpcyA,599
-model/utility/constants.py,sha256=uTuSyH9YGG8bDd5xNbT4zA5O_lu874C-8m1MRvVppPw,1631
-model/utility/pipeline_status.py,sha256=iRLyzI3gsesyz0I4NukbJwqiRmXgSZBOdB3C1x2eMSg,4185
-model/utility/timestamp.py,sha256=i4ub7ob24edHU2yt4aOmvle_Nmlf1F_9HVGa9TJ5HgM,359
-model/zarr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-model/zarr/bar.py,sha256=0mBMTw_QmZZdEWCscRsCk0Axw6B6Yy1ujvuG1QuHcQo,504
-model/zarr/foo.py,sha256=KpXmnl2_Tp2Ig1A_WPnoWuMVhi85CvC-JIaPe6LtbTg,215
-model/zarr/zarr_manager.py,sha256=DsQsBe-lN4e2MExKRCg7PIFyra5GG_vTGKqA9hHwKS0,12357
-water_column_sonar_processing-0.0.1.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
-water_column_sonar_processing-0.0.1.dist-info/METADATA,sha256=tY2NjTDmibt3kJBvEhqAzA1xU1i-UpeZSElMBgdXvBg,3163
-water_column_sonar_processing-0.0.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-water_column_sonar_processing-0.0.1.dist-info/top_level.txt,sha256=Uq9bL3hpULwyr0JXiGOnwhgNE1h68gVXxofcjXVZpPo,15
-water_column_sonar_processing-0.0.1.dist-info/RECORD,,

water_column_sonar_processing-0.0.1.dist-info/top_level.txt

@@ -1,2 +0,0 @@
-__init__
-model