water-column-sonar-processing 25.1.3__py3-none-any.whl → 25.1.5__py3-none-any.whl

water_column_sonar_processing/aws/s3fs_manager.py

@@ -16,7 +16,7 @@ class S3FSManager:
         # self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
         self.s3_region = os.environ.get("AWS_REGION", default="us-east-1")
         self.s3fs = s3fs.S3FileSystem(
-            asynchronous=False,
+            # asynchronous=False,
             endpoint_url=endpoint_url,
             key=os.environ.get("OUTPUT_BUCKET_ACCESS_KEY"),
             secret=os.environ.get("OUTPUT_BUCKET_SECRET_ACCESS_KEY"),

water_column_sonar_processing/cruise/resample_regrid.py

@@ -197,9 +197,9 @@ class ResampleRegrid:
                 #  df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
 
                 # Get index from all cruise files. Note: should be based on which are included in cruise.
-                index = cruise_df.index[
+                index = int(cruise_df.index[
                     cruise_df["FILE_NAME"] == f"{file_name_stem}.raw"
-                ][0]
+                ][0])
 
                 # get input store
                 input_xr_zarr_store = zarr_manager.open_s3_zarr_store_with_xarray(
@@ -226,18 +226,20 @@ class ResampleRegrid:
                 min_echo_range = np.nanmin(np.float32(cruise_df["MIN_ECHO_RANGE"]))
                 max_echo_range = np.nanmax(np.float32(cruise_df["MAX_ECHO_RANGE"]))
 
-                print(
-                    "Creating empty ndarray for Sv data."
-                )  # Note: cruise_zarr dimensions are (depth, time, frequency)
+                print("Creating empty ndarray for Sv data.")  # Note: cruise dims (depth, time, frequency)
+                output_zarr_store_shape = output_zarr_store.Sv.shape
+                end_ping_time_index - start_ping_time_index
+                output_zarr_store_height = output_zarr_store_shape[0]
+                output_zarr_store_width = end_ping_time_index - start_ping_time_index
+                output_zarr_store_depth = output_zarr_store_shape[2]
                 cruise_sv_subset = np.empty(
-                    shape=output_zarr_store.Sv[
-                        :, start_ping_time_index:end_ping_time_index, :
-                    ].shape
+                    shape=(output_zarr_store_height, output_zarr_store_width, output_zarr_store_depth)
                 )
                 cruise_sv_subset[:, :, :] = np.nan
 
                 all_cruise_depth_values = zarr_manager.get_depth_values(
-                    min_echo_range=min_echo_range, max_echo_range=max_echo_range
+                    min_echo_range=min_echo_range,
+                    max_echo_range=max_echo_range
                 ) # (5262,) and
 
                 print(" ".join(list(input_xr_zarr_store.Sv.dims)))
@@ -280,17 +282,9 @@ class ResampleRegrid:
                 print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
                 #########################################################################
                 # write Sv values to cruise-level-model-store
-                output_zarr_store.Sv[:, start_ping_time_index:end_ping_time_index, :] = regrid_resample.values
-
-                #########################################################################
-                # [5] write subset of latitude/longitude
-                output_zarr_store.latitude[
-                    start_ping_time_index:end_ping_time_index
-                ] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
-                output_zarr_store.longitude[
-                    start_ping_time_index:end_ping_time_index
-                ] = geospatial.dropna()["longitude"].values
 
+                for fff in range(regrid_resample.shape[-1]):
+                    output_zarr_store.Sv[:, start_ping_time_index:end_ping_time_index, fff] = regrid_resample.values[:, :, fff]
                 #########################################################################
                 # TODO: add the "detected_seafloor_depth/" to the
                 #  L2 cruise dataarrays
@@ -311,11 +305,14 @@ class ResampleRegrid:
                         start_ping_time_index:end_ping_time_index
                     ] = detected_seafloor_depths
                 #
-                #
-                #
-                # TODO: write the time variable last so that I can parse that as check
-                #
-                #
+                #########################################################################
+                # [5] write subset of latitude/longitude
+                output_zarr_store.latitude[
+                    start_ping_time_index:end_ping_time_index
+                ] = geospatial.dropna()["latitude"].values # TODO: get from ds_sv directly, dont need geojson anymore
+                output_zarr_store.longitude[
+                    start_ping_time_index:end_ping_time_index
+                ] = geospatial.dropna()["longitude"].values
                 #########################################################################
                 #########################################################################
         except Exception as err:

water_column_sonar_processing/model/zarr_manager.py

@@ -2,6 +2,7 @@ import numcodecs
 import numpy as np
 import xarray as xr
 import zarr
+import importlib.metadata
 from numcodecs import Blosc
 
 from water_column_sonar_processing.aws import S3FSManager
@@ -249,9 +250,9 @@ class ZarrManager:
         root.attrs["sensor_name"] = sensor_name
         #
         root.attrs["processing_software_name"] = Coordinates.PROJECT_NAME.value
-        root.attrs["processing_software_version"] = (
-            "25.1.3"  # TODO: get programmatically, echopype>utils>prov.py
-        )
+
+        current_project_version = importlib.metadata.version('water_column_sonar_processing')
+        root.attrs["processing_software_version"] = current_project_version
         root.attrs["processing_software_time"] = Timestamp.get_timestamp()
         #
         root.attrs["calibration_status"] = calibration_status
@@ -290,7 +291,7 @@ class ZarrManager:
         # zarr_synchronizer: Union[str, None] = None, # TODO:
         output_bucket_name: str,
         endpoint_url=None,
-    ):
+    ) -> zarr.hierarchy.Group:
         # Mounts a Zarr store using pythons Zarr implementation. The mounted store
         #  will have read/write privileges so that store can be updated.
         print("Opening L2 Zarr store with Zarr for writing.")
@@ -316,18 +317,21 @@ class ZarrManager:
         input_bucket_name: str,
         endpoint_url=None,
     ) -> xr.Dataset:
-        print("Opening L1 Zarr store in S3 with Xarray.")
+        print("Opening L1 Zarr store in S3 with Xarray.") # TODO: Is this only used for reading from?
         try:
             zarr_path = f"s3://{input_bucket_name}/level_1/{ship_name}/{cruise_name}/{sensor_name}/{file_name_stem}.zarr"
             s3fs_manager = S3FSManager(endpoint_url=endpoint_url)
             store_s3_map = s3fs_manager.s3_map(s3_zarr_store_path=zarr_path)
-            ds = xr.open_zarr(
-                store=store_s3_map, consolidated=None
-            )  # synchronizer=SYNCHRONIZER
+            ds = xr.open_dataset(
+                filename_or_obj=store_s3_map,
+                engine="zarr",
+                chunks={}
+            )
         except Exception as err:
             print("Problem opening Zarr store in S3 as Xarray.")
             raise err
-        print("Done opening Zarr store in S3 as Xarray.")
+        finally:
+            print("Exiting opening Zarr store in S3 as Xarray.")
         return ds
 
     def open_l2_zarr_store_with_xarray(

water_column_sonar_processing/utility/constants.py

@@ -3,7 +3,7 @@ from enum import Enum, Flag, unique
 
 @unique
 class Constants(Flag):
-    TILE_SIZE = 1024 # TODO: add tile size to metadata?
+    TILE_SIZE = 1024
 
     # Average https://noaa-wcsd-zarr-pds.s3.us-east-1.amazonaws.com/level_2/Henry_B._Bigelow/HB0902/EK60/HB0902.zarr/time/927
     # chunk size is ~1.3 kB, HB0902 cruise takes ~30 seconds to load all time/lat/lon data

{water_column_sonar_processing-25.1.3.dist-info → water_column_sonar_processing-25.1.5.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.2
 Name: water_column_sonar_processing
-Version: 25.1.3
-Summary: A processing tool for water column sonar data.
+Version: 25.1.5
+Summary: Processing tool for water column sonar data.
 Author-email: Rudy Klucik <rudy.klucik@noaa.gov>
 Project-URL: Homepage, https://github.com/CI-CMG/water-column-sonar-processing
 Project-URL: Issues, https://github.com/CI-CMG/water-column-sonar-processing/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiobotocore==2.19.0
@@ -34,6 +34,19 @@ Requires-Dist: typing-extensions==4.10.0
 Requires-Dist: xarray==2024.10.0
 Requires-Dist: xbatcher==0.4.0
 Requires-Dist: zarr==2.18.3
+Provides-Extra: dev
+Requires-Dist: bandit[toml]==1.8.0; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pyinstaller; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: flake8==7.1.1; extra == "dev"
+Requires-Dist: pooch==1.8.2; extra == "dev"
+Requires-Dist: pytest~=8.3.3; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: bandit; extra == "dev"
+Provides-Extra: test
+Requires-Dist: pytest-cov; extra == "test"
 
 # Water Column Sonar Processing
 Processing tool for converting L0 data to L1 and L2 as well as generating geospatial information
@@ -80,14 +93,17 @@ Processing tool for converting L0 data to L1 and L2 as well as generating geospa
      3. Set interpreter 
 
 # Installing Dependencies
-
-  1. Add dependencies with versions to requirements.txt
-  2. ```pip install --upgrade pip && pip install -r requirements_dev.txt```
+```
+uv pip install --upgrade pip
+#uv pip install -r requirements_dev.txt
+uv pip install -r pyproject.toml --extra dev
+```
 
 
 # Pytest
 ```commandline
-pytest --disable-warnings
+uv run pytest tests
+#pytest --disable-warnings
 ```
 or
 > pytest --cache-clear --cov=src tests/ --cov-report=xml
@@ -120,7 +136,7 @@ https://colab.research.google.com/drive/1KiLMueXiz9WVB9o4RuzYeGjNZ6PsZU7a#scroll
 # Tag a Release
 Step 1 --> increment the semantic version in the zarr_manager.py "metadata" & the "pyproject.toml"
 ```commandline
-git tag -a v25.1.2 -m "Releasing version v25.1.2"
+git tag -a v25.1.5 -m "Releasing version v25.1.5"
 git push origin --tags
 ```
 

{water_column_sonar_processing-25.1.3.dist-info → water_column_sonar_processing-25.1.5.dist-info}/RECORD

@@ -3,13 +3,13 @@ water_column_sonar_processing/process.py,sha256=-yQtK3rnZq6lGAr3q02zLDe1NuMH9c0P
 water_column_sonar_processing/aws/__init__.py,sha256=KJqK8oYMn-u8n8i-Jp_lG5BvCOTjwWSjWP8yAyDlWVo,297
 water_column_sonar_processing/aws/dynamodb_manager.py,sha256=htP4Y2rmOSFtdzUFrgK14Bn-UXAFG22Ow-dDrR2alSw,13949
 water_column_sonar_processing/aws/s3_manager.py,sha256=-PCiW7YF31nGIPa1oVOVTzjTSExAAkT_IyNNnvWv2HU,16214
-water_column_sonar_processing/aws/s3fs_manager.py,sha256=Vo-DXj6vgb8t1l4LdtNu7JCtq_RfFsnl33RuGeBUXhk,2561
+water_column_sonar_processing/aws/s3fs_manager.py,sha256=oouzV9DZLplPC6vzbouWPzyfyNPABx_LGxRGJGc1vWg,2563
 water_column_sonar_processing/aws/sns_manager.py,sha256=Dp9avG5VSugSWPR1dZ-askuAw1fCZkNUHbOUP65iR-k,1867
 water_column_sonar_processing/aws/sqs_manager.py,sha256=NSUrWmnSC8h8Gf7gT0U8zFaQQ-yX89h0Q0mDLKGqp2Y,1597
 water_column_sonar_processing/cruise/__init__.py,sha256=H5hW0JMORuaFvQk_R31B4VL8RnRyKeanOOiWmqEMZJk,156
 water_column_sonar_processing/cruise/create_empty_zarr_store.py,sha256=ZsFQTDA0gXfQHlxDsXBGD1qQ0ipmx4kS81DcY6ml5Ew,7767
 water_column_sonar_processing/cruise/datatree_manager.py,sha256=Qy4dZCW8_q31lbjxbMsx3JtBS4BvQT17_2P0QD1RQcY,639
-water_column_sonar_processing/cruise/resample_regrid.py,sha256=986oZm6G8kngAmTg1GTTyxdx8Zy9fXAUGlK533GcrZ8,14413
+water_column_sonar_processing/cruise/resample_regrid.py,sha256=j_rFqR76ciY8Zd53_Me8XDUpPxart51nKY8PFyE8m9I,14612
 water_column_sonar_processing/geometry/__init__.py,sha256=GIzzc-_7pwEwbOkGpc4i_fmjWI5ymllXqzdHq_d3Rio,299
 water_column_sonar_processing/geometry/elevation_manager.py,sha256=eq9w691WJknPwWYkvO3giKTPleIxCVc2tMGR0e8ZRxQ,4267
 water_column_sonar_processing/geometry/geometry_manager.py,sha256=nz5T1vCDWHYIfQ853EqKYHDetTul7jRWS3y8Evep8QU,10855
@@ -18,17 +18,17 @@ water_column_sonar_processing/geometry/pmtile_generation.py,sha256=7Lm08Jr6YaM4n
 water_column_sonar_processing/index/__init__.py,sha256=izEObsKiOoIJ0kZCFhvaYsBd6Ga71XJxnogjrNInw68,68
 water_column_sonar_processing/index/index_manager.py,sha256=qsS6rKObJlFXKyzRuT1bk2_qW1YagW-Fg_AkQ1U_KRs,14213
 water_column_sonar_processing/model/__init__.py,sha256=FXaCdbPqxp0ogmZm9NplRirqpgMiYs1iRYgJbFbbX2Y,65
-water_column_sonar_processing/model/zarr_manager.py,sha256=UT3GSEQ6-_kd62dDZf9AOvueR8jEiUPbzoFcpiwy8IY,15505
+water_column_sonar_processing/model/zarr_manager.py,sha256=wAO_8jcKIEonTNH12Fzzdirz0XLS7qgwOJGrmcVALR8,15678
 water_column_sonar_processing/processing/__init__.py,sha256=tdpSfwnY6lbAS_yBTu4aG0SjPgCKqh6LAFvIj_t3j3U,168
 water_column_sonar_processing/processing/batch_downloader.py,sha256=qXoruHdbgzAolmroK6eRn9bWgeHFgaVQLwhJ6X5oHRE,6299
 water_column_sonar_processing/processing/raw_to_zarr.py,sha256=Sn0_zBT7yYP6abbSTlQBPA6iZSBxeVqPYYSgoroiBEU,17599
 water_column_sonar_processing/utility/__init__.py,sha256=yDObMOL0_OxKWet5wffK2-XVJgoE9iwiY2q04GZrtBQ,234
 water_column_sonar_processing/utility/cleaner.py,sha256=bNbs-hopWxtKAFBK0Eu18xdRErZCGZvtla3j-1bTwQw,619
-water_column_sonar_processing/utility/constants.py,sha256=nAThVjJGpFp8kMdv3I1g6I1hVvOBJBDbGCTEEYewB4o,2181
+water_column_sonar_processing/utility/constants.py,sha256=UtzFkvH5VE7eb8PzeKUDVt-nX6SOdlHtkul1zycF_Z0,2146
 water_column_sonar_processing/utility/pipeline_status.py,sha256=O-0SySqdRGJ6bs3zQe1NV9vkOpmsRM7zj5QoHgzYioY,4395
 water_column_sonar_processing/utility/timestamp.py,sha256=bO0oir7KxxoEHPGRkz9FCBfOligkocUyRiWRzAq8fnU,361
-water_column_sonar_processing-25.1.3.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
-water_column_sonar_processing-25.1.3.dist-info/METADATA,sha256=R0wRzg_ywppwzCzyY1MOyqRtFrPl6lyq1ltldgisbe4,5448
-water_column_sonar_processing-25.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-water_column_sonar_processing-25.1.3.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
-water_column_sonar_processing-25.1.3.dist-info/RECORD,,
+water_column_sonar_processing-25.1.5.dist-info/LICENSE,sha256=lz4IpJ5_adG3S0ali-WaIpQFVTnEAOucMDQPECUVEYw,1110
+water_column_sonar_processing-25.1.5.dist-info/METADATA,sha256=TAnHJJy4fCNPj3Dfj3-U8HW57ybmXSiPulIJ575dh-M,5959
+water_column_sonar_processing-25.1.5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+water_column_sonar_processing-25.1.5.dist-info/top_level.txt,sha256=aRYU4A7RNBlNrL4vzjytFAir3BNnmOgsvIGKKA36tg4,30
+water_column_sonar_processing-25.1.5.dist-info/RECORD,,