water-column-sonar-processing 25.3.2__py3-none-any.whl → 25.8.0__py3-none-any.whl

water_column_sonar_processing/geometry/geometry_manager.py

@@ -42,7 +42,7 @@ class GeometryManager:
         file_name_stem = Path(file_name).stem
         geo_json_name = f"{file_name_stem}.json"
 
-        print("Getting GPS data from echopype object.")
+        print("Getting GPS dataset from echopype object.")
         try:
             latitude = np.round(
                 echodata.platform.latitude.values, self.DECIMAL_PRECISION
@@ -56,7 +56,7 @@ class GeometryManager:
             #   note that nmea_times, unlike time1, can be sorted
             nmea_times = np.sort(echodata.platform.time1.values)
 
-            # 'time1' are times from the echosounder associated with the data of the transducer measurement
+            # 'time1' are times from the echosounder associated with the dataset of the transducer measurement
             time1 = echodata.environment.time1.values
 
             if len(nmea_times) < len(time1):
@@ -98,14 +98,14 @@ class GeometryManager:
 
             # create requirement for minimum linestring size
             MIN_ALLOWED_SIZE = (
-                4  # don't want to process files with less than 4 data points
+                4  # don't want to process files with less than 4 dataset points
             )
             if (
                 len(lat[~np.isnan(lat)]) < MIN_ALLOWED_SIZE
                 or len(lon[~np.isnan(lon)]) < MIN_ALLOWED_SIZE
             ):
                 raise Exception(
-                    f"There was not enough data in lat or lon to create geojson, {len(lat[~np.isnan(lat)])} found, less than {MIN_ALLOWED_SIZE}."
+                    f"There was not enough dataset in lat or lon to create geojson, {len(lat[~np.isnan(lat)])} found, less than {MIN_ALLOWED_SIZE}."
                 )
 
             # https://osoceanacoustics.github.io/echopype-examples/echopype_tour.html
@@ -124,7 +124,7 @@ class GeometryManager:
                 crs="epsg:4326",
             )
             # Note: We set np.nan to 0,0 so downstream missing values can be omitted
-            # TODO: so what ends up here is data with corruption at null island!!!
+            # TODO: so what ends up here is dataset with corruption at null island!!!
             geo_json_line = gps_gdf.to_json()
             if write_geojson:
                 print("Creating local copy of geojson file.")
@@ -180,12 +180,12 @@ class GeometryManager:
             #################################################################
             # GeoJSON FeatureCollection with IDs as "time"
         except Exception as err:
-            print(
-                f"Exception encountered extracting gps coordinates creating geojson: {err}"
+            raise RuntimeError(
+                f"Exception encountered extracting gps coordinates creating geojson, {err}"
             )
-            raise
+
         # Note: returned lat/lon values can include np.nan because they need to be aligned with
-        # the Sv data! GeoJSON needs simplification but has been filtered.
+        # the Sv dataset! GeoJSON needs simplification but has been filtered.
         # return gps_df.index.values, gps_df.latitude.values, gps_df.longitude.values
         return gps_df.index.values, lat, lon
         # TODO: if geojson is already returned with 0,0, the return here
@@ -229,9 +229,8 @@ class GeometryManager:
             indices = np.searchsorted(a=aa, v=vv)
 
             return indices, geospatial
-        except Exception as err:  # Failure
-            print(f"Exception encountered reading s3 GeoJSON: {err}")
-            raise
+        except Exception as err:
+            raise RuntimeError(f"Exception encountered reading s3 GeoJSON, {err}")
 
     ############################################################################
     # COMES from the raw-to-zarr conversion

water_column_sonar_processing/geometry/line_simplification.py

@@ -4,6 +4,10 @@ import numpy as np
 from pykalman import KalmanFilter
 from shapely.geometry import Point
 
+# import hvplot.pandas
+# from holoviews import opts
+# hv.extension('bokeh')
+
 # import matplotlib.pyplot as plt
 
 
@@ -24,6 +28,16 @@ def mph_to_knots(mph_value):
     return mph_value * 0.868976
 
 
+def mps_to_knots(mps_value):
+    return mps_value * 1.94384
+
+
+###############################################################################
+# Colab Notebook:
+# https://colab.research.google.com/drive/1Ihb1x0EeYRNwGJ4Bqi4RqQQHu9-40oDk?usp=sharing#scrollTo=hIPziqVO48Xg
+###############################################################################
+
+
 # https://shapely.readthedocs.io/en/stable/reference/shapely.MultiLineString.html#shapely.MultiLineString
 class LineSimplification:
     """
@@ -89,7 +103,7 @@ class LineSimplification:
     #######################################################
     def get_speeds(
         self,
-        times: np.ndarray,  # don't really need time, do need to segment the data first
+        times: np.ndarray,  # don't really need time, do need to segment the dataset first
         latitudes: np.ndarray,
         longitudes: np.ndarray,
     ) -> np.ndarray:
@@ -147,4 +161,15 @@ class LineSimplification:
     #######################################################
 
 
+# [(-72.2001724243164, 40.51750183105469), # latBB
+#  (-72.20023345947266, 40.51749038696289),
+#  (-72.20033264160156, 40.51750183105469), # lonAA, latBB
+#  (-72.20030212402344, 40.517391204833984),
+#  (-72.20033264160156, 40.517330169677734), # lonAA, latCC
+#  (-72.2003402709961, 40.51729965209961),
+#  (-72.20033264160156, 40.517330169677734), # lonAA, latCC
+#  (-72.20040130615234, 40.5172004699707),
+#  (-72.20050048828125, 40.51716995239258),
+#  (-72.2004623413086, 40.51710891723633)]
+
 ###########################################################

water_column_sonar_processing/geometry/pmtile_generation.py

@@ -1,10 +1,4 @@
-import glob
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-
 import fiona
-import geopandas
 import geopandas as gpd
 import numpy as np
 import pandas as pd
@@ -16,282 +10,252 @@ MAX_CONCURRENCY = 64
 MAX_WORKERS = 64
 GB = 1024**3
 
+bucket_name = "noaa-wcsd-zarr-pds"
+ship_name = "Henry_B._Bigelow"
+sensor_name = "EK60"
+
+# TODO: get pmtiles of all the evr points
+
 
 class PMTileGeneration(object):
     """
-    TODO: need to
-     - iterate through the zarr stores for all cruises
-     - generate geojson in geopandas df
-     - consolidate into singular df, one cruise per row
-     - export as _shape?_ file
-     - document next steps creating pmtiles with linux commands
-     - upload to s3
+    - iterate through the zarr stores for all cruises
+    - generate geojson in geopandas df, simplify linestrings
+    - consolidate into singular df, one cruise per row
+    - export as geojson
+    - using tippecanoe, geojson --> pmtiles w linux command
+    - upload to s3
     """
 
     #######################################################
     def __init__(
         self,
     ):
-        print("123")
+        self.bucket_name = "noaa-wcsd-zarr-pds"
+        self.ship_name = "Henry_B._Bigelow"
+        self.sensor_name = "EK60"
 
     #######################################################
-    # This uses a local collection of file-level geojson files to create the data
-    def generate_geojson_feature_collection(self):
-        # This was used to read from noaa-wcsd-model-pds bucket geojson files and then to
-        # generate the geopandas dataframe which could be exported to another comprehensive
-        # geojson file. That
-        result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
-        # result = result[:100]
-        jjj = 0
-        pieces = []
-        for jjj in range(len(result)):
-            file_name = os.path.normpath(result[jjj]).split(os.sep)[-1]
-            file_stem = os.path.splitext(os.path.basename(file_name))[0]
-            geom = gpd.read_file(result[jjj]).iloc[0]["geometry"]
-            # TDOO: Filter (0,0) coordinates
-            if len(geom.coords.xy[0]) < 2:
-                continue
-            geom = LineString(list(zip(geom.coords.xy[1], geom.coords.xy[0])))
-            pieces.append(
-                {
-                    "ship_name": os.path.normpath(result[jjj]).split(os.sep)[-4],
-                    "cruise_name": os.path.normpath(result[jjj]).split(os.sep)[-3],
-                    "file_stem": file_stem,
-                    "file_path": result[jjj],
-                    "geom": geom,
-                }
-            )
-        df = pd.DataFrame(pieces)
-        print(df)
-        gps_gdf = gpd.GeoDataFrame(
-            data=df[
-                ["ship_name", "cruise_name", "file_stem"]
-            ],  # try again with file_stem
-            geometry=df["geom"],
-            crs="EPSG:4326",
-        )
-        print(fiona.supported_drivers)
-        # gps_gdf.to_file('dataframe.shp', crs='epsg:4326')
-        # Convert geojson feature collection to pmtiles
-        gps_gdf.to_file("dataframe.geojson", driver="GeoJSON", crs="epsg:4326")
-        print("done")
-        """
-        # need to eliminate visits to null island
-        tippecanoe --no-feature-limit -zg --projection=EPSG:4326 -o dataframe.pmtiles -l cruises dataframe.geojson
-
-        https://docs.protomaps.com/pmtiles/create
-        PMTiles
-        https://drive.google.com/file/d/17Bi-UIXB9IJkIz30BHpiKHXYpCOgRFge/view?usp=sharing
-
-        Viewer
-        https://protomaps.github.io/PMTiles/#map=8.91/56.0234/-166.6346
-        """
+    def check_all_cruises(self, bucket_name, cruises):
+        completed = []
+        for cruise_name in cruises:
+            print(cruise_name)
+            try:
+                zarr_store = f"{cruise_name}.zarr"
+                s3_zarr_store_path = f"{bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{zarr_store}"
+                cruise = xr.open_dataset(
+                    filename_or_obj=f"s3://{s3_zarr_store_path}",
+                    engine="zarr",
+                    storage_options={"anon": True},
+                )
+                width = cruise.Sv.shape[1]
+                height = cruise.Sv.shape[0]
+                depth = cruise.Sv.shape[2]
+                print(
+                    f"height: {height}, width: {width}, depth: {depth} = {width * height * depth}"
+                )
+                lats = cruise.latitude.to_numpy()
+                percent_done = np.count_nonzero(~np.isnan(lats)) / width
+                if percent_done != 1.0:
+                    print(
+                        f"percent done: {np.round(percent_done, 2)}, {np.count_nonzero(~np.isnan(cruise.latitude.values))}, {width}"
+                    )
+                else:
+                    completed.append(cruise_name)
+            except Exception as err:
+                raise RuntimeError(f"Problem parsing Zarr stores, {err}")
+        return completed
 
     #######################################################
-    # TODO: temporary using this to get info
-    def get_info_from_zarr_store(
-        self,
-        ship_name,
-        cruise_names,
-    ):
-        # TODO: NOT USED ANYWHERE
-        total_size = 0
-        # s3_fs = s3fs.S3FileSystem(anon=True)
-        for cruise_name in cruise_names:
-            s3_path = f"s3://noaa-wcsd-zarr-pds/level_2/{ship_name}/{cruise_name}/EK60/{cruise_name}.zarr"
-            # zarr_store = s3fs.S3Map(root=s3_path, s3=s3_fs)
-            xr_store = xr.open_dataset(
-                filename_or_obj=s3_path,
+    def get_cruise_geometry(self, cruise_name, index):
+        print(cruise_name)
+        try:
+            pieces = []
+            zarr_store = f"{cruise_name}.zarr"
+            s3_zarr_store_path = f"{bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{zarr_store}"
+            cruise = xr.open_dataset(
+                filename_or_obj=f"s3://{s3_zarr_store_path}",
                 engine="zarr",
                 storage_options={"anon": True},
-                chunks={},  # this allows the engine to define the chunk scheme
+                chunks={},
                 cache=True,
             )
-            print(f"Cruise: {cruise_name}, shape: {xr_store.time.shape[0]}")
-            total_size = total_size + xr_store.time.shape[0]
-
-    def get_geospatial_info_from_zarr_store(
-        self,
-        ship_name,
-        cruise_name,
-    ):
-        """
-        Open Zarr store, create geometry, write to geojson, return name
-        """
-        # s3_fs = s3fs.S3FileSystem(anon=True)
-        gps_gdf = geopandas.GeoDataFrame(
-            columns=["id", "ship", "cruise", "sensor", "geometry"],
-            geometry="geometry",
-            crs="EPSG:4326",
-        )
-        s3_path = f"s3://noaa-wcsd-zarr-pds/level_2/{ship_name}/{cruise_name}/EK60/{cruise_name}.zarr"
-        # TODO: try-except to allow failures
-        print("opening store")
-        xr_store = xr.open_dataset(
-            filename_or_obj=s3_path,
-            engine="zarr",
-            storage_options={"anon": True},
-            chunks={},  # this allows the engine to define the chunk scheme
-            cache=True,
-        )
-        print(xr_store.Sv.shape)
-        # ---Read Zarr Store Time/Latitude/Longitude--- #
-        latitude = xr_store.latitude.values
-        longitude = xr_store.longitude.values
-        if np.isnan(latitude).any() or np.isnan(longitude).any():
-            print(f"there was missing lat-lon data for {cruise_name}")
-            return None
-        # ---Add To GeoPandas Dataframe--- #
-        # TODO: experiment with tolerance "0.001"
-        geom = LineString(list(zip(longitude, latitude))).simplify(
-            tolerance=0.001, preserve_topology=True
-        )
-        gps_gdf.loc[0] = (
-            0,
-            "Henry_B._Bigelow",
-            cruise_name,
-            "EK60",
-            geom,
-        )  # (ship, cruise, sensor, geometry)
-        gps_gdf.set_index("id", inplace=True)
-        gps_gdf.to_file(
-            f"dataframe_{cruise_name}.geojson", driver="GeoJSON"
-        )  # , engine="pyogrio")
-        return cruise_name
+            latitude_array = cruise.latitude.to_numpy()
+            longitude_array = cruise.longitude.to_numpy()
+            if np.isnan(latitude_array).any() or np.isnan(longitude_array).any():
+                raise RuntimeError(
+                    f"There was missing lat-lon dataset for, {cruise_name}"
+                )
+            geom = LineString(list(zip(longitude_array, latitude_array))).simplify(
+                tolerance=0.001,  # preserve_topology=True # 113
+            )  # TODO: do speed check, convert linestrings to multilinestrings
+            print(len(geom.coords))
+            pieces.append(
+                {
+                    "id": index,
+                    "ship_name": ship_name,
+                    "cruise_name": cruise_name,
+                    "sensor_name": sensor_name,
+                    "geom": geom,
+                }
+            )
+            df = pd.DataFrame(pieces)
+            gps_gdf = gpd.GeoDataFrame(
+                data=df[["id", "ship_name", "cruise_name", "sensor_name"]],
+                geometry=df["geom"],
+                crs="EPSG:4326",
+            )
+            print(gps_gdf)
+            # {'DXF': 'rw', 'CSV': 'raw', 'OpenFileGDB': 'raw', 'ESRIJSON': 'r', 'ESRI Shapefile': 'raw', 'FlatGeobuf': 'raw', 'GeoJSON': 'raw', 'GeoJSONSeq': 'raw', 'GPKG': 'raw', 'GML': 'rw', 'OGR_GMT': 'rw', 'GPX': 'rw', 'MapInfo File': 'raw', 'DGN': 'raw', 'S57': 'r', 'SQLite': 'raw', 'TopoJSON': 'r'}
+            if "GeoJSON" not in fiona.supported_drivers.keys():
+                raise RuntimeError("Missing GeoJSON driver")
 
-    #######################################################
-    def open_zarr_stores_with_thread_pool_executor(
-        self,
-        cruises: list,
-    ):
-        # 'cruises' is a list of cruises to process
-        completed_cruises = []
-        try:
-            with ThreadPoolExecutor(max_workers=32) as executor:
-                futures = [
-                    executor.submit(
-                        self.get_geospatial_info_from_zarr_store,
-                        "Henry_B._Bigelow",  # ship_name
-                        cruise,  # cruise_name
-                    )
-                    for cruise in cruises
-                ]
-                for future in as_completed(futures):
-                    result = future.result()
-                    if result:
-                        completed_cruises.extend([result])
+            gps_gdf.set_index("id", inplace=True)
+            # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON") #, crs="epsg:4326")
+            return gps_gdf
         except Exception as err:
-            print(err)
-        print("Done opening zarr stores using thread pool.")
-        return completed_cruises  # Took ~12 minutes
+            raise RuntimeError(f"Problem parsing Zarr stores, {err}")
 
     #######################################################
-    # https://docs.protomaps.com/pmtiles/create
-    def aggregate_geojson_into_dataframe(self):
-        """
-        iterate through cruises, threadpoolexecute geojson creation, aggregate geojson files into df,
-        """
-        gps_gdf = geopandas.GeoDataFrame(
+    def aggregate_geojson_into_dataframe(self, geoms):
+        gps_gdf = gpd.GeoDataFrame(
             columns=["id", "ship", "cruise", "sensor", "geometry"],
             geometry="geometry",
             crs="EPSG:4326",
         )
-
-        file_type = "dataframe_*.geojson"
-        geojson_files = glob.glob(file_type)
-        for jjj in range(len(geojson_files)):
-            print(jjj)
-            geom = geopandas.read_file(geojson_files[jjj])
-            gps_gdf.loc[jjj] = (
-                jjj,
-                geom.ship[0],
-                geom.cruise[0],
-                geom.sensor[0],
-                geom.geometry[0],
+        for iii, geom in enumerate(geoms):
+            gps_gdf.loc[iii] = (
+                iii,
+                geom.ship_name[iii],
+                geom.cruise_name[iii],
+                geom.sensor_name[iii],
+                geom.geometry[iii],
             )
-            # gps_gdf.loc[0] = (0, "Henry_B._Bigelow", cruise_name, "EK60", geom)  # (ship, cruise, sensor, geometry)
-        print(gps_gdf)
         gps_gdf.set_index("id", inplace=True)
         gps_gdf.to_file(
-            "data.geojson",
+            filename="dataset.geojson",
             driver="GeoJSON",
-            engine="pyogrio",
+            engine="fiona",  # or "pyogrio"
             layer_options={"ID_GENERATE": "YES"},
+            crs="EPSG:4326",
+            id_generate=True,  # required for the feature click selection
         )
-        return list(gps_gdf.cruise)
-
-        # gps_gdf.loc[iii] = (iii, "Henry_B._Bigelow", cruise_name, "EK60", geom)  # (ship, cruise, sensor, geometry)
-        # print('writing to file')
-        # print(gps_gdf)
-        # gps_gdf.set_index('id', inplace=True)
-        # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON", engine="pyogrio", layer_options={"ID_GENERATE": "YES"})
-        # https://gdal.org/en/latest/drivers/vector/jsonfg.html
-        # gps_gdf.to_file(
-        #     f"data.geojson",
-        #     driver="GeoJSON",
-        #     engine="pyogrio",
-        #     layer_options={"ID_FIELD": "id"}
-        # )
-        # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON", engine="pyogrio", id_generate=True)
-
+        print(gps_gdf)
 
-# print(fiona.supported_drivers) # {'DXF': 'rw', 'CSV': 'raw', 'OpenFileGDB': 'raw', 'ESRIJSON': 'r', 'ESRI Shapefile': 'raw', 'FlatGeobuf': 'raw', 'GeoJSON': 'raw', 'GeoJSONSeq': 'raw', 'GPKG': 'raw', 'GML': 'rw', 'OGR_GMT': 'rw', 'GPX': 'rw', 'MapInfo File': 'raw', 'DGN': 'raw', 'S57': 'r', 'SQLite': 'raw', 'TopoJSON': 'r'}
-# gps_gdf.to_file('dataframe.shp', crs="EPSG:4326", engine="fiona")
-# Convert geojson feature collection to pmtiles
-# gps_gdf.to_file("dataframe.geojson", driver="GeoJSON", crs="EPSG:4326", engine="fiona")
-# print("done")
-# ---Export Shapefile--- #
+    #######################################################
+    def create_collection_geojson(self):
+        cruises = [
+            "HB0706",
+            "HB0707",
+            "HB0710",
+            "HB0711",
+            "HB0802",
+            "HB0803",
+            "HB0805",
+            "HB0806",
+            "HB0807",
+            "HB0901",
+            "HB0902",
+            "HB0903",
+            "HB0904",
+            "HB0905",
+            "HB1002",
+            "HB1006",
+            "HB1102",
+            "HB1103",
+            "HB1105",
+            "HB1201",
+            "HB1206",
+            "HB1301",
+            "HB1303",
+            "HB1304",
+            "HB1401",
+            "HB1402",
+            "HB1403",
+            "HB1405",
+            "HB1501",
+            "HB1502",
+            "HB1503",
+            "HB1506",
+            "HB1507",
+            "HB1601",
+            "HB1603",
+            "HB1604",
+            "HB1701",
+            "HB1702",
+            "HB1801",
+            "HB1802",
+            "HB1803",
+            "HB1804",
+            "HB1805",
+            "HB1806",
+            "HB1901",
+            "HB1902",
+            "HB1903",
+            "HB1904",
+            "HB1906",
+            "HB1907",
+            "HB2001",
+            "HB2006",
+            "HB2007",
+            "HB20ORT",
+            "HB20TR",
+        ]
+        completed_cruises = self.check_all_cruises(
+            bucket_name=bucket_name, cruises=cruises
+        )  # TODO: threadpool this
+        ### create linestring ###
+        geometries = []
+        for jjj, completed_cruise in enumerate(
+            completed_cruises
+        ):  # TODO: threadpool this
+            geometries.append(
+                self.get_cruise_geometry(cruise_name=completed_cruise, index=jjj)
+            )
+        #
+        self.aggregate_geojson_into_dataframe(geoms=geometries)
+        #
+        print(
+            'Now run this: "tippecanoe --no-feature-limit -zg -o dataset.pmtiles -l cruises dataset.geojson --force"'
+        )
+        # # water-column-sonar-id.pmtiles
+        # linux command: "tippecanoe --no-feature-limit -zg -o water-column-sonar-id.pmtiles -l cruises dataset.geojson --force"
+        #   note: 'cruises' is the name of the layer
+        #   size is ~3.3 MB for the pmtiles
+        # then drag-and-drop here: https://pmtiles.io/#map=6.79/39.802/-71.51
 
+    #######################################################
+    # TODO: copy the .pmtiles file to the s3 bucket "noaa-wcsd-pds-index"
+    #######################################################
 
-# gps_gdf.set_geometry(col='geometry', inplace=True)
-# gps_gdf.__geo_interface__
-# gps_gdf.set_index('id', inplace=True)
-# gps_gdf.to_file(f"dataframe3.geojson", driver="GeoJSON", crs="EPSG:4326", engine="fiona", index=True)
+    #######################################################
+    # TODO: get threadpool working
+    # def open_zarr_stores_with_thread_pool_executor(
+    #     self,
+    #     cruises: list,
+    # ):
+    #     # 'cruises' is a list of cruises to process
+    #     completed_cruises = []
+    #     try:
+    #         with ThreadPoolExecutor(max_workers=32) as executor:
+    #             futures = [
+    #                 executor.submit(
+    #                     self.get_geospatial_info_from_zarr_store,
+    #                     "Henry_B._Bigelow",  # ship_name
+    #                     cruise,  # cruise_name
+    #                 )
+    #                 for cruise in cruises
+    #             ]
+    #             for future in as_completed(futures):
+    #                 result = future.result()
+    #                 if result:
+    #                     completed_cruises.extend([result])
+    #     except Exception as err:
+    #         raise RuntimeError(f"Problem, {err}")
+    #     print("Done opening zarr stores using thread pool.")
+    #     return completed_cruises  # Took ~12 minutes
 
-### this gives the right layer id values
-# gps_gdf.to_file(f"dataframe6.geojson", driver="GeoJSON", engine="pyogrio", layer_options={"ID_GENERATE": "YES"})
-# jq '{"type": "FeatureCollection", "features": [.[] | .features[]]}' --slurp input*.geojson > output.geojson
-# tippecanoe -zg --projection=EPSG:4326 -o water-column-sonar-id.pmtiles -l cruises output.geojson
-# tippecanoe -zg --convert-stringified-ids-to-numbers --projection=EPSG:4326 -o water-column-sonar-id.pmtiles -l cruises dataframe*.geojson
-# {
-# "type": "FeatureCollection",
-# "name": "dataframe5",
-# "features": [
-# { "type": "Feature", "id": 0, "properties": { "id": 0, "ship": "Henry_B._Bigelow", "cruise": "HB0706", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -72.120498657226562, 39.659671783447266 ], [ -72.120773315429688, 39.660198211669922 ] ] } },
-# { "type": "Feature", "id": 1, "properties": { "id": 1, "ship": "Henry_B._Bigelow", "cruise": "HB0707", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -71.797836303710938, 41.003166198730469 ], [ -71.797996520996094, 41.002998352050781 ], [ -71.798583984375, 41.002994537353516 ] ] } },
-# { "type": "Feature", "id": 2, "properties": { "id": 2, "ship": "Henry_B._Bigelow", "cruise": "HB0710", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -72.489486694335938, 40.331901550292969 ], [ -72.490760803222656, 40.33099365234375 ] ] } }
-# ]
-# }
+    #######################################################
 
-# # https://docs.protomaps.com/pmtiles/create
-# #ogr2ogr -t_srs EPSG:4326 data.geojson dataframe.shp
-# # Only need to do the second one here...
-# tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises dataframe.geojson
-# tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises --coalesce-densest-as-needed --extend-zooms-if-still-dropping dataframe*.geojson
-# # used this to combine all the geojson files into single pmtile file (2024-12-03):
-# tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises --coalesce-densest-as-needed --extend-zooms-if-still-dropping dataframe*.geojson
-#
-# TODO:
-#     run each one of the cruises in a separate ospool workflow.
-#     each process gets own store
 
 ###########################################################
-
-# s3_manager = S3Manager()  # endpoint_url=endpoint_url)
-# # s3fs_manager = S3FSManager()
-# # input_bucket_name = "test_input_bucket"
-# # s3_manager.create_bucket(bucket_name=input_bucket_name)
-# ship_name = "Henry_B._Bigelow"
-# cruise_name = "HB0706"
-# sensor_name = "EK60"
-#
-# # ---Scan Bucket For All Zarr Stores--- #
-# # https://noaa-wcsd-zarr-pds.s3.amazonaws.com/index.html#level_2/Henry_B._Bigelow/HB0706/EK60/HB0706.zarr/
-# path_to_zarr_store = f"s3://noaa-wcsd-zarr-pds/level_2/Henry_B._Bigelow/HB0706/EK60/HB0706.zarr"
-# s3 = s3fs.S3FileSystem()
-# zarr_store = s3fs.S3Map(path_to_zarr_store, s3=s3)
-# ds_zarr = xr.open_zarr(zarr_store, consolidated=None)
-# print(ds_zarr.Sv.shape)
-
-
-# total = [246847, 89911, 169763, 658047, 887640, 708771, 187099, 3672813, 4095002, 763268, 162727, 189454, 1925270, 3575857, 1031920, 1167590, 3737415, 4099957, 3990725, 3619996, 3573052, 2973090, 55851, 143192, 1550164, 3692819, 668400, 489735, 393260, 1311234, 242989, 4515760, 1303091, 704663, 270645, 3886437, 4204381, 1062090, 428639, 541455, 4206506, 298561, 1279329, 137416, 139836, 228947, 517949]