water-column-sonar-processing 0.0.9__py3-none-any.whl → 26.1.9__py3-none-any.whl

water_column_sonar_processing/geometry/pmtile_generation.py

@@ -1,15 +1,8 @@
-import glob
-import os
-from pathlib import Path
 import fiona
-import s3fs
+import geopandas as gpd
 import numpy as np
 import pandas as pd
 import xarray as xr
-import geopandas
-import geopandas as gpd
-import pyogrio
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from shapely.geometry import LineString
 
 MAX_POOL_CONNECTIONS = 64
@@ -17,246 +10,257 @@ MAX_CONCURRENCY = 64
 MAX_WORKERS = 64
 GB = 1024**3
 
+bucket_name = "noaa-wcsd-zarr-pds"
+ship_name = "Henry_B._Bigelow"
+sensor_name = "EK60"
+
+# TODO: get pmtiles of all the evr points
+
 
 class PMTileGeneration(object):
+    """
+    - iterate through the zarr stores for all cruises
+    - generate geojson in geopandas df, simplify linestrings
+    - consolidate into singular df, one cruise per row
+    - export as geojson
+    - using tippecanoe, geojson --> pmtiles w linux command
+    - upload to s3
+    """
+
     #######################################################
     def __init__(
         self,
     ):
-        print("123")
+        self.bucket_name = "noaa-wcsd-zarr-pds"
+        self.ship_name = "Henry_B._Bigelow"
+        self.sensor_name = "EK60"
 
     #######################################################
-    # This uses a local collection of file-level geojson files to create the data
-    def generate_geojson_feature_collection(self):
-        # This was used to read from noaa-wcsd-model-pds bucket geojson files and then to
-        # generate the geopandas dataframe which could be exported to another comprehensive
-        # geojson file. That
-        result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
-        # result = result[:100]
-        jjj = 0
-        pieces = []
-        for jjj in range(len(result)):
-            file_name = os.path.normpath(result[jjj]).split(os.sep)[-1]
-            file_stem = os.path.splitext(os.path.basename(file_name))[0]
-            geom = gpd.read_file(result[jjj]).iloc[0]["geometry"]
-            # TDOO: Filter (0,0) coordinates
-            if len(geom.coords.xy[0]) < 2:
-                continue
-            geom = LineString(list(zip(geom.coords.xy[1], geom.coords.xy[0])))
+    @staticmethod
+    def check_all_cruises(bucket_name, cruises):
+        completed = []
+        for cruise_name in cruises:
+            print(cruise_name)
+            try:
+                zarr_store = f"{cruise_name}.zarr"
+                s3_zarr_store_path = f"{bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{zarr_store}"
+                kwargs = {"consolidated": False}
+                cruise = xr.open_dataset(
+                    filename_or_obj=f"s3://{s3_zarr_store_path}",
+                    engine="zarr",
+                    storage_options={"anon": True},
+                    **kwargs,
+                )
+                width = cruise.Sv.shape[1]
+                height = cruise.Sv.shape[0]
+                depth = cruise.Sv.shape[2]
+                print(
+                    f"height: {height}, width: {width}, depth: {depth} = {width * height * depth}"
+                )
+                lats = cruise.latitude.to_numpy()
+                percent_done = np.count_nonzero(~np.isnan(lats)) / width
+                if percent_done != 1.0:
+                    print(
+                        f"percent done: {np.round(percent_done, 2)}, {np.count_nonzero(~np.isnan(cruise.latitude.values))}, {width}"
+                    )
+                else:
+                    completed.append(cruise_name)
+            except Exception as err:
+                raise RuntimeError(f"Problem parsing Zarr stores, {err}")
+        return completed
+
+    #######################################################
+    @staticmethod
+    def get_cruise_geometry(cruise_name, index):
+        print(cruise_name)
+        try:
+            pieces = []
+            zarr_store = f"{cruise_name}.zarr"
+            s3_zarr_store_path = f"{bucket_name}/level_2/{ship_name}/{cruise_name}/{sensor_name}/{zarr_store}"
+            cruise = xr.open_dataset(
+                filename_or_obj=f"s3://{s3_zarr_store_path}",
+                engine="zarr",
+                storage_options={"anon": True},
+                chunks={},
+                cache=True,
+            )
+            latitude_array = cruise.latitude.to_numpy()
+            longitude_array = cruise.longitude.to_numpy()
+            if np.isnan(latitude_array).any() or np.isnan(longitude_array).any():
+                raise RuntimeError(
+                    f"There was missing lat-lon dataset for, {cruise_name}"
+                )
+            geom = LineString(list(zip(longitude_array, latitude_array))).simplify(
+                tolerance=0.001,  # preserve_topology=True # 113
+            )  # TODO: do speed check, convert linestrings to multilinestrings
+            print(len(geom.coords))
             pieces.append(
                 {
-                    "ship_name": os.path.normpath(result[jjj]).split(os.sep)[-4],
-                    "cruise_name": os.path.normpath(result[jjj]).split(os.sep)[-3],
-                    "file_stem": file_stem,
-                    "file_path": result[jjj],
+                    "id": index,
+                    "ship_name": ship_name,
+                    "cruise_name": cruise_name,
+                    "sensor_name": sensor_name,
                     "geom": geom,
                 }
             )
-        df = pd.DataFrame(pieces)
-        print(df)
-        gps_gdf = gpd.GeoDataFrame(
-            data=df[
-                ["ship_name", "cruise_name", "file_stem"]
-            ],  # try again with file_stem
-            geometry=df["geom"],
-            crs="EPSG:4326",
-        )
-        print(fiona.supported_drivers)
-        # gps_gdf.to_file('dataframe.shp', crs='epsg:4326')
-        # Convert geojson feature collection to pmtiles
-        gps_gdf.to_file("dataframe.geojson", driver="GeoJSON", crs="epsg:4326")
-        print("done")
-        """
-        # need to eliminate visits to null island
-        tippecanoe --no-feature-limit -zg --projection=EPSG:4326 -o dataframe.pmtiles -l cruises dataframe.geojson
-
-        https://docs.protomaps.com/pmtiles/create
-        PMTiles
-        https://drive.google.com/file/d/17Bi-UIXB9IJkIz30BHpiKHXYpCOgRFge/view?usp=sharing
-
-        Viewer
-        https://protomaps.github.io/PMTiles/#map=8.91/56.0234/-166.6346
-        """
-
-    #######################################################
-    # TODO: temporary using this to get info
-    def get_info_from_zarr_store(
-        self,
-        ship_name,
-        cruise_names,
-    ):
-        total_size = 0
-        s3_fs = s3fs.S3FileSystem(anon=True)
-        for cruise_name in cruise_names:
-            path_to_zarr_store = f"s3://noaa-wcsd-zarr-pds/level_2/{ship_name}/{cruise_name}/EK60/{cruise_name}.zarr"
-            zarr_store = s3fs.S3Map(root=path_to_zarr_store, s3=s3_fs)
-            xr_store = xr.open_zarr(store=zarr_store, consolidated=None)
-            print(f'Cruise: {cruise_name}, shape: {xr_store.time.shape[0]}')
-            total_size = total_size + xr_store.time.shape[0]
-
-    def get_geospatial_info_from_zarr_store(
-        self,
-        ship_name,
-        cruise_name,
-    ):
-        """
-        Open Zarr store, create geometry, write to geojson, return name
-        """
-        s3_fs = s3fs.S3FileSystem(anon=True)
-        gps_gdf = geopandas.GeoDataFrame(
-            columns=["id", "ship", "cruise", "sensor", "geometry"],
-            geometry="geometry",
-            crs="EPSG:4326"
-        )
-        path_to_zarr_store = f"s3://noaa-wcsd-zarr-pds/level_2/{ship_name}/{cruise_name}/EK60/{cruise_name}.zarr"
-        # file_name = os.path.normpath(path_to_zarr_store).split(os.sep)[-1]
-        # file_stem = os.path.splitext(os.path.basename(file_name))[0]
-        zarr_store = s3fs.S3Map(root=path_to_zarr_store, s3=s3_fs)
-        # ---Open Zarr Store--- #
-        # TODO: try-except to allow failures
-        print('opening store')
-        # xr_store = xr.open_zarr(store=zarr_store, consolidated=False)
-        xr_store = xr.open_zarr(store=zarr_store, consolidated=None)
-        print(xr_store.Sv.shape)
-        # ---Read Zarr Store Time/Latitude/Longitude--- #
-        latitude = xr_store.latitude.values
-        longitude = xr_store.longitude.values
-        if np.isnan(latitude).any() or np.isnan(longitude).any():
-            print(f'there was missing lat-lon data for {cruise_name}')
-            return None
-        # ---Add To GeoPandas Dataframe--- #
-        # TODO: experiment with tolerance "0.001"
-        geom = LineString(list(zip(longitude, latitude))).simplify(tolerance=0.001, preserve_topology=True)
-        gps_gdf.loc[0] = (0, "Henry_B._Bigelow", cruise_name, "EK60", geom)  # (ship, cruise, sensor, geometry)
-        gps_gdf.set_index('id', inplace=True)
-        gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON") #, engine="pyogrio")
-        return cruise_name
+            df = pd.DataFrame(pieces)
+            gps_gdf = gpd.GeoDataFrame(
+                data=df[["id", "ship_name", "cruise_name", "sensor_name"]],
+                geometry=df["geom"],
+                crs="EPSG:4326",
+            )
+            print(gps_gdf)
+            # {'DXF': 'rw', 'CSV': 'raw', 'OpenFileGDB': 'raw', 'ESRIJSON': 'r', 'ESRI Shapefile': 'raw', 'FlatGeobuf': 'raw', 'GeoJSON': 'raw', 'GeoJSONSeq': 'raw', 'GPKG': 'raw', 'GML': 'rw', 'OGR_GMT': 'rw', 'GPX': 'rw', 'MapInfo File': 'raw', 'DGN': 'raw', 'S57': 'r', 'SQLite': 'raw', 'TopoJSON': 'r'}
+            if "GeoJSON" not in fiona.supported_drivers.keys():
+                raise RuntimeError("Missing GeoJSON driver")
 
-    #######################################################
-    def open_zarr_stores_with_thread_pool_executor(
-            self,
-            cruises: list,
-    ):
-        # 'cruises' is a list of cruises to process
-        completed_cruises = []
-        try:
-            with ThreadPoolExecutor(max_workers=32) as executor:
-                futures = [
-                    executor.submit(
-                        self.get_geospatial_info_from_zarr_store,
-                        "Henry_B._Bigelow",  # ship_name
-                        cruise,  # cruise_name
-                    )
-                    for cruise in cruises
-                ]
-                for future in as_completed(futures):
-                    result = future.result()
-                    if result:
-                        completed_cruises.extend([result])
+            gps_gdf.set_index("id", inplace=True)
+            # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON") #, crs="epsg:4326")
+            return gps_gdf
         except Exception as err:
-            print(err)
-        print("Done opening zarr stores using thread pool.")
-        return completed_cruises # Took ~12 minutes
+            raise RuntimeError(f"Problem parsing Zarr stores, {err}")
 
     #######################################################
-    # https://docs.protomaps.com/pmtiles/create
-    def aggregate_geojson_into_dataframe(
-        self
-    ):
-        """
-        iterate through cruises, threadpoolexecute geojson creation, aggregate geojson files into df,
-        """
-        gps_gdf = geopandas.GeoDataFrame(
+    @staticmethod
+    def aggregate_geojson_into_dataframe(geoms):
+        gps_gdf = gpd.GeoDataFrame(
             columns=["id", "ship", "cruise", "sensor", "geometry"],
             geometry="geometry",
-            crs="EPSG:4326"
+            crs="EPSG:4326",
+        )
+        for iii, geom in enumerate(geoms):
+            gps_gdf.loc[iii] = (
+                iii,
+                geom.ship_name[iii],
+                geom.cruise_name[iii],
+                geom.sensor_name[iii],
+                geom.geometry[iii],
+            )
+        gps_gdf.set_index("id", inplace=True)
+        gps_gdf.to_file(
+            filename="dataset.geojson",
+            driver="GeoJSON",
+            engine="fiona",  # or "pyogrio"
+            layer_options={"ID_GENERATE": "YES"},
+            crs="EPSG:4326",
+            id_generate=True,  # required for the feature click selection
         )
-
-        file_type = 'dataframe_*.geojson'
-        geojson_files = glob.glob(file_type)
-        for jjj in range(len(geojson_files)):
-            print(jjj)
-            geom = geopandas.read_file(geojson_files[jjj])
-            gps_gdf.loc[jjj] = (jjj, geom.ship[0], geom.cruise[0], geom.sensor[0], geom.geometry[0])
-            #gps_gdf.loc[0] = (0, "Henry_B._Bigelow", cruise_name, "EK60", geom)  # (ship, cruise, sensor, geometry)
         print(gps_gdf)
-        gps_gdf.set_index('id', inplace=True)
-        gps_gdf.to_file(f"data.geojson", driver="GeoJSON", engine="pyogrio", layer_options={"ID_GENERATE": "YES"})
-        return list(gps_gdf.cruise)
 
-        # gps_gdf.loc[iii] = (iii, "Henry_B._Bigelow", cruise_name, "EK60", geom)  # (ship, cruise, sensor, geometry)
-        #print('writing to file')
-        #print(gps_gdf)
-        # gps_gdf.set_index('id', inplace=True)
-        # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON", engine="pyogrio", layer_options={"ID_GENERATE": "YES"})
-        # https://gdal.org/en/latest/drivers/vector/jsonfg.html
-        # gps_gdf.to_file(
-        #     f"data.geojson",
-        #     driver="GeoJSON",
-        #     engine="pyogrio",
-        #     layer_options={"ID_FIELD": "id"}
-        # )
-        # gps_gdf.to_file(f"dataframe_{cruise_name}.geojson", driver="GeoJSON", engine="pyogrio", id_generate=True)
-
-# print(fiona.supported_drivers) # {'DXF': 'rw', 'CSV': 'raw', 'OpenFileGDB': 'raw', 'ESRIJSON': 'r', 'ESRI Shapefile': 'raw', 'FlatGeobuf': 'raw', 'GeoJSON': 'raw', 'GeoJSONSeq': 'raw', 'GPKG': 'raw', 'GML': 'rw', 'OGR_GMT': 'rw', 'GPX': 'rw', 'MapInfo File': 'raw', 'DGN': 'raw', 'S57': 'r', 'SQLite': 'raw', 'TopoJSON': 'r'}
-#gps_gdf.to_file('dataframe.shp', crs="EPSG:4326", engine="fiona")
-# Convert geojson feature collection to pmtiles
-#gps_gdf.to_file("dataframe.geojson", driver="GeoJSON", crs="EPSG:4326", engine="fiona")
-#print("done")
-# ---Export Shapefile--- #
+    #######################################################
+    def create_collection_geojson(self):
+        cruises = [
+            "HB0706",
+            "HB0707",
+            "HB0710",
+            "HB0711",
+            "HB0802",
+            "HB0803",
+            "HB0805",
+            "HB0806",
+            "HB0807",
+            "HB0901",
+            "HB0902",
+            "HB0903",
+            "HB0904",
+            "HB0905",
+            "HB1002",
+            "HB1006",
+            "HB1102",
+            "HB1103",
+            "HB1105",
+            "HB1201",
+            "HB1206",
+            "HB1301",
+            "HB1303",
+            "HB1304",
+            "HB1401",
+            "HB1402",
+            "HB1403",
+            "HB1405",
+            "HB1501",
+            "HB1502",
+            "HB1503",
+            "HB1506",
+            "HB1507",
+            "HB1601",
+            "HB1603",
+            "HB1604",
+            "HB1701",
+            "HB1702",
+            "HB1801",
+            "HB1802",
+            "HB1803",
+            "HB1804",
+            "HB1805",
+            "HB1806",
+            "HB1901",
+            "HB1902",
+            "HB1903",
+            "HB1904",
+            "HB1906",
+            "HB1907",
+            "HB2001",
+            "HB2006",
+            "HB2007",
+            "HB20ORT",
+            "HB20TR",
+        ]
+        completed_cruises = self.check_all_cruises(
+            bucket_name=bucket_name, cruises=cruises
+        )  # TODO: threadpool this
+        ### create linestring ###
+        geometries = []
+        for jjj, completed_cruise in enumerate(
+            completed_cruises
+        ):  # TODO: threadpool this
+            geometries.append(
+                self.get_cruise_geometry(cruise_name=completed_cruise, index=jjj)
+            )
+        #
+        self.aggregate_geojson_into_dataframe(geoms=geometries)
+        #
+        print(
+            'Now run this: "tippecanoe --no-feature-limit -zg -o dataset.pmtiles -l cruises dataset.geojson --force"'
+        )
+        # # water-column-sonar-id.pmtiles
+        # linux command: "tippecanoe --no-feature-limit -zg -o water-column-sonar-id.pmtiles -l cruises dataset.geojson --force"
+        #   note: 'cruises' is the name of the layer
+        #   size is ~3.3 MB for the pmtiles
+        # then drag-and-drop here: https://pmtiles.io/#map=6.79/39.802/-71.51
 
+    #######################################################
+    # TODO: copy the .pmtiles file to the s3 bucket "noaa-wcsd-pds-index"
+    #######################################################
 
+    #######################################################
+    # TODO: get threadpool working
+    # def open_zarr_stores_with_thread_pool_executor(
+    #     self,
+    #     cruises: list,
+    # ):
+    #     # 'cruises' is a list of cruises to process
+    #     completed_cruises = []
+    #     try:
+    #         with ThreadPoolExecutor(max_workers=32) as executor:
+    #             futures = [
+    #                 executor.submit(
+    #                     self.get_geospatial_info_from_zarr_store,
+    #                     "Henry_B._Bigelow",  # ship_name
+    #                     cruise,  # cruise_name
+    #                 )
+    #                 for cruise in cruises
+    #             ]
+    #             for future in as_completed(futures):
+    #                 result = future.result()
+    #                 if result:
+    #                     completed_cruises.extend([result])
+    #     except Exception as err:
+    #         raise RuntimeError(f"Problem, {err}")
+    #     print("Done opening zarr stores using thread pool.")
+    #     return completed_cruises  # Took ~12 minutes
 
-#gps_gdf.set_geometry(col='geometry', inplace=True)
-#gps_gdf.__geo_interface__
-#gps_gdf.set_index('id', inplace=True)
-#gps_gdf.to_file(f"dataframe3.geojson", driver="GeoJSON", crs="EPSG:4326", engine="fiona", index=True)
+    #######################################################
 
-### this gives the right layer id values
-#gps_gdf.to_file(f"dataframe6.geojson", driver="GeoJSON", engine="pyogrio", layer_options={"ID_GENERATE": "YES"})
-# jq '{"type": "FeatureCollection", "features": [.[] | .features[]]}' --slurp input*.geojson > output.geojson
-#tippecanoe -zg --projection=EPSG:4326 -o water-column-sonar-id.pmtiles -l cruises output.geojson
-#tippecanoe -zg --convert-stringified-ids-to-numbers --projection=EPSG:4326 -o water-column-sonar-id.pmtiles -l cruises dataframe*.geojson
-# {
-# "type": "FeatureCollection",
-# "name": "dataframe5",
-# "features": [
-# { "type": "Feature", "id": 0, "properties": { "id": 0, "ship": "Henry_B._Bigelow", "cruise": "HB0706", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -72.120498657226562, 39.659671783447266 ], [ -72.120773315429688, 39.660198211669922 ] ] } },
-# { "type": "Feature", "id": 1, "properties": { "id": 1, "ship": "Henry_B._Bigelow", "cruise": "HB0707", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -71.797836303710938, 41.003166198730469 ], [ -71.797996520996094, 41.002998352050781 ], [ -71.798583984375, 41.002994537353516 ] ] } },
-# { "type": "Feature", "id": 2, "properties": { "id": 2, "ship": "Henry_B._Bigelow", "cruise": "HB0710", "sensor": "EK60" }, "geometry": { "type": "LineString", "coordinates": [ [ -72.489486694335938, 40.331901550292969 ], [ -72.490760803222656, 40.33099365234375 ] ] } }
-# ]
-# }
-"""
-# https://docs.protomaps.com/pmtiles/create
-#ogr2ogr -t_srs EPSG:4326 data.geojson dataframe.shp
-# Only need to do the second one here...
-tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises dataframe.geojson
-tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises --coalesce-densest-as-needed --extend-zooms-if-still-dropping dataframe*.geojson
-# used this to combine all the geojson files into single pmtile file (2024-12-03):
-tippecanoe -zg --projection=EPSG:4326 -o data.pmtiles -l cruises --coalesce-densest-as-needed --extend-zooms-if-still-dropping dataframe*.geojson
 
-TODO:
-    run each one of the cruises in a separate ospool workflow.
-    each process gets own store    
-"""
 ###########################################################
-
-# s3_manager = S3Manager()  # endpoint_url=endpoint_url)
-# # s3fs_manager = S3FSManager()
-# # input_bucket_name = "test_input_bucket"
-# # s3_manager.create_bucket(bucket_name=input_bucket_name)
-# ship_name = "Henry_B._Bigelow"
-# cruise_name = "HB0706"
-# sensor_name = "EK60"
-#
-# # ---Scan Bucket For All Zarr Stores--- #
-# # https://noaa-wcsd-zarr-pds.s3.amazonaws.com/index.html#level_2/Henry_B._Bigelow/HB0706/EK60/HB0706.zarr/
-# path_to_zarr_store = f"s3://noaa-wcsd-zarr-pds/level_2/Henry_B._Bigelow/HB0706/EK60/HB0706.zarr"
-# s3 = s3fs.S3FileSystem()
-# zarr_store = s3fs.S3Map(path_to_zarr_store, s3=s3)
-# ds_zarr = xr.open_zarr(zarr_store, consolidated=None)
-# print(ds_zarr.Sv.shape)
-
-
-
-total = [246847, 89911, 169763, 658047, 887640, 708771, 187099, 3672813, 4095002, 763268, 162727, 189454, 1925270, 3575857, 1031920, 1167590, 3737415, 4099957, 3990725, 3619996, 3573052, 2973090, 55851, 143192, 1550164, 3692819, 668400, 489735, 393260, 1311234, 242989, 4515760, 1303091, 704663, 270645, 3886437, 4204381, 1062090, 428639, 541455, 4206506, 298561, 1279329, 137416, 139836, 228947, 517949]

water_column_sonar_processing/geometry/spatiotemporal.py

@@ -0,0 +1,106 @@
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+from shapely.geometry import Point
+
+from water_column_sonar_processing.model import ZarrManager
+
+
+# Convert "meters per second" to "knots"
+# meters_per_second_to_knots = lambda mps_value: mps_value * 1.94384
+
+
+class Spatiotemporal:
+    #######################################################
+    def __init__(
+        self,
+    ):
+        self.NANOSECONDS_PER_SECOND = 1e9
+        self.CUTOFF_DISTANCE_METERS = 50.0
+        self.CUTOFF_TIME_SECONDS = 10.0
+
+    #######################################################
+    @staticmethod
+    def meters_per_second_to_knots(
+        mps_value,
+    ):
+        return mps_value * 1.94384
+
+    #######################################################
+    def compute_speed_and_distance(
+        self,
+        times_ns,  #: np.ndarray[tuple[int], np.dtype[np.int64]],
+        latitudes,  #: np.ndarray,
+        longitudes,  #: np.ndarray,
+    ) -> pd.DataFrame:
+        try:
+            # fix times
+            times = np.array([np.datetime64(int(i), "ns") for i in times_ns])
+            geom = [Point(xy) for xy in zip(longitudes, latitudes)]
+            points_df = gpd.GeoDataFrame({"geometry": geom}, crs="EPSG:4326")
+            # Conversion to a rectilinear projection coordinate system where distance can be calculated with pythagorean theorem
+            # EPSG:4087, WGS 84 / World Equidistant Cylindrical
+            # https://epsg.io/4087
+            points_df.to_crs(epsg=4087, inplace=True)
+            distance_diffs = points_df.distance(points_df.geometry.shift())
+            distance_diffs[0] = distance_diffs[1]  # missing first datapoint, backfill
+            # Issue: np.max(distance_diffs) = 3397 meters
+            time_diffs_ns = np.append(0, (times[1:] - times[:-1]).astype(int))
+            time_diffs_ns[0] = time_diffs_ns[1]  # missing first datapoint, backfill
+            time_diffs_seconds = time_diffs_ns / self.NANOSECONDS_PER_SECOND
+            # Calculate the speed in knots
+            speed_meters_per_second = np.array(
+                (distance_diffs / time_diffs_ns * self.NANOSECONDS_PER_SECOND),
+                dtype=np.float32,
+            )
+            knots = self.meters_per_second_to_knots(speed_meters_per_second)
+            metrics_df = pd.DataFrame(
+                {
+                    "speed_knots": knots.astype(dtype=np.float32),
+                    "distance_meters": distance_diffs.to_numpy(dtype=np.float32),
+                    "diff_seconds": time_diffs_seconds.astype(np.float32),
+                },
+                index=times,
+            )
+            #
+            return metrics_df
+        except Exception as err:
+            raise RuntimeError(f"Exception encountered, {err}")
+
+    #######################################################
+    def add_speed_and_distance(
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        bucket_name,
+        endpoint_url=None,
+    ) -> None:
+        try:
+            zarr_manager = ZarrManager()
+            zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
+                ship_name=ship_name,
+                cruise_name=cruise_name,
+                sensor_name=sensor_name,
+                output_bucket_name=bucket_name,
+                endpoint_url=endpoint_url,
+            )
+            longitudes = zarr_store["longitude"][:]
+            latitudes = zarr_store["latitude"][:]
+            times = zarr_store["time"][:]
+            #
+            metrics_df = self.compute_speed_and_distance(
+                times_ns=times,
+                latitudes=latitudes,
+                longitudes=longitudes,
+            )
+            # Write the speed and distance to the output zarr store
+            zarr_store["speed"][:] = metrics_df.speed_knots.values
+            zarr_store["distance"][:] = metrics_df.distance_meters.values
+        except Exception as err:
+            raise RuntimeError(
+                f"Exception encountered writing the speed and distance, {err}"
+            )
+
+
+###########################################################