water-column-sonar-processing 0.0.1__py3-none-any.whl → 26.1.14__py3-none-any.whl

model/cruise/create_empty_zarr_store.py

@@ -1,166 +0,0 @@
-import os
-import numcodecs
-import numpy as np
-from ..utility.cleaner import Cleaner
-from ..aws.dynamodb_manager import DynamoDBManager
-from ..aws.s3_manager import S3Manager
-from ..zarr.zarr_manager import ZarrManager
-
-numcodecs.blosc.use_threads = False
-numcodecs.blosc.set_nthreads(1)
-
-TEMPDIR = "/tmp"
-
-# TODO: when ready switch to version 3 of zarr spec
-# ZARR_V3_EXPERIMENTAL_API = 1
-# creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
-
-class CreateEmptyZarrStore:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        self.__overwrite = True
-        # TODO: create output_bucket and input_bucket variables here?
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-
-    #######################################################
-
-    def upload_zarr_store_to_s3(
-            self,
-            local_directory: str,
-            object_prefix: str,
-            cruise_name: str,
-    ) -> None:
-        print('uploading zarr store to s3')
-        s3_manager = S3Manager()
-        #
-        print('Starting upload with thread pool executor.')
-        # # 'all_files' is passed a list of lists: [[local_path, s3_key], [...], ...]
-        all_files = []
-        for subdir, dirs, files in os.walk(f"{local_directory}/{cruise_name}.zarr"):
-            for file in files:
-                local_path = os.path.join(subdir, file)
-                # 'level_2/Henry_B._Bigelow/HB0806/EK60/HB0806.zarr/.zattrs'
-                s3_key = f'{object_prefix}/{cruise_name}.zarr{local_path.split(f"{cruise_name}.zarr")[-1]}'
-                all_files.append([local_path, s3_key])
-        #
-        # print(all_files)
-        s3_manager.upload_files_with_thread_pool_executor(
-            all_files=all_files,
-        )
-        print('Done uploading with thread pool executor.')
-        # TODO: move to common place
-
-    #######################################################
-    def create_cruise_level_zarr_store(
-            self,
-            ship_name: str,
-            cruise_name: str,
-            sensor_name: str,
-            table_name: str
-    ) -> None:
-        try:
-            # HB0806 - 123, HB0903 - 220
-            dynamo_db_manager = DynamoDBManager()
-
-            df = dynamo_db_manager.get_table_as_df(
-                table_name=table_name,
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name
-            )
-
-            # filter the dataframe just for enums >= LEVEL_1_PROCESSING
-            # df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
-
-            # TODO: VERIFY GEOJSON EXISTS as prerequisite!!!
-
-            print(f"DataFrame shape: {df.shape}")
-            cruise_channels = list(set([i for sublist in df['CHANNELS'].dropna() for i in sublist]))
-            cruise_channels.sort()
-
-            consolidated_zarr_width = np.sum(df['NUM_PING_TIME_DROPNA'].dropna().astype(int))
-
-            # [3] calculate the max/min measurement resolutions for the whole cruise
-            cruise_min_echo_range = float(np.min(df['MIN_ECHO_RANGE'].dropna().astype(float)))
-
-            # [4] calculate the maximum of the max depth values
-            cruise_max_echo_range = float(np.max(df['MAX_ECHO_RANGE'].dropna().astype(float)))
-            print(f"cruise_min_echo_range: {cruise_min_echo_range}, cruise_max_echo_range: {cruise_max_echo_range}")
-
-            # [5] get number of channels
-            cruise_frequencies = [float(i) for i in df['FREQUENCIES'].dropna().values.flatten()[0]]
-            print(cruise_frequencies)
-
-            new_width = int(consolidated_zarr_width)
-            print(f"new_width: {new_width}")
-            #################################################################
-            store_name = f"{cruise_name}.zarr"
-            print(store_name)
-            ################################################################
-            # Delete existing zarr store if it exists
-            s3_manager = S3Manager()
-            zarr_prefix = os.path.join("level_2", ship_name, cruise_name, sensor_name)
-            child_objects = s3_manager.get_child_objects(
-                bucket_name=self.output_bucket_name,
-                sub_prefix=zarr_prefix,
-            )
-            if len(child_objects) > 0:
-                s3_manager.delete_nodd_objects(
-                    objects=child_objects,
-                )
-            ################################################################
-            # Create new zarr store
-            zarr_manager = ZarrManager()
-            new_height = len(zarr_manager.get_depth_values(
-                min_echo_range=cruise_min_echo_range,
-                max_echo_range=cruise_max_echo_range
-            ))
-            print(f"new_height: {new_height}")
-
-            zarr_manager.create_zarr_store(
-                path=TEMPDIR,
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name,
-                frequencies=cruise_frequencies,
-                width=new_width,
-                min_echo_range=cruise_min_echo_range,
-                max_echo_range=cruise_max_echo_range,
-                calibration_status=True,
-            )
-            #################################################################
-            self.upload_zarr_store_to_s3(
-                local_directory=TEMPDIR,
-                object_prefix=zarr_prefix,
-                cruise_name=cruise_name,
-            )
-            # https://noaa-wcsd-zarr-pds.s3.amazonaws.com/index.html
-            #################################################################
-            # Verify count of the files uploaded
-            # count = self.__get_file_count(store_name=store_name)
-            # #
-            # raw_zarr_files = self.__get_s3_files(  # TODO: just need count
-            #     bucket_name=self.__output_bucket,
-            #     sub_prefix=os.path.join(zarr_prefix, store_name),
-            # )
-            # if len(raw_zarr_files) != count:
-            #     print(f'Problem writing {store_name} with proper count {count}.')
-            #     raise Exception("File count doesnt equal number of s3 Zarr store files.")
-            # else:
-            #     print("File counts match.")
-            #################################################################
-            # Success
-            # TODO: update enum in dynamodb
-            #################################################################
-        except Exception as err:
-            print(f"Problem trying to create new cruise zarr store: {err}")
-        finally:
-            cleaner = Cleaner()
-            cleaner.delete_local_files()
-        print("Done creating cruise level zarr store")
-
-
-###########################################################

model/cruise/resample_regrid.py

@@ -1,248 +0,0 @@
-import gc
-import os
-from pathlib import Path
-import numcodecs
-import numpy as np
-import xarray as xr
-import pandas as pd
-
-from ..geospatial.geometry_manager import GeoManager
-from ..aws.dynamodb_manager import DynamoDBManager
-from ..zarr.zarr_manager import ZarrManager
-
-numcodecs.blosc.use_threads = False
-numcodecs.blosc.set_nthreads(1)
-
-
-# TODO: when ready switch to version 3 of zarr spec
-#  ZARR_V3_EXPERIMENTAL_API = 1
-#  creates the latlon data: foo = ep.consolidate.add_location(ds_Sv, echodata)
-
-class ResampleRegrid:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        self.__overwrite = True
-        self.input_bucket_name = os.environ.get("INPUT_BUCKET_NAME")
-        self.output_bucket_name = os.environ.get("OUTPUT_BUCKET_NAME")
-        self.dtype = 'float32'
-
-    #################################################################
-    def interpolate_data(
-            self,
-            input_xr,
-            ping_times,
-            all_cruise_depth_values,
-    ) -> np.ndarray:
-        print("Interpolating data.")
-        try:
-            data = np.empty((
-                len(all_cruise_depth_values),
-                len(ping_times),
-                len(input_xr.frequency_nominal)
-            ), dtype=self.dtype)
-
-            data[:] = np.nan
-
-            regrid_resample = xr.DataArray(
-                data=data,
-                dims=("depth", "time", "frequency"),
-                coords={
-                    "depth": all_cruise_depth_values,
-                    "time": ping_times,
-                    "frequency": input_xr.frequency_nominal.values,
-                }
-            )
-
-            channels = input_xr.channel.values
-            for channel in range(len(channels)):  # TODO: leaving off here, need to subset for just indices in time axis
-                print(np.nanmax(input_xr.echo_range.sel(channel=input_xr.channel[channel]).values))
-                #
-                max_depths = np.nanmax(
-                    a=input_xr.echo_range.sel(channel=input_xr.channel[channel]).values,
-                    axis=1
-                )
-                superset_of_max_depths = set(
-                    np.nanmax(input_xr.echo_range.sel(channel=input_xr.channel[channel]).values, 1)
-                )
-                set_of_max_depths = list({x for x in superset_of_max_depths if x == x})  # removes nan's
-                # iterate through partitions of data with similar depths and resample
-                for select_max_depth in set_of_max_depths:
-                    # TODO: for nan just skip and leave all nan's
-                    select_indices = [i for i in range(0, len(max_depths)) if max_depths[i] == select_max_depth]
-
-                    # now create new DataArray with proper dimension and indices
-                    # data_select = input_xr.Sv.sel(
-                    #     channel=input_xr.channel[channel]
-                    # ).values[select_indices, :].T  # TODO: dont like this transpose
-                    data_select = input_xr.Sv.sel(channel=input_xr.channel[channel])[select_indices, :].T.values
-                    # change from ".values[select_indices, :].T" to "[select_indices, :].values.T"
-
-                    times_select = input_xr.ping_time.values[select_indices]
-                    depths_select = input_xr.echo_range.sel(
-                        channel=input_xr.channel[channel]
-                    ).values[select_indices[0], :]  # '0' because all others in group should be same
-
-                    da_select = xr.DataArray(
-                        data=data_select,
-                        dims=("depth", "time"),
-                        coords={
-                            "depth": depths_select,
-                            "time": times_select,
-                        }
-                    ).dropna(dim='depth')
-                    resampled = da_select.interp(depth=all_cruise_depth_values, method="nearest")
-                    # write to the resample array
-                    regrid_resample.loc[
-                        dict(time=times_select, frequency=input_xr.frequency_nominal.values[channel])
-                    ] = resampled
-                    print(f"updated {len(times_select)} ping times")
-        except Exception as err:
-            print(f'Problem finding the dynamodb table: {err}')
-            raise err
-        print("Done interpolating data.")
-        return regrid_resample
-
-    #################################################################
-    def resample_regrid(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            table_name,
-    ) -> None:
-        """
-        The goal here is to interpolate the data against the depth values already populated
-        in the existing file level zarr stores. We open the cruise-level store with zarr for
-        read/write operations. We open the file-level store with Xarray to leverage tools for
-        resampling and subsetting the data.
-        """
-        print("Interpolating data.")
-        try:
-            zarr_manager = ZarrManager()
-            # s3_manager = S3Manager()
-            geo_manager = GeoManager()
-            # get zarr store
-            output_zarr_store = zarr_manager.open_s3_zarr_store_with_zarr(
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name,
-                # zarr_synchronizer=?  # TODO: pass in for parallelization
-            )
-
-            # get dynamo stuff
-            dynamo_db_manager = DynamoDBManager()
-            cruise_df = dynamo_db_manager.get_table_as_df(
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name,
-                table_name=table_name,
-            )
-
-            #########################################################
-            #########################################################
-            # TODO: iterate files here
-            all_file_names = cruise_df['FILE_NAME']
-            for file_name in all_file_names:
-                gc.collect()
-                file_name_stem = Path(file_name).stem
-                # file_name_stem = "D20070724-T151330"
-                print(f"Processing file: {file_name_stem}.")
-                # if f"{file_name_stem}.raw" not in list(cruise_df['FILE_NAME']):
-                #     raise Exception(f"Raw file file_stem not found in dynamodb.")
-
-                # status = PipelineStatus['LEVEL_1_PROCESSING']
-                # TODO: filter rows by enum success, filter the dataframe just for enums >= LEVEL_1_PROCESSING
-                #  df[df['PIPELINE_STATUS'] < PipelineStatus.LEVEL_1_PROCESSING] = np.nan
-
-                # Get index from all cruise files. Note: should be based on which are included in cruise.
-                index = cruise_df.index[cruise_df['FILE_NAME'] == f"{file_name_stem}.raw"][0]
-
-                # get input store
-                input_xr_zarr_store = zarr_manager.open_s3_zarr_store_with_xarray(
-                    ship_name=ship_name,
-                    cruise_name=cruise_name,
-                    sensor_name=sensor_name,
-                    file_name_stem=file_name_stem,
-                )
-                #########################################################################
-                # [3] Get needed indices
-                # Offset from start index to insert new data. Note that missing values are excluded.
-                ping_time_cumsum = np.insert(
-                    np.cumsum(cruise_df['NUM_PING_TIME_DROPNA'].dropna().to_numpy(dtype=int)),
-                    obj=0,
-                    values=0
-                )
-                start_ping_time_index = ping_time_cumsum[index]
-                end_ping_time_index = ping_time_cumsum[index + 1]
-
-                min_echo_range = np.nanmin(np.float32(cruise_df['MIN_ECHO_RANGE']))
-                max_echo_range = np.nanmax(np.float32(cruise_df['MAX_ECHO_RANGE']))
-
-                print("Creating empty ndarray for Sv data.")  # Note: cruise_zarr dimensions are (depth, time, frequency)
-                cruise_sv_subset = np.empty(
-                    shape=output_zarr_store.Sv[:, start_ping_time_index:end_ping_time_index, :].shape
-                )
-                cruise_sv_subset[:, :, :] = np.nan  # (5208, 9778, 4)
-
-                all_cruise_depth_values = zarr_manager.get_depth_values(
-                    min_echo_range=min_echo_range,
-                    max_echo_range=max_echo_range
-                )
-
-                print(" ".join(list(input_xr_zarr_store.Sv.dims)))
-                if set(input_xr_zarr_store.Sv.dims) != {'channel', 'ping_time', 'range_sample'}:
-                    raise Exception("Xarray dimensions are not as expected.")
-
-                # get geojson
-                indices, geospatial = geo_manager.read_s3_geo_json(
-                    ship_name=ship_name,
-                    cruise_name=cruise_name,
-                    sensor_name=sensor_name,
-                    file_name_stem=file_name_stem,
-                    input_xr_zarr_store=input_xr_zarr_store,
-                )
-
-                input_xr = input_xr_zarr_store.isel(ping_time=indices)
-
-                ping_times = input_xr.ping_time.values
-                # Date format: numpy.datetime64('2007-07-20T02:10:25.845073920') converts to "1184897425.845074"
-                epoch_seconds = [(pd.Timestamp(i) - pd.Timestamp('1970-01-01')) / pd.Timedelta('1s') for i in ping_times]
-                output_zarr_store.time[start_ping_time_index:end_ping_time_index] = epoch_seconds
-
-                # --- UPDATING --- #
-
-                regrid_resample = self.interpolate_data(
-                    input_xr=input_xr,
-                    ping_times=ping_times,
-                    all_cruise_depth_values=all_cruise_depth_values,
-                )
-
-                print(f"start_ping_time_index: {start_ping_time_index}, end_ping_time_index: {end_ping_time_index}")
-
-                #########################################################################
-                # write Sv values to cruise-level-zarr-store
-                for channel in range(len(input_xr.channel.values)):  # doesn't like being written in one fell swoop :(
-                    output_zarr_store.Sv[
-                        :,
-                        start_ping_time_index:end_ping_time_index,
-                        channel
-                    ] = regrid_resample[:, :, channel]
-
-                #########################################################################
-                # [5] write subset of latitude/longitude
-                output_zarr_store.latitude[start_ping_time_index:end_ping_time_index] = geospatial.dropna()[
-                    'latitude'
-                ].values
-                output_zarr_store.longitude[start_ping_time_index:end_ping_time_index] = geospatial.dropna()[
-                    'longitude'
-                ].values
-        except Exception as err:
-            print(f'Problem interpolating the data: {err}')
-            raise err
-        print("Done interpolating data.")
-
-    #######################################################
-
-###########################################################

model/geospatial/geometry_manager.py

@@ -1,194 +0,0 @@
-from pathlib import Path
-import numpy as np
-import geopandas
-import pandas as pd
-
-from ..utility.cleaner import Cleaner
-from ..aws.s3_manager import S3Manager
-
-"""
-//  [Decimal / Places / Degrees	/ Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
-  //  0   1.0	        1° 00′ 0″	        country or large region                             111.32 km	  102.47 km	  78.71 km	43.496 km
-  //  1	  0.1	        0° 06′ 0″         large city or district                              11.132 km	  10.247 km	  7.871 km	4.3496 km
-  //  2	  0.01	      0° 00′ 36″        town or village                                     1.1132 km	  1.0247 km	  787.1 m	  434.96 m
-  //  3	  0.001	      0° 00′ 3.6″       neighborhood, street                                111.32 m	  102.47 m	  78.71 m	  43.496 m
-  //  4	  0.0001	    0° 00′ 0.36″      individual street, land parcel                      11.132 m	  10.247 m	  7.871 m	  4.3496 m
-  //  5	  0.00001	    0° 00′ 0.036″     individual trees, door entrance	                    1.1132 m	  1.0247 m	  787.1 mm	434.96 mm
-  //  6	  0.000001	  0° 00′ 0.0036″    individual humans                                   111.32 mm	  102.47 mm	  78.71 mm	43.496 mm
-  //  7	  0.0000001	  0° 00′ 0.00036″   practical limit of commercial surveying	            11.132 mm	  10.247 mm	  7.871 mm	4.3496 mm
-"""
-
-
-class GeoManager:
-    #######################################################
-    def __init__(
-            self,
-    ):
-        self.DECIMAL_PRECISION = 5  # precision for GPS coordinates
-        self.SIMPLIFICATION_TOLERANCE = 0.0001  # RDP simplification to street level
-
-    #######################################################
-    def read_echodata_gps_data(
-            self,
-            echodata,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name,
-            write_geojson=True,
-    ) -> tuple:
-        file_name_stem = Path(file_name).stem
-        geo_json_name = f"{file_name_stem}.json"
-
-        print('Getting GPS data from echopype object.')
-        try:
-            latitude = np.round(echodata.platform.latitude.values, self.DECIMAL_PRECISION)
-            longitude = np.round(echodata.platform.longitude.values, self.DECIMAL_PRECISION)
-
-            # RE: time coordinates: https://github.com/OSOceanAcoustics/echopype/issues/656#issue-1219104771
-            # 'nmea_times' are times from the nmea datalogger associated with GPS
-            #   note that nmea_times, unlike time1, can be sorted
-            nmea_times = np.sort(echodata.platform.time1.values)
-
-            # 'time1' are times from the echosounder associated with the data of the transducer measurement
-            time1 = echodata.environment.time1.values
-
-            if len(nmea_times) < len(time1):
-                raise Exception("Problem: Not enough NMEA times available to extrapolate time1.")
-
-            # Align 'sv_times' to 'nmea_times'
-            if not (np.all(time1[:-1] <= time1[1:]) and np.all(nmea_times[:-1] <= nmea_times[1:])):
-                raise Exception("Problem: NMEA times are not sorted.")
-
-            # Finds the indices where 'v' can be inserted just to the right of 'a'
-            indices = np.searchsorted(a=nmea_times, v=time1, side="right") - 1
-            lat = latitude[indices]
-            lat[indices < 0] = np.nan  # values recorded before indexing are set to nan
-            lon = longitude[indices]
-            lon[indices < 0] = np.nan
-
-            if not (np.all(lat[~np.isnan(lat)] >= -90.) and np.all(lat[~np.isnan(lat)] <= 90.) and np.all(lon[~np.isnan(lon)] >= -180.) and np.all(lon[~np.isnan(lon)] <= 180.)):
-                raise Exception("Problem: GPS Data falls outside allowed bounds.")
-
-            # check for visits to null island
-            null_island_indices = list(
-                set.intersection(set(np.where(np.abs(lat) < 1e-3)[0]), set(np.where(np.abs(lon) < 1e-3)[0]))
-            )
-            lat[null_island_indices] = np.nan
-            lon[null_island_indices] = np.nan
-
-            # create requirement for minimum linestring size
-            MIN_ALLOWED_SIZE = 4  # don't want to process files with less than 4 data points
-            if len(lat[~np.isnan(lat)]) < MIN_ALLOWED_SIZE or len(lon[~np.isnan(lon)]) < MIN_ALLOWED_SIZE:
-                raise Exception(
-                    f"There was not enough data in lat or lon to create geojson, {len(lat[~np.isnan(lat)])} found, less than {MIN_ALLOWED_SIZE}."
-                )
-
-            # https://osoceanacoustics.github.io/echopype-examples/echopype_tour.html
-            gps_df = pd.DataFrame({
-                'latitude': lat,
-                'longitude': lon,
-                'time': time1
-            }).set_index(['time']).fillna(0)
-
-            # Note: We set np.nan to 0,0 so downstream missing values can be omitted
-            gps_gdf = geopandas.GeoDataFrame(
-                gps_df,
-                geometry=geopandas.points_from_xy(
-                    gps_df['longitude'],
-                    gps_df['latitude']
-                ),
-                crs="epsg:4326"
-            )
-            # Note: We set np.nan to 0,0 so downstream missing values can be omitted
-
-            geo_json_line = gps_gdf.to_json()
-            if write_geojson:
-                print('Creating local copy of geojson file.')
-                with open(geo_json_name, "w") as write_file:
-                    write_file.write(geo_json_line)
-
-                geo_json_prefix = f"spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}"
-
-                print('Checking s3 and deleting any existing GeoJSON file.')
-                s3_manager = S3Manager()
-                s3_objects = s3_manager.list_nodd_objects(prefix=f"{geo_json_prefix}/{geo_json_name}")
-                if len(s3_objects) > 0:
-                    print('GeoJSON already exists in s3, deleting existing and continuing.')
-                    s3_manager.delete_nodd_objects(objects=s3_objects)
-
-                print('Upload GeoJSON to s3.')
-                s3_manager.upload_nodd_file(
-                    file_name=geo_json_name,  # file_name
-                    key=f"{geo_json_prefix}/{geo_json_name}"  # key
-                )
-
-                # TODO: delete geo_json file
-                cleaner = Cleaner()
-                cleaner.delete_local_files(file_types=['*.json'])
-
-            #################################################################
-            # TODO: simplify with shapely
-            # linestring = shapely.geometry.LineString(
-            #     [xy for xy in zip(gps_gdf.longitude, gps_gdf.latitude)]
-            # )
-            # len(linestring.coords)
-            # line_simplified = linestring.simplify(
-            #     tolerance=self.SIMPLIFICATION_TOLERANCE,
-            #     preserve_topology=True
-            # )
-            # print(f"Total number of points for original linestring: {len(linestring.coords)}")
-            # print(f"Total number of points needed for the simplified linestring: {len(line_simplified.coords)}")
-            # print(line_simplified)
-            # geo_json_line_simplified = shapely.to_geojson(line_simplified)
-            #################################################################
-            # GeoJSON FeatureCollection with IDs as "time"
-        except Exception as err:
-            print(f'Exception encountered extracting gps coordinates creating geojson: {err}')
-            raise
-        # Note: returned lat/lon values can include np.nan because they need to be aligned with
-        # the Sv data! GeoJSON needs simplification but has been filtered.
-        return gps_df.index.values, gps_df.latitude.values, gps_df.longitude.values
-        # TODO: if geojson is already returned with 0,0, the return here
-        #  can include np.nan values?
-
-    #######################################################
-    def read_s3_geo_json(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name_stem,
-            input_xr_zarr_store,
-    ):
-        try:
-            s3_manager = S3Manager()
-            geo_json = s3_manager.read_s3_json(
-                ship_name=ship_name,
-                cruise_name=cruise_name,
-                sensor_name=sensor_name,
-                file_name_stem=file_name_stem,
-            )
-            ###
-            geospatial = geopandas.GeoDataFrame.from_features(geo_json['features']).set_index(
-                pd.json_normalize(geo_json["features"])["id"].values
-            )
-            null_island_indices = list(
-                set.intersection(
-                    set(np.where(np.abs(geospatial.latitude.values) < 1e-3)[0]),
-                    set(np.where(np.abs(geospatial.longitude.values) < 1e-3)[0])
-                )
-            )
-            geospatial.iloc[null_island_indices] = np.nan
-            ###
-            geospatial_index = geospatial.dropna().index.values.astype('datetime64[ns]')
-            aa = input_xr_zarr_store.ping_time.values.tolist()
-            vv = geospatial_index.tolist()
-            indices = np.searchsorted(a=aa, v=vv)
-
-            return indices, geospatial
-        except Exception as err:  # Failure
-            print(f'Exception encountered reading s3 GeoJSON: {err}')
-            raise
-
-###########################################################