water-column-sonar-processing 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

{geometry_manager → water_column_sonar_processing/geometry}/geometry_manager.py

@@ -1,10 +1,11 @@
 from pathlib import Path
-import numpy as np
+
 import geopandas
+import numpy as np
 import pandas as pd
 
-from utility.cleaner import Cleaner
-from aws_manager.s3_manager import S3Manager
+from water_column_sonar_processing.aws.s3_manager import S3Manager
+from water_column_sonar_processing.utility.cleaner import Cleaner
 
 """
 //  [Decimal / Places / Degrees	/ Object that can be recognized at scale / N/S or E/W at equator, E/W at 23N/S, E/W at 45N/S, E/W at 67N/S]
@@ -22,28 +23,32 @@ from aws_manager.s3_manager import S3Manager
 class GeometryManager:
     #######################################################
     def __init__(
-            self,
+        self,
     ):
         self.DECIMAL_PRECISION = 5  # precision for GPS coordinates
         self.SIMPLIFICATION_TOLERANCE = 0.0001  # RDP simplification to street level
 
     #######################################################
     def read_echodata_gps_data(
-            self,
-            echodata,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name,
-            write_geojson=True,
+        self,
+        echodata,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        file_name,
+        write_geojson=True,
     ) -> tuple:
         file_name_stem = Path(file_name).stem
         geo_json_name = f"{file_name_stem}.json"
 
-        print('Getting GPS data from echopype object.')
+        print("Getting GPS data from echopype object.")
         try:
-            latitude = np.round(echodata.platform.latitude.values, self.DECIMAL_PRECISION)
-            longitude = np.round(echodata.platform.longitude.values, self.DECIMAL_PRECISION)
+            latitude = np.round(
+                echodata.platform.latitude.values, self.DECIMAL_PRECISION
+            )
+            longitude = np.round(
+                echodata.platform.longitude.values, self.DECIMAL_PRECISION
+            )
 
             # RE: time coordinates: https://github.com/OSOceanAcoustics/echopype/issues/656#issue-1219104771
             # 'nmea_times' are times from the nmea datalogger associated with GPS
@@ -54,10 +59,15 @@ class GeometryManager:
             time1 = echodata.environment.time1.values
 
             if len(nmea_times) < len(time1):
-                raise Exception("Problem: Not enough NMEA times available to extrapolate time1.")
+                raise Exception(
+                    "Problem: Not enough NMEA times available to extrapolate time1."
+                )
 
             # Align 'sv_times' to 'nmea_times'
-            if not (np.all(time1[:-1] <= time1[1:]) and np.all(nmea_times[:-1] <= nmea_times[1:])):
+            if not (
+                np.all(time1[:-1] <= time1[1:])
+                and np.all(nmea_times[:-1] <= nmea_times[1:])
+            ):
                 raise Exception("Problem: NMEA times are not sorted.")
 
             # Finds the indices where 'v' can be inserted just to the right of 'a'
@@ -67,65 +77,83 @@ class GeometryManager:
             lon = longitude[indices]
             lon[indices < 0] = np.nan
 
-            if not (np.all(lat[~np.isnan(lat)] >= -90.) and np.all(lat[~np.isnan(lat)] <= 90.) and np.all(lon[~np.isnan(lon)] >= -180.) and np.all(lon[~np.isnan(lon)] <= 180.)):
+            if not (
+                np.all(lat[~np.isnan(lat)] >= -90.0)
+                and np.all(lat[~np.isnan(lat)] <= 90.0)
+                and np.all(lon[~np.isnan(lon)] >= -180.0)
+                and np.all(lon[~np.isnan(lon)] <= 180.0)
+            ):
                 raise Exception("Problem: GPS Data falls outside allowed bounds.")
 
             # check for visits to null island
             null_island_indices = list(
-                set.intersection(set(np.where(np.abs(lat) < 1e-3)[0]), set(np.where(np.abs(lon) < 1e-3)[0]))
+                set.intersection(
+                    set(np.where(np.abs(lat) < 1e-3)[0]),
+                    set(np.where(np.abs(lon) < 1e-3)[0]),
+                )
             )
             lat[null_island_indices] = np.nan
             lon[null_island_indices] = np.nan
 
             # create requirement for minimum linestring size
-            MIN_ALLOWED_SIZE = 4  # don't want to process files with less than 4 data points
-            if len(lat[~np.isnan(lat)]) < MIN_ALLOWED_SIZE or len(lon[~np.isnan(lon)]) < MIN_ALLOWED_SIZE:
+            MIN_ALLOWED_SIZE = (
+                4  # don't want to process files with less than 4 data points
+            )
+            if (
+                len(lat[~np.isnan(lat)]) < MIN_ALLOWED_SIZE
+                or len(lon[~np.isnan(lon)]) < MIN_ALLOWED_SIZE
+            ):
                 raise Exception(
                     f"There was not enough data in lat or lon to create geojson, {len(lat[~np.isnan(lat)])} found, less than {MIN_ALLOWED_SIZE}."
                 )
 
             # https://osoceanacoustics.github.io/echopype-examples/echopype_tour.html
-            gps_df = pd.DataFrame({
-                'latitude': lat,
-                'longitude': lon,
-                'time': time1
-            }).set_index(['time']).fillna(0)
+            gps_df = (
+                pd.DataFrame({"latitude": lat, "longitude": lon, "time": time1})
+                .set_index(["time"])
+                .fillna(0)
+            )
 
             # Note: We set np.nan to 0,0 so downstream missing values can be omitted
             gps_gdf = geopandas.GeoDataFrame(
                 gps_df,
                 geometry=geopandas.points_from_xy(
-                    gps_df['longitude'],
-                    gps_df['latitude']
+                    gps_df["longitude"], gps_df["latitude"]
                 ),
-                crs="epsg:4326"
+                crs="epsg:4326",
             )
             # Note: We set np.nan to 0,0 so downstream missing values can be omitted
 
             geo_json_line = gps_gdf.to_json()
             if write_geojson:
-                print('Creating local copy of geojson file.')
+                print("Creating local copy of geojson file.")
                 with open(geo_json_name, "w") as write_file:
                     write_file.write(geo_json_line)
 
-                geo_json_prefix = f"spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}"
+                geo_json_prefix = (
+                    f"spatial/geojson/{ship_name}/{cruise_name}/{sensor_name}"
+                )
 
-                print('Checking s3 and deleting any existing GeoJSON file.')
+                print("Checking s3 and deleting any existing GeoJSON file.")
                 s3_manager = S3Manager()
-                s3_objects = s3_manager.list_nodd_objects(prefix=f"{geo_json_prefix}/{geo_json_name}")
+                s3_objects = s3_manager.list_nodd_objects(
+                    prefix=f"{geo_json_prefix}/{geo_json_name}"
+                )
                 if len(s3_objects) > 0:
-                    print('GeoJSON already exists in s3, deleting existing and continuing.')
+                    print(
+                        "GeoJSON already exists in s3, deleting existing and continuing."
+                    )
                     s3_manager.delete_nodd_objects(objects=s3_objects)
 
-                print('Upload GeoJSON to s3.')
+                print("Upload GeoJSON to s3.")
                 s3_manager.upload_nodd_file(
                     file_name=geo_json_name,  # file_name
-                    key=f"{geo_json_prefix}/{geo_json_name}"  # key
+                    key=f"{geo_json_prefix}/{geo_json_name}",  # key
                 )
 
                 # TODO: delete geo_json file
                 cleaner = Cleaner()
-                cleaner.delete_local_files(file_types=['*.json'])
+                cleaner.delete_local_files(file_types=["*.json"])
 
             #################################################################
             # TODO: simplify with shapely
@@ -144,7 +172,9 @@ class GeometryManager:
             #################################################################
             # GeoJSON FeatureCollection with IDs as "time"
         except Exception as err:
-            print(f'Exception encountered extracting gps coordinates creating geojson: {err}')
+            print(
+                f"Exception encountered extracting gps coordinates creating geojson: {err}"
+            )
             raise
         # Note: returned lat/lon values can include np.nan because they need to be aligned with
         # the Sv data! GeoJSON needs simplification but has been filtered.
@@ -154,12 +184,12 @@ class GeometryManager:
 
     #######################################################
     def read_s3_geo_json(
-            self,
-            ship_name,
-            cruise_name,
-            sensor_name,
-            file_name_stem,
-            input_xr_zarr_store,
+        self,
+        ship_name,
+        cruise_name,
+        sensor_name,
+        file_name_stem,
+        input_xr_zarr_store,
     ):
         try:
             s3_manager = S3Manager()
@@ -170,25 +200,26 @@ class GeometryManager:
                 file_name_stem=file_name_stem,
             )
             ###
-            geospatial = geopandas.GeoDataFrame.from_features(geo_json['features']).set_index(
-                pd.json_normalize(geo_json["features"])["id"].values
-            )
+            geospatial = geopandas.GeoDataFrame.from_features(
+                geo_json["features"]
+            ).set_index(pd.json_normalize(geo_json["features"])["id"].values)
             null_island_indices = list(
                 set.intersection(
                     set(np.where(np.abs(geospatial.latitude.values) < 1e-3)[0]),
-                    set(np.where(np.abs(geospatial.longitude.values) < 1e-3)[0])
+                    set(np.where(np.abs(geospatial.longitude.values) < 1e-3)[0]),
                 )
             )
             geospatial.iloc[null_island_indices] = np.nan
             ###
-            geospatial_index = geospatial.dropna().index.values.astype('datetime64[ns]')
+            geospatial_index = geospatial.dropna().index.values.astype("datetime64[ns]")
             aa = input_xr_zarr_store.ping_time.values.tolist()
             vv = geospatial_index.tolist()
             indices = np.searchsorted(a=aa, v=vv)
 
             return indices, geospatial
         except Exception as err:  # Failure
-            print(f'Exception encountered reading s3 GeoJSON: {err}')
+            print(f"Exception encountered reading s3 GeoJSON: {err}")
             raise
 
+
 ###########################################################

{geometry_manager → water_column_sonar_processing/geometry}/geometry_simplification.py

@@ -31,51 +31,52 @@
 
     
 """
-class GeometrySimplification(object):
+
+
+class GeometrySimplification:
     # TODO: in the future move to standalone library
     #######################################################
     def __init__(
-            self,
+        self,
     ):
         pass
 
     #######################################################
     def speed_check(
-            self,
-            speed_knots=50,
+        self,
+        speed_knots=50,
     ) -> None:
         print(speed_knots)
         pass
 
     def remove_null_island_values(
-            self,
-            epsilon=1e-5,
+        self,
+        epsilon=1e-5,
     ) -> None:
         print(epsilon)
         pass
 
     def stream_geometry(
-            self,
+        self,
     ) -> None:
         pass
 
     def break_linestring_into_multi_linestring(
-            self,
+        self,
     ) -> None:
         # For any line-strings across the antimeridian, break into multilinestring
         pass
 
     def simplify(
-            self,
+        self,
     ) -> None:
         pass
 
-    def kalman_filter(
-            self
-    ):
+    def kalman_filter(self):
         # for cruises with bad signal, filter so that
         pass
 
     #######################################################
 
+
 ###########################################################

{geometry_manager → water_column_sonar_processing/geometry}/pmtile_generation.py

@@ -1,29 +1,25 @@
-# import json
-# This is a sample Python script.
-import pandas as pd
-# import numpy as np
 import os
-# from glob import glob
 from pathlib import Path
-import geopandas
-# import shapely
-from shapely.geometry import LineString
+
 # from shapely import wkt
 # import json
 # from shapely.geometry import shape, GeometryCollection
 import fiona
+import geopandas
+import pandas as pd
+from shapely.geometry import LineString
 
 
 class PMTileGeneration(object):
     #######################################################
     def __init__(
-            self,
+        self,
     ):
-        pass
+        print("123")
 
     #######################################################
     def generate_geojson_feature_collection(self):
-        # This was used to read from noaa-wcsd-zarr_manager-pds bucket geojson files and then to
+        # This was used to read from noaa-wcsd-model-pds bucket geojson files and then to
         # generate the geopandas dataframe which could be exported to another comprehensive
         # geojson file. That
         result = list(Path("/Users/r2d2/Documents/echofish/geojson").rglob("*.json"))
@@ -33,30 +29,34 @@ class PMTileGeneration(object):
         for iii in range(len(result)):
             file_name = os.path.normpath(result[iii]).split(os.sep)[-1]
             file_stem = os.path.splitext(os.path.basename(file_name))[0]
-            geom = geopandas.read_file(result[iii]).iloc[0]['geometry']
+            geom = geopandas.read_file(result[iii]).iloc[0]["geometry"]
             # TDOO: Filter (0,0) coordinates
             if len(geom.coords.xy[0]) < 2:
                 continue
             geom = LineString(list(zip(geom.coords.xy[1], geom.coords.xy[0])))
-            pieces.append({
-                'ship_name': os.path.normpath(result[iii]).split(os.sep)[-4],
-                'cruise_name': os.path.normpath(result[iii]).split(os.sep)[-3],
-                'file_stem': file_stem,
-                'file_path': result[iii],
-                'geom': geom,
-            })
+            pieces.append(
+                {
+                    "ship_name": os.path.normpath(result[iii]).split(os.sep)[-4],
+                    "cruise_name": os.path.normpath(result[iii]).split(os.sep)[-3],
+                    "file_stem": file_stem,
+                    "file_path": result[iii],
+                    "geom": geom,
+                }
+            )
         df = pd.DataFrame(pieces)
         print(df)
         gps_gdf = geopandas.GeoDataFrame(
-            data=df[['ship_name', 'cruise_name', 'file_stem']],  # try again with file_stem
-            geometry=df['geom'],
-            crs='EPSG:4326'
+            data=df[
+                ["ship_name", "cruise_name", "file_stem"]
+            ],  # try again with file_stem
+            geometry=df["geom"],
+            crs="EPSG:4326",
         )
         print(fiona.supported_drivers)
         # gps_gdf.to_file('dataframe.shp', crs='epsg:4326')
         # Convert geojson feature collection to pmtiles
-        gps_gdf.to_file('dataframe.geojson', driver='GeoJSON', crs='epsg:4326')
-        print('done')
+        gps_gdf.to_file("dataframe.geojson", driver="GeoJSON", crs="epsg:4326")
+        print("done")
         """
         # need to eliminate visits to null island
         tippecanoe --no-feature-limit -zg --projection=EPSG:4326 -o dataframe.pmtiles -l cruises dataframe.geojson
@@ -71,4 +71,5 @@ class PMTileGeneration(object):
 
     #######################################################
 
+
 ###########################################################

water_column_sonar_processing/index/__init__.py

@@ -0,0 +1,3 @@
+from .index_manager import IndexManager
+
+__all__ = ["IndexManager"]