ngiab-data-preprocess 4.2.1__py3-none-any.whl → 4.3.0__py3-none-any.whl

data_processing/gpkg_utils.py

@@ -2,11 +2,12 @@ import logging
 import sqlite3
 import struct
 from pathlib import Path
-from typing import List, Tuple, Union
+from typing import List, Tuple, Dict
 
 import pyproj
 from data_processing.file_paths import file_paths
-from shapely.geometry import Point, Polygon
+from shapely.geometry import Point
+from shapely.geometry.base import BaseGeometry
 from shapely.ops import transform
 from shapely.wkb import loads
 
@@ -27,7 +28,7 @@ class GeoPackage:
         self.conn.close()
 
 
-def verify_indices(gpkg: str = file_paths.conus_hydrofabric) -> None:
+def verify_indices(gpkg: Path = file_paths.conus_hydrofabric) -> None:
     """
     Verify that the indices in the specified geopackage are correct.
     If they are not, create the correct indices.
@@ -92,12 +93,9 @@ def add_triggers_to_gpkg(gpkg: Path) -> None:
     logger.debug(f"Added triggers to subset gpkg {gpkg}")
 
 
-# whenever this is imported, check if the indices are correct
-if file_paths.conus_hydrofabric.is_file():
-    verify_indices()
 
 
-def blob_to_geometry(blob: bytes) -> Union[Point, Polygon]:
+def blob_to_geometry(blob: bytes) -> BaseGeometry | None:
     """
     Convert a blob to a geometry.
     from http://www.geopackage.org/spec/#gpb_format
@@ -120,7 +118,7 @@ def blob_to_geometry(blob: bytes) -> Union[Point, Polygon]:
     return geometry
 
 
-def blob_to_centre_point(blob: bytes) -> Point:
+def blob_to_centre_point(blob: bytes) -> Point | None:
     """
     Convert a blob to a geometry.
     from http://www.geopackage.org/spec/#gpb_format
@@ -151,7 +149,7 @@ def blob_to_centre_point(blob: bytes) -> Point:
     return Point(x, y)
 
 
-def convert_to_5070(shapely_geometry):
+def convert_to_5070(shapely_geometry: Point) -> Point:
     # convert to web mercator
     if shapely_geometry.is_empty:
         return shapely_geometry
@@ -164,7 +162,7 @@ def convert_to_5070(shapely_geometry):
     return new_geometry
 
 
-def get_catid_from_point(coords):
+def get_catid_from_point(coords: Dict[str, float]) -> str:
     """
     Retrieves the watershed boundary ID (catid) of the watershed that contains the given point.
 
@@ -196,6 +194,8 @@ def get_catid_from_point(coords):
         # check the geometries to see which one contains the point
         for result in results:
             geom = blob_to_geometry(result[1])
+            if geom is None:
+                continue
             if geom.contains(point):
                 return result[0]
     return results[0][0]
@@ -321,16 +321,16 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     if table == "network":
         # Look for the network entry that has a toid not in the flowpath or nexus tables
         network_toids = [x[2] for x in contents]
-        print(f"Network toids: {len(network_toids)}")
+        logger.debug(f"Network toids: {len(network_toids)}")
         sql = "SELECT id FROM flowpaths"
         flowpath_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
-        print(f"Flowpath ids: {len(flowpath_ids)}")
+        logger.debug(f"Flowpath ids: {len(flowpath_ids)}")
         sql = "SELECT id FROM nexus"
         nexus_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
-        print(f"Nexus ids: {len(nexus_ids)}")
+        logger.debug(f"Nexus ids: {len(nexus_ids)}")
         bad_ids = set(network_toids) - set(flowpath_ids + nexus_ids)
-        print(bad_ids)
-        print(f"Removing {len(bad_ids)} network entries that are not in flowpaths or nexuses")
+        logger.debug(bad_ids)
+        logger.info(f"Removing {len(bad_ids)} network entries that are not in flowpaths or nexuses")
         # id column is second after fid
         contents = [x for x in contents if x[1] not in bad_ids]
 
@@ -398,7 +398,7 @@ def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name
     dest_db.close()
 
 
-def get_table_crs_short(gpkg: str, table: str) -> str:
+def get_table_crs_short(gpkg: str | Path, table: str) -> str:
     """
     Gets the CRS of the specified table in the specified geopackage as a short string. e.g. EPSG:5070
 
@@ -518,7 +518,7 @@ def get_available_tables(gpkg: Path) -> List[str]:
     return tables
 
 
-def get_cat_to_nhd_feature_id(gpkg: Path = file_paths.conus_hydrofabric) -> dict:
+def get_cat_to_nhd_feature_id(gpkg: Path = file_paths.conus_hydrofabric) -> Dict[str, int]:
     available_tables = get_available_tables(gpkg)
     possible_tables = ["flowpath_edge_list", "network"]
 
@@ -535,7 +535,7 @@ def get_cat_to_nhd_feature_id(gpkg: Path = file_paths.conus_hydrofabric) -> dict
     sql_query = f"SELECT divide_id, hf_id FROM {table_name} WHERE divide_id IS NOT NULL AND hf_id IS NOT NULL"
 
     with sqlite3.connect(gpkg) as conn:
-        result = conn.execute(sql_query).fetchall()
+        result: List[Tuple[str, str]] = conn.execute(sql_query).fetchall()
 
     mapping = {}
     for cat, feature in result:

data_processing/graph_utils.py

@@ -2,7 +2,7 @@ import logging
 import sqlite3
 from functools import cache
 from pathlib import Path
-from typing import List, Set, Union
+from typing import List, Optional, Set, Union
 
 import igraph as ig
 from data_processing.file_paths import file_paths
@@ -11,7 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 def get_from_to_id_pairs(
-    hydrofabric: Path = file_paths.conus_hydrofabric, ids: Set = None
+    hydrofabric: Path = file_paths.conus_hydrofabric, ids: Optional[Set | List] = None
 ) -> List[tuple]:
     """
     Retrieves the from and to IDs from the specified hydrofabric.
@@ -112,7 +112,7 @@ def get_graph() -> ig.Graph:
     return network_graph
 
 
-def get_outlet_id(wb_or_cat_id: str) -> str:
+def get_outlet_id(wb_or_cat_id: str) -> str | None:
     """
     Retrieves the ID of the node downstream of the given node in the hydrological network.
 
@@ -209,7 +209,7 @@ def get_upstream_ids(names: Union[str, List[str]], include_outlet: bool = True)
         if name in parent_ids:
             continue
         try:
-            if "cat" in name:
+            if "cat" in name:  # type: ignore # If name is None, this will raise an error, which is handled below
                 node_index = graph.vs.find(cat=name).index
             else:
                 node_index = graph.vs.find(name=name).index

data_processing/s3fs_utils.py

@@ -32,7 +32,7 @@ class S3ParallelFileSystem(S3FileSystem):
                     "head_object", Bucket=bucket, Key=key, **version_kw, **self.req_kw
                 )
             )["ContentLength"]
-        except Exception as e:
+        except Exception:
             # Fall back to single request if HEAD fails
             return await self._download_chunk(bucket, key, {}, version_kw)
 

data_processing/subset.py

@@ -64,12 +64,11 @@ def subset_vpu(
 
 
 def subset(
-    cat_ids: List[str],
+    cat_ids: str | List[str],
     hydrofabric: Path = file_paths.conus_hydrofabric,
     output_gpkg_path: Path = Path(),
     include_outlet: bool = True,
 ):
-    print(cat_ids)
     upstream_ids = list(get_upstream_ids(cat_ids, include_outlet))
 
     if not output_gpkg_path:

data_sources/source_validation.py

@@ -1,19 +1,29 @@
 import gzip
+import json
 import os
 import tarfile
 import warnings
-import json
-import requests
-from data_processing.file_paths import file_paths
-from tqdm import TqdmExperimentalWarning
 from time import sleep
+
 import boto3
-from botocore.exceptions import ClientError
+import psutil
+import requests
 from boto3.s3.transfer import TransferConfig
+from botocore.exceptions import ClientError
+import botocore
+from data_processing.file_paths import file_paths
 from rich.console import Console
+from rich.progress import (Progress, 
+                           SpinnerColumn, 
+                           TextColumn, 
+                           TimeElapsedColumn, 
+                           BarColumn, 
+                           DownloadColumn, 
+                           TransferSpeedColumn)
 from rich.prompt import Prompt
-from rich.progress import Progress, TextColumn, TimeElapsedColumn, SpinnerColumn
-import psutil
+from tqdm import TqdmExperimentalWarning
+from data_processing.gpkg_utils import verify_indices
+import sqlite3
 
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 
@@ -23,25 +33,22 @@ S3_KEY = "hydrofabrics/community/conus_nextgen.tar.gz"
 S3_REGION = "us-east-1"
 hydrofabric_url = f"https://{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com/{S3_KEY}"
 
+
 def decompress_gzip_tar(file_path, output_dir):
-    # use rich to display "decompressing" message with a progress bar that just counts down from 30s
-    # actually measuring this is hard and it usually takes ~20s to decompress
     console.print("Decompressing Hydrofabric...", style="bold green")
     progress = Progress(
         SpinnerColumn(),
         TextColumn("[progress.description]{task.description}"),
-        TimeElapsedColumn(),        
+        TimeElapsedColumn(),
     )
     task = progress.add_task("Decompressing", total=1)
-    progress.start()
-    with gzip.open(file_path, "rb") as f_in:
-        with tarfile.open(fileobj=f_in) as tar:
-            # Extract all contents
-            for member in tar:
-                tar.extract(member, path=output_dir)
-                # Update the progress bar
-    progress.update(task, completed=1)
-    progress.stop()
+    with progress:
+        with gzip.open(file_path, "rb") as f_in:
+            with tarfile.open(fileobj=f_in) as tar:
+                # Extract all contents
+                for member in tar:
+                    tar.extract(member, path=output_dir)
+                    progress.update(task, advance=1 / len(tar.getmembers()))
 
 
 def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
@@ -53,10 +60,13 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
     if os.path.exists(save_path):
         console.print(f"File already exists: {save_path}", style="bold yellow")
         os.remove(save_path)
-
+    
+    client_config = botocore.config.Config(
+        max_pool_connections=75
+    )
     # Initialize S3 client
     s3_client = boto3.client(
-        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region
+        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region, config=client_config
     )
     # Disable request signing for public buckets
     s3_client._request_signer.sign = lambda *args, **kwargs: None
@@ -92,19 +102,15 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
         use_threads=True,
     )
 
-    console.print(f"Downloading {key} to {save_path}...", style="bold green")
-    console.print(
-        f"The file downloads faster with no progress indicator, this should take around 30s",
-        style="bold yellow",
-    )
-    console.print(
-        f"Please use network monitoring on your computer if you wish to track the download",
-        style="green",
-    )
 
     try:
+        dl_progress = Progress(BarColumn(), DownloadColumn(), TransferSpeedColumn())
         # Download file using optimized transfer config
-        s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config)
+        with dl_progress:
+            task = dl_progress.add_task("Downloading...", total=total_size)
+            s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config,
+                                    Callback=lambda bytes_downloaded: dl_progress.update(
+                                        task, advance=bytes_downloaded))
         return True
     except Exception as e:
         console.print(f"Error downloading file: {e}", style="bold red")
@@ -122,6 +128,14 @@ def get_headers():
 
 
 def download_and_update_hf():
+
+    if file_paths.conus_hydrofabric.is_file():
+        console.print(
+            f"Hydrofabric already exists at {file_paths.conus_hydrofabric}, removing it to download the latest version.",
+            style="bold yellow",
+        )
+        file_paths.conus_hydrofabric.unlink()
+        
     download_from_s3(
         file_paths.conus_hydrofabric.with_suffix(".tar.gz"),
         bucket="communityhydrofabric",
@@ -207,6 +221,17 @@ def validate_hydrofabric():
             )
             sleep(2)
             return
+    
+    # moved this from gpkg_utils to here to avoid potential nested rich live displays
+    if file_paths.conus_hydrofabric.is_file():
+        valid_hf = False
+        while not valid_hf:
+            try:
+                verify_indices()
+                valid_hf = True
+            except sqlite3.DatabaseError:
+                console.print(f"Hydrofabric {file_paths.conus_hydrofabric} is corrupted. Redownloading...", style="red")
+                download_and_update_hf()
 
 
 def validate_output_dir():
@@ -220,7 +245,7 @@ def validate_output_dir():
             response = Prompt.ask("Enter the path to the working directory")
         if response == "" or response.lower() == "y":
             response = "~/ngiab_preprocess_output/"
-        file_paths.set_working_dir(response)
+        file_paths.set_working_dir(response)  # type: ignore
 
 
 def validate_all():

map_app/__main__.py

@@ -39,12 +39,13 @@ def main():
         Timer(2, set_logs_to_warning).start()
         with open("app.log", "a") as f:
             f.write("Running in debug mode\n")
-        app.run(debug=True, host="0.0.0.0", port="8080")
+        app.run(debug=True, host="0.0.0.0", port="8080")  # type: ignore
     else:
         Timer(1, open_browser).start()
         with open("app.log", "a") as f:
             f.write("Running in production mode\n")
-        app.run(host="0.0.0.0", port="0")
+        app.run(host="0.0.0.0", port="0")  # type: ignore
+
 
 if __name__ == "__main__":
     main()

map_app/static/css/main.css

@@ -28,10 +28,9 @@ main {
 
 .maplibregl-popup-content {
   background: var(--surface-color) !important;
-
 }
 
-#toggle-button {
+#toggle-button-gages, #toggle-button-camels, #toggle-button-nwm, #toggle-button-aorc  {
   position: relative;
   top: 20px;
   left: 20px;
@@ -46,11 +45,23 @@ main {
   z-index: 1;
 }
 
-#toggle-button:hover {
+#toggle-button-gages:hover, #toggle-button-camels:hover, #toggle-button-nwm:hover, #toggle-button-aorc:hover {
   scale: 1.1;
   box-shadow: var(--shadow-md);
 }
 
+#toggle-button-camels {
+  left: 30px;
+}
+
+#toggle-button-nwm {
+  left: 40px;
+}
+
+#toggle-button-aorc {
+  left: 50px;
+}
+
 body {
   font-family: 'Inter', system-ui, -apple-system, sans-serif;
   margin: 0;
@@ -109,7 +120,7 @@ h2 {
 }
 
 #selected-basins,
-#cli-command {
+#cli-command,#cli-prefix {
   background: var(--code-bg);
   padding: 16px;
   border-radius: var(--border-radius);
@@ -119,6 +130,7 @@ h2 {
   color: var(--text-color);
 }
 
+
 button {
   background-color: var(--primary-color);
   color: light-dark(white, #f1f5f9);
@@ -204,6 +216,11 @@ input[type="datetime-local"] {
   display: inline-block;
 }
 
+#command-builder{
+    display: inline-block;
+    padding:16px ;
+}
+
 .command-container {
   background: var(--surface-color);
   border: 1px solid var(--border-color);
@@ -222,7 +239,7 @@ input[type="datetime-local"] {
   border-top-right-radius: var(--border-radius);
 }
 
-.command-header span {
+.command-header>span {
   font-size: 0.875rem;
   color: var(--secondary-text);
   font-weight: 500;
@@ -254,7 +271,8 @@ input[type="datetime-local"] {
 }
 
 .command-content {
-  padding: 16px;
+    display:inline;
+  padding: 0px !important;
   background: var(--code-bg);
   font-family: 'Monaco', 'Consolas', monospace;
   font-size: 0.875rem;
@@ -263,6 +281,11 @@ input[type="datetime-local"] {
   border-bottom-left-radius: var(--border-radius);
   border-bottom-right-radius: var(--border-radius);
   color: var(--text-color);
+
+}
+
+#cli-prefix{
+    opacity: 0;
 }
 
 .copy-button.copied {
@@ -281,17 +304,17 @@ input[type="datetime-local"] {
   body {
       padding: 16px;
   }
-  
+
   main {
       width: 90vw;
   }
-  
+
   .time-input {
       flex-direction: column;
       align-items: flex-start;
   }
-  
+
   input[type="datetime-local"] {
       width: 100%;
   }
-}
+}

map_app/static/css/toggle.css

@@ -34,24 +34,27 @@
     background-color: white;
     border-radius: 18px;
     box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
-    transition: transform 0.3s ease, box-shadow 0.3s ease;
+    transition:
+        transform 0.3s ease,
+        box-shadow 0.3s ease;
     display: flex;
     justify-content: center;
     align-items: center;
     font-size: 14px;
     font-weight: bold;
-    color: #4CAF50; /* Default color for the selected text */
+    color: #4caf50; /* Default color for the selected text */
 }
 
 /* Toggle Text (NWM and AORC labels) */
 .toggle-text {
     position: absolute;
     top: 50%;
+    min-width: 40px;
+    text-align: center;
     transform: translateY(-50%);
     font-size: 14px;
     font-weight: bold;
     color: #888; /* Grey color for non-selected text */
-    transition: color 0.3s ease;
 }
 
 .toggle-text-left {
@@ -70,7 +73,7 @@
 .toggle-input:checked + .toggle-label .toggle-handle {
     transform: translateX(56px);
     box-shadow: 0 0 10px rgba(0, 123, 255, 0.8); /* Blue glow effect */
-    color: #007BFF; /* Blue color for the selected text */
+    color: #007bff; /* Blue color for the selected text */
 }
 
 .toggle-input:checked + .toggle-label .toggle-text-left {
@@ -79,4 +82,4 @@
 
 .toggle-input:checked + .toggle-label .toggle-text-right {
     color: #888; /* Grey color for non-selected text */
-}
+}