ngiab-data-preprocess 4.2.2__py3-none-any.whl → 4.4.0__py3-none-any.whl

data_sources/source_validation.py

@@ -1,19 +1,29 @@
 import gzip
+import json
 import os
 import tarfile
 import warnings
-import json
-import requests
-from data_processing.file_paths import file_paths
-from tqdm import TqdmExperimentalWarning
 from time import sleep
+
 import boto3
-from botocore.exceptions import ClientError
+import psutil
+import requests
 from boto3.s3.transfer import TransferConfig
+from botocore.exceptions import ClientError
+import botocore
+from data_processing.file_paths import file_paths
 from rich.console import Console
+from rich.progress import (Progress, 
+                           SpinnerColumn, 
+                           TextColumn, 
+                           TimeElapsedColumn, 
+                           BarColumn, 
+                           DownloadColumn, 
+                           TransferSpeedColumn)
 from rich.prompt import Prompt
-from rich.progress import Progress, TextColumn, TimeElapsedColumn, SpinnerColumn
-import psutil
+from tqdm import TqdmExperimentalWarning
+from data_processing.gpkg_utils import verify_indices
+import sqlite3
 
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 
@@ -23,25 +33,22 @@ S3_KEY = "hydrofabrics/community/conus_nextgen.tar.gz"
 S3_REGION = "us-east-1"
 hydrofabric_url = f"https://{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com/{S3_KEY}"
 
+
 def decompress_gzip_tar(file_path, output_dir):
-    # use rich to display "decompressing" message with a progress bar that just counts down from 30s
-    # actually measuring this is hard and it usually takes ~20s to decompress
     console.print("Decompressing Hydrofabric...", style="bold green")
     progress = Progress(
         SpinnerColumn(),
         TextColumn("[progress.description]{task.description}"),
-        TimeElapsedColumn(),        
+        TimeElapsedColumn(),
     )
     task = progress.add_task("Decompressing", total=1)
-    progress.start()
-    with gzip.open(file_path, "rb") as f_in:
-        with tarfile.open(fileobj=f_in) as tar:
-            # Extract all contents
-            for member in tar:
-                tar.extract(member, path=output_dir)
-                # Update the progress bar
-    progress.update(task, completed=1)
-    progress.stop()
+    with progress:
+        with gzip.open(file_path, "rb") as f_in:
+            with tarfile.open(fileobj=f_in) as tar:
+                # Extract all contents
+                for member in tar:
+                    tar.extract(member, path=output_dir)
+                    progress.update(task, advance=1 / len(tar.getmembers()))
 
 
 def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
@@ -53,10 +60,13 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
     if os.path.exists(save_path):
         console.print(f"File already exists: {save_path}", style="bold yellow")
         os.remove(save_path)
-
+    
+    client_config = botocore.config.Config(
+        max_pool_connections=75
+    )
     # Initialize S3 client
     s3_client = boto3.client(
-        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region
+        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region, config=client_config
     )
     # Disable request signing for public buckets
     s3_client._request_signer.sign = lambda *args, **kwargs: None
@@ -92,19 +102,15 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
         use_threads=True,
     )
 
-    console.print(f"Downloading {key} to {save_path}...", style="bold green")
-    console.print(
-        f"The file downloads faster with no progress indicator, this should take around 30s",
-        style="bold yellow",
-    )
-    console.print(
-        f"Please use network monitoring on your computer if you wish to track the download",
-        style="green",
-    )
 
     try:
+        dl_progress = Progress(BarColumn(), DownloadColumn(), TransferSpeedColumn())
         # Download file using optimized transfer config
-        s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config)
+        with dl_progress:
+            task = dl_progress.add_task("Downloading...", total=total_size)
+            s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config,
+                                    Callback=lambda bytes_downloaded: dl_progress.update(
+                                        task, advance=bytes_downloaded))
         return True
     except Exception as e:
         console.print(f"Error downloading file: {e}", style="bold red")
@@ -122,11 +128,33 @@ def get_headers():
 
 
 def download_and_update_hf():
+
+    if file_paths.conus_hydrofabric.is_file():
+        console.print(
+            f"Hydrofabric already exists at {file_paths.conus_hydrofabric}, removing it to download the latest version.",
+            style="bold yellow",
+        )
+        file_paths.conus_hydrofabric.unlink()
+        
     download_from_s3(
         file_paths.conus_hydrofabric.with_suffix(".tar.gz"),
         bucket="communityhydrofabric",
         key="hydrofabrics/community/conus_nextgen.tar.gz",
     )
+
+    if file_paths.hydrofabric_graph.is_file():
+        console.print(
+            f"Hydrofabric graph already exists at {file_paths.hydrofabric_graph}, removing it to download the latest version.",
+            style="bold yellow",
+        )
+        file_paths.hydrofabric_graph.unlink()
+
+    download_from_s3(
+        file_paths.hydrofabric_graph,
+        bucket="communityhydrofabric",
+        key="hydrofabrics/community/conus_igraph_network.gpickle"
+    )
+
     status, headers = get_headers()
 
     if status == 200:
@@ -139,11 +167,10 @@ def download_and_update_hf():
         file_paths.conus_hydrofabric.parent,
     )
 
-
 def validate_hydrofabric():
     if not file_paths.conus_hydrofabric.is_file():
         response = Prompt.ask(
-            "Hydrofabric is missing. Would you like to download it now?",
+            "Hydrofabric files are missing. Would you like to download them now?",
             default="y",
             choices=["y", "n"],
         )
@@ -207,6 +234,17 @@ def validate_hydrofabric():
             )
             sleep(2)
             return
+    
+    # moved this from gpkg_utils to here to avoid potential nested rich live displays
+    if file_paths.conus_hydrofabric.is_file():
+        valid_hf = False
+        while not valid_hf:
+            try:
+                verify_indices()
+                valid_hf = True
+            except sqlite3.DatabaseError:
+                console.print(f"Hydrofabric {file_paths.conus_hydrofabric} is corrupted. Redownloading...", style="red")
+                download_and_update_hf()
 
 
 def validate_output_dir():
@@ -220,7 +258,7 @@ def validate_output_dir():
             response = Prompt.ask("Enter the path to the working directory")
         if response == "" or response.lower() == "y":
             response = "~/ngiab_preprocess_output/"
-        file_paths.set_working_dir(response)
+        file_paths.set_working_dir(response)  # type: ignore
 
 
 def validate_all():

map_app/__main__.py

@@ -39,12 +39,13 @@ def main():
         Timer(2, set_logs_to_warning).start()
         with open("app.log", "a") as f:
             f.write("Running in debug mode\n")
-        app.run(debug=True, host="0.0.0.0", port="8080")
+        app.run(debug=True, host="0.0.0.0", port="8080")  # type: ignore
     else:
         Timer(1, open_browser).start()
         with open("app.log", "a") as f:
             f.write("Running in production mode\n")
-        app.run(host="0.0.0.0", port="0")
+        app.run(host="0.0.0.0", port="0")  # type: ignore
+
 
 if __name__ == "__main__":
     main()

map_app/static/css/main.css

@@ -28,10 +28,9 @@ main {
 
 .maplibregl-popup-content {
   background: var(--surface-color) !important;
-
 }
 
-#toggle-button {
+#toggle-button-gages, #toggle-button-camels, #toggle-button-nwm, #toggle-button-aorc  {
   position: relative;
   top: 20px;
   left: 20px;
@@ -46,11 +45,23 @@ main {
   z-index: 1;
 }
 
-#toggle-button:hover {
+#toggle-button-gages:hover, #toggle-button-camels:hover, #toggle-button-nwm:hover, #toggle-button-aorc:hover {
   scale: 1.1;
   box-shadow: var(--shadow-md);
 }
 
+#toggle-button-camels {
+  left: 30px;
+}
+
+#toggle-button-nwm {
+  left: 40px;
+}
+
+#toggle-button-aorc {
+  left: 50px;
+}
+
 body {
   font-family: 'Inter', system-ui, -apple-system, sans-serif;
   margin: 0;

map_app/static/js/data_processing.js

@@ -4,67 +4,44 @@ async function subset() {
         alert('Please select at least one basin in the map before subsetting');
         return;
     }
-    console.log('subsetting');
-    document.getElementById('subset-button').disabled = true;
-    document.getElementById('subset-loading').style.visibility = "visible";
-    const startTime = performance.now(); // Start the timer
-    document.getElementById('output-path').innerHTML = "Subsetting...";
-    fetch('/subset', {
+    fetch('/subset_check', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify([cat_id]),
     })
-        .then(response => response.text())
-        .then(filename => {
-            console.log(filename);
-            const endTime = performance.now(); // Stop the timer
-            const duration = endTime - startTime; // Calculate the duration in milliseconds
-            console.log('Request took ' + duration / 1000 + ' milliseconds');
-            document.getElementById('output-path').innerHTML = "Done in " + duration / 1000 + "s, subset to <a href='file://" + filename + "'>" + filename + "</a>";
-        })
-        .catch(error => {
-            console.error('Error:', error);
-        }).finally(() => {
-            document.getElementById('subset-button').disabled = false;
-            document.getElementById('subset-loading').style.visibility = "hidden";
-        });
+    .then((response) => {
+    // 409 if that subset gpkg path already exists
+        if (response.status == 409) {
+            console.log("check response")
+            if (!confirm('A geopackage already exists with that catchment name. Overwrite?')) {
+                alert("Subset canceled.");
+                return;
+            }
+        } 
+        const startTime = performance.now(); // Start the timer
+        fetch('/subset', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify([cat_id]),
+            })
+                .then(response => response.text())
+                .then(filename => {
+                    console.log(filename);
+                    const endTime = performance.now(); // Stop the timer
+                    const duration = endTime - startTime; // Calculate the duration in milliseconds
+                    console.log('Request took ' + duration / 1000 + ' milliseconds');
+                    document.getElementById('output-path').innerHTML = "Done in " + (duration / 1000).toFixed(2) + "s, subset to <a href='file://" + filename + "'>" + filename + "</a>";
+                })
+                .catch(error => {
+                    console.error('Error:', error);
+                }).finally(() => {
+                    document.getElementById('subset-button').disabled = false;
+                    document.getElementById('subset-loading').style.visibility = "hidden";
+                });
+    });
 }
 
-
-// async function subset_to_file() {
-//     if (Object.keys(cat_id_dict).length === 0) {
-//         alert('Please select at least one basin in the map before subsetting');
-//         return;
-//     }
-//     console.log('subsetting to file');
-//     document.getElementById('subset-to-file-button').disabled = true;
-//     document.getElementById('subset-to-file-loading').style.visibility = "visible";
-//     const startTime = performance.now(); // Start the timer
-//     document.getElementById('output-path').innerHTML = "Subsetting...";
-//     fetch('/subset_to_file', {
-//         method: 'POST',
-//         headers: { 'Content-Type': 'application/json' },
-//         body: JSON.stringify(cat_id_dict),
-//     })
-//         .then(response => response.text())
-//         .then(filename => {
-//             console.log(filename);
-//             const endTime = performance.now(); // Stop the timer
-//             const duration = endTime - startTime; // Calculate the duration in milliseconds
-//             console.log('Request took ' + duration / 1000 + ' milliseconds');
-//             document.getElementById('output-path').innerHTML = "Done in " + duration / 1000 + "s, subset to <a href='file://" + filename + "'>" + filename + "</a>";
-//         })
-//         .catch(error => {
-//             console.error('Error:', error);
-//         }).finally(() => {
-//             document.getElementById('subset-to-file-button').disabled = false;
-//             document.getElementById('subset-to-file-loading').style.visibility = "hidden";
-//         });
-// }
-
 async function forcings() {
-
-
     if (document.getElementById('output-path').textContent === '') {
         alert('Please subset the data before getting forcings');
         return;
@@ -139,6 +116,5 @@ async function realization() {
 
 // These functions are exported by data_processing.js
 document.getElementById('subset-button').addEventListener('click', subset);
-// document.getElementById('subset-to-file-button').addEventListener('click', subset_to_file);
 document.getElementById('forcings-button').addEventListener('click', forcings);
 document.getElementById('realization-button').addEventListener('click', realization);