ngiab-data-preprocess 4.5.1__py3-none-any.whl → 4.6.0__py3-none-any.whl

data_sources/source_validation.py

@@ -1,29 +1,31 @@
 import gzip
 import json
 import os
+import sqlite3
 import tarfile
 import warnings
 from time import sleep
 
 import boto3
+import botocore
 import psutil
 import requests
 from boto3.s3.transfer import TransferConfig
 from botocore.exceptions import ClientError
-import botocore
-from data_processing.file_paths import file_paths
+from data_processing.file_paths import FilePaths
+from data_processing.gpkg_utils import verify_indices
 from rich.console import Console
-from rich.progress import (Progress, 
-                           SpinnerColumn, 
-                           TextColumn, 
-                           TimeElapsedColumn, 
-                           BarColumn, 
-                           DownloadColumn, 
-                           TransferSpeedColumn)
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TransferSpeedColumn,
+)
 from rich.prompt import Prompt
 from tqdm import TqdmExperimentalWarning
-from data_processing.gpkg_utils import verify_indices
-import sqlite3
 
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 
@@ -60,13 +62,15 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
     if os.path.exists(save_path):
         console.print(f"File already exists: {save_path}", style="bold yellow")
         os.remove(save_path)
-    
-    client_config = botocore.config.Config(
-        max_pool_connections=75
-    )
+
+    client_config = botocore.config.Config(max_pool_connections=75)
     # Initialize S3 client
     s3_client = boto3.client(
-        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region, config=client_config
+        "s3",
+        aws_access_key_id="",
+        aws_secret_access_key="",
+        region_name=region,
+        config=client_config,
     )
     # Disable request signing for public buckets
     s3_client._request_signer.sign = lambda *args, **kwargs: None
@@ -102,15 +106,20 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
         use_threads=True,
     )
 
-
     try:
         dl_progress = Progress(BarColumn(), DownloadColumn(), TransferSpeedColumn())
         # Download file using optimized transfer config
         with dl_progress:
             task = dl_progress.add_task("Downloading...", total=total_size)
-            s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config,
-                                    Callback=lambda bytes_downloaded: dl_progress.update(
-                                        task, advance=bytes_downloaded))
+            s3_client.download_file(
+                Bucket=bucket,
+                Key=key,
+                Filename=save_path,
+                Config=config,
+                Callback=lambda bytes_downloaded: dl_progress.update(
+                    task, advance=bytes_downloaded
+                ),
+            )
         return True
     except Exception as e:
         console.print(f"Error downloading file: {e}", style="bold red")
@@ -128,47 +137,47 @@ def get_headers():
 
 
 def download_and_update_hf():
-
-    if file_paths.conus_hydrofabric.is_file():
+    if FilePaths.conus_hydrofabric.is_file():
         console.print(
-            f"Hydrofabric already exists at {file_paths.conus_hydrofabric}, removing it to download the latest version.",
+            f"Hydrofabric already exists at {FilePaths.conus_hydrofabric}, removing it to download the latest version.",
             style="bold yellow",
         )
-        file_paths.conus_hydrofabric.unlink()
-        
+        FilePaths.conus_hydrofabric.unlink()
+
     download_from_s3(
-        file_paths.conus_hydrofabric.with_suffix(".tar.gz"),
+        FilePaths.conus_hydrofabric.with_suffix(".tar.gz"),
         bucket="communityhydrofabric",
         key="hydrofabrics/community/conus_nextgen.tar.gz",
     )
 
-    if file_paths.hydrofabric_graph.is_file():
+    if FilePaths.hydrofabric_graph.is_file():
         console.print(
-            f"Hydrofabric graph already exists at {file_paths.hydrofabric_graph}, removing it to download the latest version.",
+            f"Hydrofabric graph already exists at {FilePaths.hydrofabric_graph}, removing it to download the latest version.",
             style="bold yellow",
         )
-        file_paths.hydrofabric_graph.unlink()
+        FilePaths.hydrofabric_graph.unlink()
 
     download_from_s3(
-        file_paths.hydrofabric_graph,
+        FilePaths.hydrofabric_graph,
         bucket="communityhydrofabric",
-        key="hydrofabrics/community/conus_igraph_network.gpickle"
+        key="hydrofabrics/community/conus_igraph_network.gpickle",
     )
 
     status, headers = get_headers()
 
     if status == 200:
         # write headers to a file
-        with open(file_paths.hydrofabric_download_log, "w") as f:
+        with open(FilePaths.hydrofabric_download_log, "w") as f:
             json.dump(dict(headers), f)
 
     decompress_gzip_tar(
-        file_paths.conus_hydrofabric.with_suffix(".tar.gz"),
-        file_paths.conus_hydrofabric.parent,
+        FilePaths.conus_hydrofabric.with_suffix(".tar.gz"),
+        FilePaths.conus_hydrofabric.parent,
     )
 
+
 def validate_hydrofabric():
-    if not file_paths.conus_hydrofabric.is_file():
+    if not FilePaths.conus_hydrofabric.is_file():
         response = Prompt.ask(
             "Hydrofabric files are missing. Would you like to download them now?",
             default="y",
@@ -180,11 +189,11 @@ def validate_hydrofabric():
             console.print("Exiting...", style="bold red")
             exit()
 
-    if file_paths.no_update_hf.exists():
+    if FilePaths.no_update_hf.exists():
         # skip the updates
         return
 
-    if not file_paths.hydrofabric_download_log.is_file():
+    if not FilePaths.hydrofabric_download_log.is_file():
         response = Prompt.ask(
             "Hydrofabric version information unavailable, Would you like to fetch the updated version?",
             default="y",
@@ -195,13 +204,13 @@ def validate_hydrofabric():
         else:
             console.print("Continuing... ", style="bold yellow")
             console.print(
-                f"To disable this warning, create an empty file called {file_paths.no_update_hf.resolve()}",
+                f"To disable this warning, create an empty file called {FilePaths.no_update_hf.resolve()}",
                 style="bold yellow",
             )
             sleep(2)
             return
 
-    with open(file_paths.hydrofabric_download_log, "r") as f:
+    with open(FilePaths.hydrofabric_download_log, "r") as f:
         content = f.read()
         headers = json.loads(content)
 
@@ -229,26 +238,29 @@ def validate_hydrofabric():
         else:
             console.print("Continuing... ", style="bold yellow")
             console.print(
-                f"To disable this warning, create an empty file called {file_paths.no_update_hf.resolve()}",
+                f"To disable this warning, create an empty file called {FilePaths.no_update_hf.resolve()}",
                 style="bold yellow",
             )
             sleep(2)
             return
-    
+
     # moved this from gpkg_utils to here to avoid potential nested rich live displays
-    if file_paths.conus_hydrofabric.is_file():
+    if FilePaths.conus_hydrofabric.is_file():
         valid_hf = False
         while not valid_hf:
             try:
                 verify_indices()
                 valid_hf = True
             except sqlite3.DatabaseError:
-                console.print(f"Hydrofabric {file_paths.conus_hydrofabric} is corrupted. Redownloading...", style="red")
+                console.print(
+                    f"Hydrofabric {FilePaths.conus_hydrofabric} is corrupted. Redownloading...",
+                    style="red",
+                )
                 download_and_update_hf()
 
 
 def validate_output_dir():
-    if not file_paths.config_file.is_file():
+    if not FilePaths.config_file.is_file():
         response = Prompt.ask(
             "Output directory is not set. Would you like to use the default? ~/ngiab_preprocess_output/",
             default="y",
@@ -258,7 +270,7 @@ def validate_output_dir():
             response = Prompt.ask("Enter the path to the working directory")
         if response == "" or response.lower() == "y":
             response = "~/ngiab_preprocess_output/"
-        file_paths.set_working_dir(response)  # type: ignore
+        FilePaths.set_working_dir(response)  # type: ignore
 
 
 def validate_all():

map_app/__main__.py

@@ -4,8 +4,9 @@ import logging
 import webbrowser
 from threading import Timer
 
-from data_processing.file_paths import file_paths
+from data_processing.file_paths import FilePaths
 from data_processing.graph_utils import get_graph
+
 from map_app import app, console_handler
 
 
@@ -35,7 +36,7 @@ def main():
     # call this once to cache the graph
     Timer(1, get_graph).start()
 
-    if file_paths.dev_file.is_file():
+    if FilePaths.dev_file.is_file():
         Timer(2, set_logs_to_warning).start()
         with open("app.log", "a") as f:
             f.write("Running in debug mode\n")

map_app/static/css/main.css

@@ -318,3 +318,44 @@ input[type="datetime-local"] {
       width: 100%;
   }
 }
+
+#progress {
+  background-color: black;
+  border-radius: 13px;
+  height: 20px;
+  padding: 3px;
+  position: relative;
+  display: grid;
+  align-items: center;
+  overflow: hidden;
+}
+
+#bar {
+  background-color: rgb(49,96,153);
+  width: 100%;
+  height: 20px;
+  border-radius: 10px;
+  position: absolute;
+  animation: none;
+  transform-origin: 0% 50%;
+}
+
+
+@keyframes indeterminateAnimation {
+  0% {
+    transform:  translateX(0) scaleX(0);
+  }
+  40% {
+    transform:  translateX(0) scaleX(0.4);
+  }
+  100% {
+    transform:  translateX(100%) scaleX(0.5);
+  }
+}
+
+#bar-text {
+  position: absolute;
+  margin: auto;
+  width: 100%;
+  text-align: center;
+}

map_app/static/css/toggle.css

@@ -79,7 +79,36 @@
 .toggle-input:checked + .toggle-label .toggle-text-left {
     color: #888; /* Grey color for non-selected text */
 }
-
+  
 .toggle-input:checked + .toggle-label .toggle-text-right {
     color: #888; /* Grey color for non-selected text */
 }
+
+/* toggle switch for the subset option */
+.toggle-switch-subset {
+    position: absolute;
+    bottom: 20px;
+    left: 20px;
+    z-index: 10;
+    width: 240px;
+}
+
+.toggle-text-subset {
+    font-size: 10px;
+}
+
+/* I didn't mean to have four styles for one element but it looked terrible without this*/
+.toggle-text-left-subset {
+    left: 20px;
+}
+
+.toggle-handle-subset {
+    width: 120px;
+    font-size: 10px;
+}
+
+.toggle-input:checked + .toggle-label .toggle-handle-subset {
+    transform: translateX(116px);
+    box-shadow: 0 0 10px rgba(0, 123, 255, 0.8); /* Blue glow effect */
+    color: #007bff; /* Blue color for the selected text */
+}

map_app/static/js/data_processing.js

@@ -9,20 +9,29 @@ async function subset() {
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify([cat_id]),
     })
-    .then((response) => {
+    .then(async response => {
     // 409 if that subset gpkg path already exists
         if (response.status == 409) {
+            const filename = await response.text();
             console.log("check response")
             if (!confirm('A geopackage already exists with that catchment name. Overwrite?')) {
                 alert("Subset canceled.");
+                document.getElementById('output-path').innerHTML = "Subset canceled. Geopackage located at " + filename;
                 return;
             }
         } 
+        // check what kind of subset
+        // get the position of the subset toggle
+        // false means subset by nexus, true means subset by catchment
+        var nexus_catchment = document.getElementById('subset-toggle').checked;
+        var subset_type = nexus_catchment ? 'catchment' : 'nexus';
+
         const startTime = performance.now(); // Start the timer
         fetch('/subset', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify([cat_id]),
+                // body: JSON.stringify([cat_id]),
+                body: JSON.stringify({ 'cat_id': [cat_id], 'subset_type': subset_type}),
             })
                 .then(response => response.text())
                 .then(filename => {
@@ -41,46 +50,101 @@ async function subset() {
     });
 }
 
-async function forcings() {
-    if (document.getElementById('output-path').textContent === '') {
-        alert('Please subset the data before getting forcings');
-        return;
-    }
-    console.log('getting forcings');
-    document.getElementById('forcings-button').disabled = true;
-    document.getElementById('forcings-loading').style.visibility = "visible";
-
-    const forcing_dir = document.getElementById('output-path').textContent;
-    const start_time = document.getElementById('start-time').value;
-    const end_time = document.getElementById('end-time').value;
-    if (forcing_dir === '' || start_time === '' || end_time === '') {
-        alert('Please enter a valid output path, start time, and end time');
-        document.getElementById('time-warning').style.color = 'red';
-        return;
-    }
+function updateProgressBar(percent) {
+    var bar = document.getElementById("bar");
+    bar.style.width = percent + "%";
+    var barText = document.getElementById("bar-text");
+    barText.textContent = percent + "%";
+}
 
-    // get the position of the nwm aorc forcing toggle
-    // false means nwm forcing, true means aorc forcing
-    var nwm_aorc = document.getElementById('datasource-toggle').checked;
-    var source = nwm_aorc ? 'aorc' : 'nwm';
-    console.log('source:', source);
+function pollForcingsProgress(progressFile) {
+    const interval = setInterval(() => {
+        fetch('/forcings_progress', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(progressFile),
+        })
+            .then(response => response.text())
+            .then(data => {
+                if (data == "NaN") {
+                    document.getElementById('forcings-output-path').textContent = "Downloading data...";
+                    document.getElementById('bar-text').textContent = "Downloading...";
+                    document.getElementById('bar').style.animation = "indeterminateAnimation 1s infinite linear";
+                } else {
+                    const percent = parseInt(data, 10);
+                    updateProgressBar(percent);
+                    if (percent > 0 && percent < 100) {
+                        document.getElementById('bar').style.animation = "none"; // stop the indeterminate animation
+                        document.getElementById('forcings-output-path').textContent = "Calculating zonal statistics. See progress below.";
+                    } else if (percent >= 100) {
+                        updateProgressBar(100); // Ensure the progress bar is full
+                        clearInterval(interval);
+                        document.getElementById('forcings-output-path').textContent = "Forcings generated successfully";
+                    }
+                }
+            })
+            .catch(error => {
+                console.error('Progress polling error:', error);
+                clearInterval(interval);
+            });
+    }, 1000); // Poll every second
+}
 
-    document.getElementById('forcings-output-path').textContent = "Generating forcings...";
-    fetch('/forcings', {
+async function forcings() {
+    fetch('/subset_check', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ 'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time , 'source': source}),
-    }).then(response => response.text())
-        .then(response_code => {
-            document.getElementById('forcings-output-path').textContent = "Forcings generated successfully";
-        })
-        .catch(error => {
-            console.error('Error:', error);
-        }).finally(() => {
-            document.getElementById('forcings-button').disabled = false;
-            document.getElementById('forcings-loading').style.visibility = "hidden";
+        body: JSON.stringify([cat_id]),
+    })
+    .then(async response => {
+        // 409 if that subset gpkg path already exists
+        if (response.status == 409) {
+            const filename = await response.text();
+            console.log('getting forcings');
+            document.getElementById('forcings-button').disabled = true;
+            document.getElementById('forcings-loading').style.visibility = "visible";
 
-        });
+            const forcing_dir = filename;
+            console.log('forcing_dir:', forcing_dir);
+            const start_time = document.getElementById('start-time').value;
+            const end_time = document.getElementById('end-time').value;
+            if (forcing_dir === '' || start_time === '' || end_time === '') {
+                alert('Please enter a valid output path, start time, and end time');
+                document.getElementById('time-warning').style.color = 'red';
+                return;
+            }
+
+            // get the position of the nwm aorc forcing toggle
+            // false means nwm forcing, true means aorc forcing
+            var nwm_aorc = document.getElementById('datasource-toggle').checked;
+            var source = nwm_aorc ? 'aorc' : 'nwm';
+            console.log('source:', source);
+
+            fetch('/make_forcings_progress_file', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(forcing_dir),
+            })
+            .then(async (response) => response.text())
+            .then(progressFile => { 
+                pollForcingsProgress(progressFile); // Start polling for progress
+            })
+            fetch('/forcings', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ 'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time , 'source': source}),
+            })
+            .then(response => response.text())
+            .catch(error => {
+                console.error('Error:', error);
+            }).finally(() => {
+                document.getElementById('forcings-button').disabled = false;        
+            });
+        } else {
+            alert('No existing geopackage found. Please subset the data before getting forcings');
+            return;
+        }
+    })
 }
 
 async function realization() {

map_app/static/js/main.js

@@ -133,27 +133,63 @@ function update_map(cat_id, e) {
   $('#selected-basins').text(cat_id)
   map.setFilter('selected-catchments', ['any', ['in', 'divide_id', cat_id]]);
   map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ""]])
-  fetch('/get_upstream_catids', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(cat_id),
-  })
-  .then(response => response.json())
-  .then(data => {
-    map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
-    if (data.length === 0) {
-      new maplibregl.Popup()
-        .setLngLat(e.lngLat)
-        .setHTML('No upstreams')
-        .addTo(map);
-    }
-  });
+  // get the position of the subset toggle
+  // false means subset by nexus, true means subset by catchment
+  var nexus_catchment = document.getElementById('subset-toggle').checked;
+  var subset_type = nexus_catchment ? 'catchment' : 'nexus';
+  console.log('subset_type:', subset_type);
+
+  if (subset_type == 'catchment') {
+    fetch('/get_upstream_catids', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(cat_id),
+    })
+    .then(response => response.json())
+    .then(data => {
+      map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
+      if (data.length === 0) {
+        new maplibregl.Popup()
+          .setLngLat(e.lngLat)
+          .setHTML('No upstreams')
+          .addTo(map);
+      }
+    });
+  } else {
+    fetch('/get_upstream_wbids', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(cat_id),
+    })
+    .then(response => response.json())
+    .then(data => {
+      map.setFilter('upstream-catchments', ['any', ['in', 'divide_id', ...data]]);
+      if (data.length === 0) {
+        new maplibregl.Popup()
+          .setLngLat(e.lngLat)
+          .setHTML('No upstreams')
+          .addTo(map);
+      }
+    });
+  }
 }
+let lastClickedLngLat = null; 
 map.on('click', 'catchments', (e) => {
   cat_id = e.features[0].properties.divide_id;
+  lastClickedLngLat = e.lngLat; // Store the last clicked location
   update_map(cat_id, e);
 });
 
+// When you want to use it (e.g., in your toggle handler):
+document.getElementById("subset-toggle").addEventListener('change', function() {
+  const cat_id = document.getElementById('selected-basins').textContent;
+  if (cat_id && cat_id !== 'None - get clicking!' && lastClickedLngLat) {
+    // Create a fake event with the last clicked location
+    const fakeEvent = { lngLat: lastClickedLngLat };
+    update_map(cat_id, fakeEvent);
+  }
+});
+
 // Create a popup, but don't add it to the map yet.
 const popup = new maplibregl.Popup({
   closeButton: false,

map_app/templates/index.html

@@ -27,6 +27,15 @@
                 <button id="toggle-button-camels">Show CAMELS basins</button>
                 <button id="toggle-button-nwm">Overlay NWM chunks</button>
                 <button id="toggle-button-aorc">Overlay AORC chunks</button>
+                <br>
+                <div class="toggle-switch toggle-switch-subset">
+                    <input type="checkbox" id="subset-toggle" class="toggle-input">
+                    <label for="subset-toggle" class="toggle-label">
+                        <span class="toggle-text toggle-text-subset toggle-text-left toggle-text-left-subset">Subset by nexus</span>
+                        <span class="toggle-text toggle-text-subset toggle-text-right">Subset by catchment</span>
+                        <span class="toggle-handle toggle-handle-subset"></span>
+                    </label>
+                </div>
             </div>
 
             <div class="command-container">
@@ -126,10 +135,16 @@
                     </label>
                 </div>
                 <button id="forcings-button">Create Forcing From Zarrs</button>
-                <img class=" loading" src="{{ url_for('static', filename='resources/loading.gif') }}" alt="Loading"
+                <img class="loading" src="{{ url_for('static', filename='resources/loading.gif') }}" alt="Loading"
                     id="forcings-loading">
+                <br>
             </div>
+            <br>
             <div id="forcings-output-path"></div>
+            <div id="progress">
+                <div id="bar"></div>
+                <div id="bar-text"></div>
+            </div>
         </section>
 
         <section id="realization">