ngiab-data-preprocess 4.6.5__py3-none-any.whl → 4.6.7__py3-none-any.whl

data_processing/dataset_utils.py

@@ -107,9 +107,13 @@ def clip_dataset_to_bounds(
     """
     # check time range here in case just this function is imported and not the whole module
     start_time, end_time = validate_time_range(dataset, start_time, end_time)
+    samplex = dataset.x.values[:2]
+    intervalx = samplex[1] - samplex[0]
+    sampley = dataset.y.values[:2]
+    intervaly = sampley[1] - sampley[0]
     dataset = dataset.sel(
-        x=slice(bounds[0], bounds[2]),
-        y=slice(bounds[1], bounds[3]),
+        x=slice(bounds[0]-intervalx, bounds[2]+intervalx),
+        y=slice(bounds[1]-intervaly, bounds[3]+intervaly),
         time=slice(start_time, end_time),
     )
     logger.info("Selected time range and clipped to bounds")

data_processing/gpkg_utils.py

@@ -530,7 +530,7 @@ def get_cat_to_nhd_feature_id(gpkg: Path = FilePaths.conus_hydrofabric) -> Dict[
         )
 
     table_name = list(tables)[0]
-    sql_query = f"SELECT divide_id, hf_id FROM {table_name} WHERE divide_id IS NOT NULL AND hf_id IS NOT NULL"
+    sql_query = f"SELECT divide_id, hf_id FROM {table_name} WHERE divide_id IS NOT NULL AND hf_id IS NOT NULL ORDER BY hf_hydroseq DESC"
 
     with sqlite3.connect(gpkg) as conn:
         result: List[Tuple[str, str]] = conn.execute(sql_query).fetchall()
@@ -539,6 +539,7 @@ def get_cat_to_nhd_feature_id(gpkg: Path = FilePaths.conus_hydrofabric) -> Dict[
     for cat, feature in result:
         # the ids are stored as floats this converts to int to match nwm output
         # numeric ids should be stored as strings.
+        # Because of the ORDER BY above, the lowest hf_hydroseq "wins"
         mapping[cat] = int(feature)
 
     return mapping

map_app/static/css/toggle.css

@@ -79,7 +79,7 @@
 .toggle-input:checked + .toggle-label .toggle-text-left {
     color: #888; /* Grey color for non-selected text */
 }
-  
+
 .toggle-input:checked + .toggle-label .toggle-text-right {
     color: #888; /* Grey color for non-selected text */
 }
@@ -91,7 +91,7 @@
   top: -40px;
 }
 
-.menu-switch__input {
+#menuToggle .menu-switch__input {
   opacity: 0;
   width: 100%;
   height: 100%;
@@ -162,7 +162,7 @@
         height: 1.4em;
         position: relative;
         top: -0.2em;
-        margin-right: 1em; 
+        margin-right: 1em;
         vertical-align: top;
         cursor: pointer;
         text-align: center;

map_app/static/js/data_processing.js

@@ -19,12 +19,13 @@ async function subset() {
                 document.getElementById('output-path').innerHTML = "Subset canceled. Geopackage located at " + filename;
                 return;
             }
-        } 
+        }
         // check what kind of subset
-        // get the position of the subset toggle
-        // false means subset by nexus, true means subset by catchment
-        var nexus_catchment = document.getElementById('subset-toggle').checked;
-        var subset_type = nexus_catchment ? 'catchment' : 'nexus';
+        if (document.getElementById('radio-nexus').checked) {
+            var subset_type = 'nexus'
+        } else {
+            var subset_type = 'catchment'
+        }
 
         const startTime = performance.now(); // Start the timer
         fetch('/subset', {
@@ -126,7 +127,7 @@ async function forcings() {
                 body: JSON.stringify(forcing_dir),
             })
             .then(async (response) => response.text())
-            .then(progressFile => { 
+            .then(progressFile => {
                 pollForcingsProgress(progressFile); // Start polling for progress
             })
             fetch('/forcings', {
@@ -138,7 +139,7 @@ async function forcings() {
             .catch(error => {
                 console.error('Error:', error);
             }).finally(() => {
-                document.getElementById('forcings-button').disabled = false;        
+                document.getElementById('forcings-button').disabled = false;
             });
         } else {
             alert('No existing geopackage found. Please subset the data before getting forcings');

ngiab_data_cli/__main__.py

@@ -177,7 +177,7 @@ def main() -> None:
             else:
                 logging.info("Subsetting hydrofabric")
                 include_outlet = True
-                if args.gage:
+                if args.gage or args.subset_type == "catchment":
                     include_outlet = False
                 subset(
                     feature_to_subset,

ngiab_data_cli/arguments.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 # Constants
 DATE_FORMAT = "%Y-%m-%d"  # used for datetime parsing
+DATE_FORMAT2 = "%Y-%m-%d %H:%M"  # used for datetime parsing
 DATE_FORMAT_HINT = "YYYY-MM-DD"  # printed in help message
 
 
@@ -91,13 +92,13 @@ def parse_arguments() -> argparse.Namespace:
     parser.add_argument(
         "--start_date",
         "--start",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
         help=f"Start date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
         "--end_date",
         "--end",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
         help=f"End date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
@@ -147,6 +148,13 @@ def parse_arguments() -> argparse.Namespace:
         choices=["aorc", "nwm"],
         default="nwm",
     )
+    parser.add_argument(
+        "--subset_type",
+        type=str,
+        help="By nexus: get everything flowing into the downstream nexus of the selected catchment. By catchment: get everything flowing into the selected catchment.",
+        choices=["nexus", "catchment"],
+        default="nexus",
+    )
     parser.add_argument(
         "-a",
         "--all",

{ngiab_data_preprocess-4.6.5.dist-info → ngiab_data_preprocess-4.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.6.5
+Version: 4.6.7
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -75,13 +75,13 @@ This repository contains tools for preparing data to run a [NextGen](https://git
 
 ## What does this tool do?
 
-This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.  
-It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).  
+This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.
+It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).
 The raw forcing data is [nwm retrospective v3 forcing](https://noaa-nwm-retrospective-3-0-pds.s3.amazonaws.com/index.html#CONUS/zarr/forcing/) data or the [AORC 1km gridded data](https://noaa-nws-aorc-v1-1-1km.s3.amazonaws.com/index.html) depending on user input
 
-1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg). 
-2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy. 
-3. Creates **configuration files** for a default NGIAB model run. 
+1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg).
+2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy.
+3. Creates **configuration files** for a default NGIAB model run.
     -  realization.json  - ngen model configuration
     -  troute.yaml - routing configuration.
     -  **per catchment** model configuration
@@ -136,13 +136,13 @@ curl -LsSf https://astral.sh/uv/install.sh | sh
 # Create a virtual environment in the current directory
 uv venv
 
-# Install the tool in the virtual environment 
+# Install the tool in the virtual environment
 uv pip install ngiab_data_preprocess
 
 # To run the cli
 uv run cli --help
 
-# To run the map 
+# To run the map
 uv run map_app
 ```
 
@@ -160,7 +160,7 @@ UV automatically detects any virtual environments in the current directory and w
 (notebook) jovyan@jupyter-user:~$ conda deactivate
 jovyan@jupyter-user:~$
 # The interactive map won't work on 2i2c
-```    
+```
 
 ```bash
 # This tool is likely to not work without a virtual environment
@@ -205,7 +205,7 @@ To install and run the tool, follow these steps:
 
 Running the `map_app` tool will open the app in a new browser tab.
 
-Install-free: `uvx --from ngiab-data-preprocess map_app`  
+Install-free: `uvx --from ngiab-data-preprocess map_app`
 Installed with uv: `uv run map_app`
 
 ## Using the map interface
@@ -225,7 +225,7 @@ Once all the steps are finished, you can run NGIAB on the folder shown underneat
 
 ## Running the CLI
 
-Install-free: `uvx ngiab-prep`  
+Install-free: `uvx ngiab-prep`
 Installed with uv: `uv run cli`
 
 ## Arguments
@@ -236,6 +236,7 @@ Installed with uv: `uv run cli`
 - `-l`, `--latlon`: Use latitude and longitude instead of catid. Expects comma-separated values via the CLI, e.g., `python -m ngiab_data_cli -i 54.33,-69.4 -l -s`.
 - `-g`, `--gage`: Use gage ID instead of catid. Expects a single gage ID via the CLI, e.g., `python -m ngiab_data_cli -i 01646500 -g -s`.
 - `-s`, `--subset`: Subset the hydrofabric to the given feature.
+- `--subset_type`: Specify the subset type. `nexus`: get everything flowing into the downstream nexus of the selected catchment. `catchment`: get everything flowing into the selected catchment.
 - `-f`, `--forcings`: Generate forcings for the given feature.
 - `-r`, `--realization`: Create a realization for the given feature.
 - `--lstm`: Configures the data for the [python lstm](https://github.com/ciroh-ua/lstm/).
@@ -259,7 +260,7 @@ Installed with uv: `uv run cli`
 
 1. Prepare everything for an NGIAB run at a given gage:
    ```bash
-   uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28 
+   uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28
    #         add --run or replace -sfr with --all to run NGIAB, too
    # to name the folder, add -o folder_name
    ```

{ngiab_data_preprocess-4.6.5.dist-info → ngiab_data_preprocess-4.6.7.dist-info}/RECORD

@@ -1,10 +1,10 @@
 data_processing/create_realization.py,sha256=b1a9Wuld9saJ-zzVDPY_kba3-ZOVbRuobyR-QiHLXyY,11550
 data_processing/dask_utils.py,sha256=A2IP94WAz8W9nek3etXKEKTOxGPf0NWSFLh8cZ5S-xU,2454
-data_processing/dataset_utils.py,sha256=pZpY4xvOKp4Sd44olRvEHi4dXP-kmOylWzVqU9QT69w,8407
+data_processing/dataset_utils.py,sha256=iLY_3dSRxd8O0wI24IM2zuJru8vtmRlph0O_ZU_mAr4,8597
 data_processing/datasets.py,sha256=_EJ1uZSWTU1HWpvF7TQSikneJqWZFikTrdo9usCV8A0,4665
 data_processing/file_paths.py,sha256=7MpwfIQewLRrDpAw1dxTjTperUwOk3EC_kthmnJSRII,4851
 data_processing/forcings.py,sha256=G_g3VSM_YN-k4FrbnUByrDR4n3fk1GVfv74kamit2CI,21775
-data_processing/gpkg_utils.py,sha256=VoQG53zaGcdmkU9M_7kRrk_vygRGJV9E-wX-T48Rzj8,20576
+data_processing/gpkg_utils.py,sha256=uERtQAMz9msJib_suffILuhUY8EcrvYw-pqkJOm2s9E,20673
 data_processing/graph_utils.py,sha256=4D72wMSiCRKCPC7JUz7XCoaISRGLuqDx6wpeO_VP8fk,8301
 data_processing/s3fs_utils.py,sha256=ki1EmA0ezV0r26re6dRWIGzL5FudGdwF9Qw1eVLR0Bc,2747
 data_processing/subset.py,sha256=EwDii-EsOiJBaDMjvYWfz7V9wQSXb887w_fk28u8dlw,3789
@@ -24,20 +24,20 @@ map_app/__main__.py,sha256=5qJGypfesfdWDWRsbTQiMs3uL5zPZKRZQq31l7qXqCc,1649
 map_app/views.py,sha256=AxLsXL8ZSnRJzg5zVYDGfVFA_6amgae41AO_E1G7W58,7923
 map_app/static/css/console.css,sha256=xN6G2MMFyKc9YW9HEVpUUTUjx2o2nokBR4nCX5c18UM,803
 map_app/static/css/main.css,sha256=pYIIk-dXW6YMpliSJKATdvgPhFVY6K26NeUuHoYyJqg,8736
-map_app/static/css/toggle.css,sha256=W-qopD2OYbciPuxtUvvQZpzAc75ps3404JhtPJ621K8,4140
+map_app/static/css/toggle.css,sha256=aUIe9AL1-_mzKvWir3lJ9W8r2oXsOjkFbtvcrohnV10,4149
 map_app/static/js/console.js,sha256=BnG0pED5B9d563sLWshDNt_q-SyoTY48sETvVoOVJkU,1377
-map_app/static/js/data_processing.js,sha256=sfeXvfPtiOvEN1kDFFtibLDOjwyudOpeOjmfXuTcR7Y,8583
+map_app/static/js/data_processing.js,sha256=F3ahDJabAgmcd9wFL8tThzRxuh6o0fT5M_T2OqLgSyw,8475
 map_app/static/js/main.js,sha256=wtLZ18hPZbnyo28dRGsOf-CAHCGxqe1PWEAGa6-S4tg,10932
 map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
 map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
 map_app/templates/index.html,sha256=qgQxvaJgkU8Ul57dMLX5n0O3BrwMow-CxGPH7_8yjXc,12195
-ngiab_data_cli/__main__.py,sha256=_8IFqL2CGZGMsC71uutqHOIjZpfYBD91yKTk4dvEa3c,11318
-ngiab_data_cli/arguments.py,sha256=97BFzlj68tt9tRCR3rTPnK6o7zd7dUCdPg8Cypa2aKA,4782
+ngiab_data_cli/__main__.py,sha256=FmHxibQtFm15aEUG1BrIIqIWNU_BoXzWCTXoR9EA-YI,11353
+ngiab_data_cli/arguments.py,sha256=Bi_Q6FRXvRxI50Pgk7goyASe-2lfUh-oDdnYkaY4KTc,5262
 ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
 ngiab_data_cli/forcing_cli.py,sha256=eIWRxRWUwPqR16fihFDEIV4VzGlNuvcD6lJW5VYjkPU,3635
-ngiab_data_preprocess-4.6.5.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
-ngiab_data_preprocess-4.6.5.dist-info/METADATA,sha256=lBHx8b6aXyJkAx5W937aJ7oATzozN7XL7rY05M8UZ8Y,14337
-ngiab_data_preprocess-4.6.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ngiab_data_preprocess-4.6.5.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
-ngiab_data_preprocess-4.6.5.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
-ngiab_data_preprocess-4.6.5.dist-info/RECORD,,
+ngiab_data_preprocess-4.6.7.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
+ngiab_data_preprocess-4.6.7.dist-info/METADATA,sha256=3U2P21OHELN_hol-fu43soXXozFw3r6Y57rLwCK2GLc,14513
+ngiab_data_preprocess-4.6.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ngiab_data_preprocess-4.6.7.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
+ngiab_data_preprocess-4.6.7.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
+ngiab_data_preprocess-4.6.7.dist-info/RECORD,,