ngiab-data-preprocess 4.6.6__py3-none-any.whl → 4.6.8__py3-none-any.whl

data_processing/dataset_utils.py

@@ -107,9 +107,13 @@ def clip_dataset_to_bounds(
     """
     # check time range here in case just this function is imported and not the whole module
     start_time, end_time = validate_time_range(dataset, start_time, end_time)
+    samplex = dataset.x.values[:2]
+    intervalx = samplex[1] - samplex[0]
+    sampley = dataset.y.values[:2]
+    intervaly = sampley[1] - sampley[0]
     dataset = dataset.sel(
-        x=slice(bounds[0], bounds[2]),
-        y=slice(bounds[1], bounds[3]),
+        x=slice(bounds[0]-intervalx, bounds[2]+intervalx),
+        y=slice(bounds[1]-intervaly, bounds[3]+intervaly),
         time=slice(start_time, end_time),
     )
     logger.info("Selected time range and clipped to bounds")

data_processing/gpkg_utils.py

@@ -530,7 +530,7 @@ def get_cat_to_nhd_feature_id(gpkg: Path = FilePaths.conus_hydrofabric) -> Dict[
         )
 
     table_name = list(tables)[0]
-    sql_query = f"SELECT divide_id, hf_id FROM {table_name} WHERE divide_id IS NOT NULL AND hf_id IS NOT NULL"
+    sql_query = f"SELECT divide_id, hf_id FROM {table_name} WHERE divide_id IS NOT NULL AND hf_id IS NOT NULL ORDER BY hf_hydroseq DESC"
 
     with sqlite3.connect(gpkg) as conn:
         result: List[Tuple[str, str]] = conn.execute(sql_query).fetchall()
@@ -539,6 +539,7 @@ def get_cat_to_nhd_feature_id(gpkg: Path = FilePaths.conus_hydrofabric) -> Dict[
     for cat, feature in result:
         # the ids are stored as floats this converts to int to match nwm output
         # numeric ids should be stored as strings.
+        # Because of the ORDER BY above, the lowest hf_hydroseq "wins"
         mapping[cat] = int(feature)
 
     return mapping

map_app/__init__.py

@@ -1,16 +1,19 @@
 from flask import Flask
 import logging
+from pathlib import Path
 from map_app.views import main, intra_module_db
 from data_sources.source_validation import validate_all
 
-with open("app.log", "w") as f:
+LOG_PATH = Path.home() / ".ngiab" / "app.log"
+
+with open(LOG_PATH, "w") as f:
     f.write("")
     f.write("Starting Application!\n")
 
 logging.basicConfig(
     level=logging.INFO,
     format="%(name)-12s: %(levelname)s - %(message)s",
-    filename="app.log",
+    filename=LOG_PATH,
     filemode="a",
 )  # Append mode
 # Example: Adding a console handler to root logger (optional)

map_app/__main__.py

@@ -3,18 +3,20 @@
 import logging
 import webbrowser
 from threading import Timer
+from pathlib import Path
 
 from data_processing.file_paths import FilePaths
 from data_processing.graph_utils import get_graph
 
 from map_app import app, console_handler
 
+LOG_PATH = Path.home() / ".ngiab" / "app.log"
 
 def open_browser():
     # find the last line in the log file that contains the port number
     # * running on http://0.0.0.0:port_number
     port_number = None
-    with open("app.log", "r") as f:
+    with open(LOG_PATH, 'r') as f:
         lines = f.readlines()
         for line in reversed(lines):
             if "Running on http" in line:
@@ -38,12 +40,12 @@ def main():
 
     if FilePaths.dev_file.is_file():
         Timer(2, set_logs_to_warning).start()
-        with open("app.log", "a") as f:
+        with open(LOG_PATH, "a") as f:
             f.write("Running in debug mode\n")
         app.run(debug=True, host="0.0.0.0", port="8080")  # type: ignore
     else:
         Timer(1, open_browser).start()
-        with open("app.log", "a") as f:
+        with open(LOG_PATH, "a") as f:
             f.write("Running in production mode\n")
         app.run(host="0.0.0.0", port="0")  # type: ignore
 

map_app/views.py

@@ -136,7 +136,7 @@ def download_forcings(data_source, start_time, end_time, paths):
 
 def compute_forcings(cached_data, paths):
     create_forcings(cached_data, paths.output_dir.stem)  # type: ignore
-    
+
 @main.route("/forcings", methods=["POST"])
 def get_forcings():
     # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
@@ -190,7 +190,7 @@ def get_catids_from_vpu():
 
 @main.route("/logs", methods=["GET"])
 def get_logs():
-    log_file_path = "app.log"
+    log_file_path = Path.home() / ".ngiab" / "app.log"
     try:
         with open(log_file_path, "r") as file:
             lines = file.readlines()

ngiab_data_cli/__main__.py

@@ -177,7 +177,7 @@ def main() -> None:
             else:
                 logging.info("Subsetting hydrofabric")
                 include_outlet = True
-                if args.gage:
+                if args.gage or args.subset_type == "catchment":
                     include_outlet = False
                 subset(
                     feature_to_subset,

ngiab_data_cli/arguments.py

@@ -3,6 +3,7 @@ from datetime import datetime
 
 # Constants
 DATE_FORMAT = "%Y-%m-%d"  # used for datetime parsing
+DATE_FORMAT2 = "%Y-%m-%d %H:%M"  # used for datetime parsing
 DATE_FORMAT_HINT = "YYYY-MM-DD"  # printed in help message
 
 
@@ -91,13 +92,13 @@ def parse_arguments() -> argparse.Namespace:
     parser.add_argument(
         "--start_date",
         "--start",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
         help=f"Start date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
         "--end_date",
         "--end",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
         help=f"End date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
@@ -147,6 +148,13 @@ def parse_arguments() -> argparse.Namespace:
         choices=["aorc", "nwm"],
         default="nwm",
     )
+    parser.add_argument(
+        "--subset_type",
+        type=str,
+        help="By nexus: get everything flowing into the downstream nexus of the selected catchment. By catchment: get everything flowing into the selected catchment.",
+        choices=["nexus", "catchment"],
+        default="nexus",
+    )
     parser.add_argument(
         "-a",
         "--all",

{ngiab_data_preprocess-4.6.6.dist-info → ngiab_data_preprocess-4.6.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.6.6
+Version: 4.6.8
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -13,6 +13,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyogrio>=0.7.2
 Requires-Dist: pyproj>=3.6.1
+Requires-Dist: pandas<3.0.0
 Requires-Dist: Flask==3.0.2
 Requires-Dist: geopandas>=1.0.0
 Requires-Dist: requests==2.32.4
@@ -28,7 +29,7 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: tqdm==4.66.4
 Requires-Dist: rich==13.7.1
 Requires-Dist: colorama==0.4.6
-Requires-Dist: bokeh==3.5.1
+Requires-Dist: bokeh==3.8.2
 Requires-Dist: boto3
 Requires-Dist: numcodecs<0.16.0
 Requires-Dist: scipy>=1.15.3
@@ -75,13 +76,13 @@ This repository contains tools for preparing data to run a [NextGen](https://git
 
 ## What does this tool do?
 
-This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.  
-It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).  
+This tool prepares data to run a NextGen-based simulation by creating a run package that can be used with NGIAB.
+It uses geometry and model attributes from the [v2.2 hydrofabric](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/conus/conus_nextgen.gpkg) more information on [all data sources here](https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/v2.2/hfv2.2-data_model.html).
 The raw forcing data is [nwm retrospective v3 forcing](https://noaa-nwm-retrospective-3-0-pds.s3.amazonaws.com/index.html#CONUS/zarr/forcing/) data or the [AORC 1km gridded data](https://noaa-nws-aorc-v1-1-1km.s3.amazonaws.com/index.html) depending on user input
 
-1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg). 
-2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy. 
-3. Creates **configuration files** for a default NGIAB model run. 
+1. **Subsets** (delineates) everything upstream of your point of interest (catchment, gage, flowpath etc) from the hydrofabric. This subset is output as a geopackage (.gpkg).
+2. Calculates **forcings** as a weighted mean of the gridded NWM or AORC forcings. Weights are calculated using [exact extract](https://isciences.github.io/exactextract/) and computed with numpy.
+3. Creates **configuration files** for a default NGIAB model run.
     -  realization.json  - ngen model configuration
     -  troute.yaml - routing configuration.
     -  **per catchment** model configuration
@@ -136,13 +137,13 @@ curl -LsSf https://astral.sh/uv/install.sh | sh
 # Create a virtual environment in the current directory
 uv venv
 
-# Install the tool in the virtual environment 
+# Install the tool in the virtual environment
 uv pip install ngiab_data_preprocess
 
 # To run the cli
 uv run cli --help
 
-# To run the map 
+# To run the map
 uv run map_app
 ```
 
@@ -160,7 +161,7 @@ UV automatically detects any virtual environments in the current directory and w
 (notebook) jovyan@jupyter-user:~$ conda deactivate
 jovyan@jupyter-user:~$
 # The interactive map won't work on 2i2c
-```    
+```
 
 ```bash
 # This tool is likely to not work without a virtual environment
@@ -205,7 +206,7 @@ To install and run the tool, follow these steps:
 
 Running the `map_app` tool will open the app in a new browser tab.
 
-Install-free: `uvx --from ngiab-data-preprocess map_app`  
+Install-free: `uvx --from ngiab-data-preprocess map_app`
 Installed with uv: `uv run map_app`
 
 ## Using the map interface
@@ -225,7 +226,7 @@ Once all the steps are finished, you can run NGIAB on the folder shown underneat
 
 ## Running the CLI
 
-Install-free: `uvx ngiab-prep`  
+Install-free: `uvx ngiab-prep`
 Installed with uv: `uv run cli`
 
 ## Arguments
@@ -236,6 +237,7 @@ Installed with uv: `uv run cli`
 - `-l`, `--latlon`: Use latitude and longitude instead of catid. Expects comma-separated values via the CLI, e.g., `python -m ngiab_data_cli -i 54.33,-69.4 -l -s`.
 - `-g`, `--gage`: Use gage ID instead of catid. Expects a single gage ID via the CLI, e.g., `python -m ngiab_data_cli -i 01646500 -g -s`.
 - `-s`, `--subset`: Subset the hydrofabric to the given feature.
+- `--subset_type`: Specify the subset type. `nexus`: get everything flowing into the downstream nexus of the selected catchment. `catchment`: get everything flowing into the selected catchment.
 - `-f`, `--forcings`: Generate forcings for the given feature.
 - `-r`, `--realization`: Create a realization for the given feature.
 - `--lstm`: Configures the data for the [python lstm](https://github.com/ciroh-ua/lstm/).
@@ -259,7 +261,7 @@ Installed with uv: `uv run cli`
 
 1. Prepare everything for an NGIAB run at a given gage:
    ```bash
-   uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28 
+   uvx ngiab-prep -i gage-10154200 -sfr --start 2022-01-01 --end 2022-02-28
    #         add --run or replace -sfr with --all to run NGIAB, too
    # to name the folder, add -o folder_name
    ```

{ngiab_data_preprocess-4.6.6.dist-info → ngiab_data_preprocess-4.6.8.dist-info}/RECORD

@@ -1,10 +1,10 @@
 data_processing/create_realization.py,sha256=b1a9Wuld9saJ-zzVDPY_kba3-ZOVbRuobyR-QiHLXyY,11550
 data_processing/dask_utils.py,sha256=A2IP94WAz8W9nek3etXKEKTOxGPf0NWSFLh8cZ5S-xU,2454
-data_processing/dataset_utils.py,sha256=pZpY4xvOKp4Sd44olRvEHi4dXP-kmOylWzVqU9QT69w,8407
+data_processing/dataset_utils.py,sha256=iLY_3dSRxd8O0wI24IM2zuJru8vtmRlph0O_ZU_mAr4,8597
 data_processing/datasets.py,sha256=_EJ1uZSWTU1HWpvF7TQSikneJqWZFikTrdo9usCV8A0,4665
 data_processing/file_paths.py,sha256=7MpwfIQewLRrDpAw1dxTjTperUwOk3EC_kthmnJSRII,4851
 data_processing/forcings.py,sha256=G_g3VSM_YN-k4FrbnUByrDR4n3fk1GVfv74kamit2CI,21775
-data_processing/gpkg_utils.py,sha256=VoQG53zaGcdmkU9M_7kRrk_vygRGJV9E-wX-T48Rzj8,20576
+data_processing/gpkg_utils.py,sha256=uERtQAMz9msJib_suffILuhUY8EcrvYw-pqkJOm2s9E,20673
 data_processing/graph_utils.py,sha256=4D72wMSiCRKCPC7JUz7XCoaISRGLuqDx6wpeO_VP8fk,8301
 data_processing/s3fs_utils.py,sha256=ki1EmA0ezV0r26re6dRWIGzL5FudGdwF9Qw1eVLR0Bc,2747
 data_processing/subset.py,sha256=EwDii-EsOiJBaDMjvYWfz7V9wQSXb887w_fk28u8dlw,3789
@@ -19,9 +19,9 @@ data_sources/noah-owp-modular-init.namelist.input,sha256=Vb7mp40hFpJogruOrXrDHwV
 data_sources/source_validation.py,sha256=DMCTo-Tad9QSXFqSDB5guI8hMzY_kQE6aW78-5nqwlU,9457
 data_sources/template.sql,sha256=ZnFqAqleEq9wgmAhNO90Wue_L9k0JAn8KF99DYtcxgs,10457
 data_sources/triggers.sql,sha256=G0d_175eNsamKAFhsbphPATvzMPuPL_iCleIhlToduQ,14906
-map_app/__init__.py,sha256=OarJao9X98kcbLyiwewN4ObWNAYkKDichcxbuWywTsA,818
-map_app/__main__.py,sha256=5qJGypfesfdWDWRsbTQiMs3uL5zPZKRZQq31l7qXqCc,1649
-map_app/views.py,sha256=AxLsXL8ZSnRJzg5zVYDGfVFA_6amgae41AO_E1G7W58,7923
+map_app/__init__.py,sha256=HvjuTiuf0vTvNLDsgmABqPPMQxOcTljX2nzgj6Pdp_g,888
+map_app/__main__.py,sha256=9wnCunreKNtvI-NEFB-bTHoKXuI5PtPznj13ctrDAOc,1717
+map_app/views.py,sha256=LRpHHw9FOzxEPY42LjJZ3XWFbKyZFKtPu9GZftdXrUY,7944
 map_app/static/css/console.css,sha256=xN6G2MMFyKc9YW9HEVpUUTUjx2o2nokBR4nCX5c18UM,803
 map_app/static/css/main.css,sha256=pYIIk-dXW6YMpliSJKATdvgPhFVY6K26NeUuHoYyJqg,8736
 map_app/static/css/toggle.css,sha256=aUIe9AL1-_mzKvWir3lJ9W8r2oXsOjkFbtvcrohnV10,4149
@@ -31,13 +31,13 @@ map_app/static/js/main.js,sha256=wtLZ18hPZbnyo28dRGsOf-CAHCGxqe1PWEAGa6-S4tg,109
 map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
 map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
 map_app/templates/index.html,sha256=qgQxvaJgkU8Ul57dMLX5n0O3BrwMow-CxGPH7_8yjXc,12195
-ngiab_data_cli/__main__.py,sha256=_8IFqL2CGZGMsC71uutqHOIjZpfYBD91yKTk4dvEa3c,11318
-ngiab_data_cli/arguments.py,sha256=97BFzlj68tt9tRCR3rTPnK6o7zd7dUCdPg8Cypa2aKA,4782
+ngiab_data_cli/__main__.py,sha256=FmHxibQtFm15aEUG1BrIIqIWNU_BoXzWCTXoR9EA-YI,11353
+ngiab_data_cli/arguments.py,sha256=Bi_Q6FRXvRxI50Pgk7goyASe-2lfUh-oDdnYkaY4KTc,5262
 ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
 ngiab_data_cli/forcing_cli.py,sha256=eIWRxRWUwPqR16fihFDEIV4VzGlNuvcD6lJW5VYjkPU,3635
-ngiab_data_preprocess-4.6.6.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
-ngiab_data_preprocess-4.6.6.dist-info/METADATA,sha256=2TEVfZHKru4tHt300zQI_tW9SoT4SWKeFnE7cDOA_y8,14337
-ngiab_data_preprocess-4.6.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ngiab_data_preprocess-4.6.6.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
-ngiab_data_preprocess-4.6.6.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
-ngiab_data_preprocess-4.6.6.dist-info/RECORD,,
+ngiab_data_preprocess-4.6.8.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
+ngiab_data_preprocess-4.6.8.dist-info/METADATA,sha256=SR4MIdIf-NJsmQIVblbUQkzQEorQymRjDhQRK4eJv8Y,14541
+ngiab_data_preprocess-4.6.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ngiab_data_preprocess-4.6.8.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
+ngiab_data_preprocess-4.6.8.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
+ngiab_data_preprocess-4.6.8.dist-info/RECORD,,

{ngiab_data_preprocess-4.6.6.dist-info → ngiab_data_preprocess-4.6.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any