ngiab-data-preprocess 4.0.5__py3-none-any.whl → 4.1.1__py3-none-any.whl

data_processing/create_realization.py

@@ -4,6 +4,7 @@ import sqlite3
 from datetime import datetime
 from pathlib import Path
 import shutil
+import requests
 
 import pandas
 import s3fs
@@ -309,18 +310,37 @@ def create_em_realization(cat_id: str, start_time: datetime, end_time: datetime)
     create_partitions(paths)
 
 
-def create_realization(cat_id: str, start_time: datetime, end_time: datetime, use_nwm_gw: bool = False):
+def create_realization(
+    cat_id: str,
+    start_time: datetime,
+    end_time: datetime,
+    use_nwm_gw: bool = False,
+    gage_id: str = None,
+):
     paths = file_paths(cat_id)
 
-    # get approximate groundwater levels from nwm output
     template_path = paths.template_cfe_nowpm_realization_config
-    
+
+    if gage_id is not None:
+        # try and download s3:communityhydrofabric/hydrofabrics/community/gage_parameters/gage_id
+        # if it doesn't exist, use the default
+        try:
+            url = f"https://communityhydrofabric.s3.us-east-1.amazonaws.com/hydrofabrics/community/gage_parameters/{gage_id}.json"
+
+            new_template = requests.get(url).json()
+            template_path = paths.config_dir / "calibrated_params.json"
+            with open(template_path, "w") as f:
+                json.dump(new_template, f)
+        except Exception as e:
+            logger.warning(f"Failed to download gage parameters")
+
     conf_df = get_model_attributes(paths.geopackage_path)
 
     if use_nwm_gw:
         gw_levels = get_approximate_gw_storage(paths, start_time)
     else:
         gw_levels = dict()
+
     make_cfe_config(conf_df, paths, gw_levels)
 
     make_noahowp_config(paths.config_dir, conf_df, start_time, end_time)

data_processing/gpkg_utils.py

@@ -317,15 +317,33 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     sql_query = f"SELECT * FROM '{table}' WHERE vpuid IN ({','.join(vpus)})"
     contents = source_db.execute(sql_query).fetchall()
 
+    if table == "network":
+        # Look for the network entry that has a toid not in the flowpath or nexus tables        
+        network_toids = [x[2] for x in contents]
+        print(f"Network toids: {len(network_toids)}")
+        sql = "SELECT id FROM flowpaths"  
+        flowpath_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Flowpath ids: {len(flowpath_ids)}")
+        sql = "SELECT id FROM nexus"
+        nexus_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Nexus ids: {len(nexus_ids)}")
+        bad_ids = set(network_toids) - set(flowpath_ids + nexus_ids)
+        print(bad_ids)
+        print(f"Removing {len(bad_ids)} network entries that are not in flowpaths or nexuses")
+        # id column is second after fid
+        contents = [x for x in contents if x[1] not in bad_ids]
+
     insert_data(dest_db, table, contents)
 
+
     if table in get_feature_tables(file_paths.conus_hydrofabric):
         fids = [str(x[0]) for x in contents]
         copy_rTree_tables(table, fids, source_db, dest_db)
 
     dest_db.commit()
     source_db.close()
-    dest_db.close()
+    dest_db.close()   
+    
 
 def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name: Path) -> None:
     """

data_processing/subset.py

@@ -22,13 +22,16 @@ subset_tables = [
     "flowpath-attributes-ml",
     "flowpaths",
     "hydrolocations",
-    "network",
     "nexus",
     "pois",  # requires flowpaths
     "lakes",  # requires pois
+    "network",
 ]
 
-def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False) -> Path:
+
+def create_subset_gpkg(
+    ids: Union[List[str], str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False
+) -> Path:
     # ids is a list of nexus and wb ids, or a single vpu id
     if not isinstance(ids, list):
         ids = [ids]
@@ -42,12 +45,13 @@ def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg
     for table in subset_tables:
         if is_vpu:
             subset_table_by_vpu(table, ids[0], hydrofabric, output_gpkg_path)
-        else:                
+        else:
             subset_table(table, ids, hydrofabric, output_gpkg_path)
 
     add_triggers_to_gpkg(output_gpkg_path)
     update_geopackage_metadata(output_gpkg_path)
 
+
 def subset_vpu(vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_paths.conus_hydrofabric):
 
     if output_gpkg_path.exists():
@@ -77,6 +81,7 @@ def subset(
     logger.info(f"Subset complete for {len(upstream_ids)} features (catchments + nexuses)")
     logger.debug(f"Subset complete for {upstream_ids} catchments")
 
+
 def move_files_to_config_dir(subset_output_dir: str) -> None:
     config_dir = subset_output_dir / "config"
     config_dir.mkdir(parents=True, exist_ok=True)

ngiab_data_cli/__main__.py

@@ -169,13 +169,20 @@ def main() -> None:
 
         if args.realization:
             logging.info(f"Creating realization from {args.start_date} to {args.end_date}...")
+            gage_id = None
+            if args.gage:
+                gage_id = args.input_feature
             if args.empirical_model:
                 create_em_realization(
                     output_folder, start_time=args.start_date, end_time=args.end_date
                 )
             else:
                 create_realization(
-                    output_folder, start_time=args.start_date, end_time=args.end_date, use_nwm_gw=args.nwm_gw
+                    output_folder,
+                    start_time=args.start_date,
+                    end_time=args.end_date,
+                    use_nwm_gw=args.nwm_gw,
+                    gage_id=gage_id,
                 )
             logging.info("Realization creation complete.")
 
@@ -198,7 +205,7 @@ def main() -> None:
             except:
                 logging.error("Docker is not running, please start Docker and try again.")
             try:
-                #command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
+                # command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
                 command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {num_partitions} local'
                 subprocess.run(command, shell=True)
                 logging.info("Next Gen run complete.")

{ngiab_data_preprocess-4.0.5.dist-info → ngiab_data_preprocess-4.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.0.5
+Version: 4.1.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -8,11 +8,11 @@ Project-URL: Issues, https://github.com/CIROH-UA/NGIAB_data_preprocess/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.13,>=3.10
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pyogrio==0.7.2
-Requires-Dist: pyproj==3.6.1
+Requires-Dist: pyogrio>=0.7.2
+Requires-Dist: pyproj>=3.6.1
 Requires-Dist: Flask==3.0.2
 Requires-Dist: geopandas>=1.0.0
 Requires-Dist: requests==2.32.2
@@ -20,19 +20,20 @@ Requires-Dist: igraph==0.11.4
 Requires-Dist: s3fs==2024.3.1
 Requires-Dist: xarray==2024.2.0
 Requires-Dist: zarr==2.17.1
-Requires-Dist: netCDF4==1.6.5
+Requires-Dist: netCDF4>=1.6.5
 Requires-Dist: dask==2024.4.1
 Requires-Dist: dask[distributed]==2024.4.1
 Requires-Dist: black==24.3.0
 Requires-Dist: isort==5.13.2
 Requires-Dist: h5netcdf==1.3.0
 Requires-Dist: exactextract==0.2.0
-Requires-Dist: numpy==1.26.4
+Requires-Dist: numpy>=1.26.4
 Requires-Dist: tqdm==4.66.4
 Requires-Dist: rich==13.7.1
 Requires-Dist: colorama==0.4.6
 Requires-Dist: bokeh==3.5.1
 Requires-Dist: boto3
+Requires-Dist: numcodecs<0.16.0
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot
@@ -85,6 +86,41 @@ For automatic interactive visualisation, please run [NGIAB](https://github.com/C
 * This tool is officially supported on macOS or Ubuntu (tested on 22.04 & 24.04). To use it on Windows, please install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 
 ## Installation and Running
+It is highly recommended to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. It works similarly to pip and conda, and I would also recommend you use it for other python projects as it is so useful.
+
+```bash
+# Install UV
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# It can be installed via pip if that fails
+# pip install uv
+
+# Create a virtual environment in the current directory
+uv venv
+
+# Install the tool in the virtual environment 
+uv pip install ngiab_data_preprocess
+
+# To run the cli
+uv run cli --help
+
+# To run the map 
+uv run map_app
+```
+
+UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
+
+### Running without install
+This package supports pipx and uvx which means you can run the tool without installing it. No virtual environment needed, just UV.
+```bash
+# run this from anywhere 
+uvx --from ngiab_data_preprocess cli --help
+# for the map
+uvx --from ngiab_data_preprocess map_app
+```
+
+## For legacy pip installation
+<details>
+  <summary>Click here to expand</summary>
 
 ```bash
 # If you're installing this on jupyterhub / 2i2c you HAVE TO DEACTIVATE THE CONDA ENV
@@ -102,8 +138,7 @@ pip install 'ngiab_data_preprocess'
 python -m map_app
 # CLI instructions at the bottom of the README
 ```
-
-The first time you run this command, it will download the hydrofabric from Lynker Spatial. If you already have it, place `conus_nextgen.gpkg` into `~/.ngiab/hydrofabric/v2.2/`.
+</details>
 
 ## Development Installation
 
@@ -117,24 +152,23 @@ To install and run the tool, follow these steps:
    git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
    cd NGIAB_data_preprocess
    ```
-2. Create a virtual environment and activate it:
+2. Create a virtual environment:
    ```bash
-   python3 -m venv env
-   source env/bin/activate
+   uv venv
    ```
 3. Install the tool:
    ```bash
-   pip install -e .
+   uv pip install -e .
    ```
 4. Run the map app:
    ```bash
-   python -m map_app
+   uv run map_app
    ```
 </details>
 
 ## Usage
 
-Running the command `python -m map_app` will open the app in a new browser tab.
+Running the command `uv run map_app` will open the app in a new browser tab.
 
 To use the tool:
 1. Select the catchment you're interested in on the map.

{ngiab_data_preprocess-4.0.5.dist-info → ngiab_data_preprocess-4.1.1.dist-info}/RECORD

@@ -1,12 +1,12 @@
-data_processing/create_realization.py,sha256=qsUXNxPgXkQ26PSTl7arXXQ3Ea6d31Y2mVTGSMopcpo,14268
+data_processing/create_realization.py,sha256=AxUDVSORjwd6IAImZpgDr8GRHKSUQF8emQi1ikfIno4,14899
 data_processing/dataset_utils.py,sha256=4qmRmK3qMWPkiWZHXhXv3c-ISbtOwr7QhciEl2ok6Ao,7314
 data_processing/datasets.py,sha256=kAtTKj2yQINUSzI0vX-kuXMPVeQBtZVfGkfpPh6Qi_0,3528
 data_processing/file_paths.py,sha256=Cp3BxbO6sD50464ciTshjb3Z0vTvL0ZeSbOJgNdOqQ0,4698
 data_processing/forcings.py,sha256=6Q9fSXa67OrXg_r9li0K9widsclN2DQUp1-oUH2tZwo,19208
-data_processing/gpkg_utils.py,sha256=oVUutZWTKm2_1xAVSuMgMUc_9OwJHNNHZWkqH7uE6yA,19403
+data_processing/gpkg_utils.py,sha256=SdniJrwhta7fPchVSCliuThPENgcs0m9Elx615yttAU,20248
 data_processing/graph_utils.py,sha256=TsCSZmQ8SPm5F2YzDKv32BxdzaHlPrC6UijZxUJbZvI,7966
 data_processing/s3fs_utils.py,sha256=WoqqwxkHpv0Qq4I5f5-gUZBCFtVQ68ehXbdOjWRKTDQ,2752
-data_processing/subset.py,sha256=8bNkiOWZcmOrHo6MXQceMulbEnXitbD1NUTgIbOaJmQ,2920
+data_processing/subset.py,sha256=WklS0CT5O4oz29fhTT46NmSOe_EFMCC93gLeg6nQ88I,2914
 data_sources/cfe-nowpm-realization-template.json,sha256=8an6q1drWD8wU1ocvdPab-GvZDvlQ-0di_-NommH3QI,3528
 data_sources/cfe-template.ini,sha256=6e5-usqjWtm3MWVvtm8CTeZTJJMxO1ZswkOXq0L9mnc,2033
 data_sources/em-catchment-template.yml,sha256=M08ixazEUHYI2PNavtI0xPZeSzcQ9bg2g0XzNT-8_u4,292
@@ -32,13 +32,13 @@ map_app/static/resources/light-style.json,sha256=DaE52qKpAkjiWSKY_z7LxreqA2rW4Zy
 map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
 map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
 map_app/templates/index.html,sha256=ITRzQEYn15sFN4qRACjaNj5muhalOeP9n_IwcdsIlUs,6631
-ngiab_data_cli/__main__.py,sha256=_m0bhAXdyLeZBuh03PKNXlJ6BUJnZDR-xZIbFyRH8iE,10164
+ngiab_data_cli/__main__.py,sha256=X_imHhWo2RBTJzJ9bUwEcAnAtZSyjKJXB60dBxea6ck,10361
 ngiab_data_cli/arguments.py,sha256=7ptImy8tLM1XvjfDr13tZszkjGVtenXo0KqllJeE3Mw,4372
 ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
 ngiab_data_cli/forcing_cli.py,sha256=lkcqWDk5H8IPyGv0DwLIZMQldqTUXpfwSX0C_RIuIJ8,3890
-ngiab_data_preprocess-4.0.5.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
-ngiab_data_preprocess-4.0.5.dist-info/METADATA,sha256=RkqbXcc3Ac-4qJvLCn-qiCnUV2DCFfoSpp-QdZcuCWw,9433
-ngiab_data_preprocess-4.0.5.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
-ngiab_data_preprocess-4.0.5.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
-ngiab_data_preprocess-4.0.5.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
-ngiab_data_preprocess-4.0.5.dist-info/RECORD,,
+ngiab_data_preprocess-4.1.1.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
+ngiab_data_preprocess-4.1.1.dist-info/METADATA,sha256=-LH19uwmvO9AB0NBtN4JECrQrwSLVZUXKtoSVqaNoWA,10310
+ngiab_data_preprocess-4.1.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+ngiab_data_preprocess-4.1.1.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
+ngiab_data_preprocess-4.1.1.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
+ngiab_data_preprocess-4.1.1.dist-info/RECORD,,

{ngiab_data_preprocess-4.0.5.dist-info → ngiab_data_preprocess-4.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.1)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any