ngiab-data-preprocess 4.0.5__tar.gz → 4.1.1__tar.gz

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.0.5
+Version: 4.1.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -8,11 +8,11 @@ Project-URL: Issues, https://github.com/CIROH-UA/NGIAB_data_preprocess/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.13,>=3.10
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pyogrio==0.7.2
-Requires-Dist: pyproj==3.6.1
+Requires-Dist: pyogrio>=0.7.2
+Requires-Dist: pyproj>=3.6.1
 Requires-Dist: Flask==3.0.2
 Requires-Dist: geopandas>=1.0.0
 Requires-Dist: requests==2.32.2
@@ -20,19 +20,20 @@ Requires-Dist: igraph==0.11.4
 Requires-Dist: s3fs==2024.3.1
 Requires-Dist: xarray==2024.2.0
 Requires-Dist: zarr==2.17.1
-Requires-Dist: netCDF4==1.6.5
+Requires-Dist: netCDF4>=1.6.5
 Requires-Dist: dask==2024.4.1
 Requires-Dist: dask[distributed]==2024.4.1
 Requires-Dist: black==24.3.0
 Requires-Dist: isort==5.13.2
 Requires-Dist: h5netcdf==1.3.0
 Requires-Dist: exactextract==0.2.0
-Requires-Dist: numpy==1.26.4
+Requires-Dist: numpy>=1.26.4
 Requires-Dist: tqdm==4.66.4
 Requires-Dist: rich==13.7.1
 Requires-Dist: colorama==0.4.6
 Requires-Dist: bokeh==3.5.1
 Requires-Dist: boto3
+Requires-Dist: numcodecs<0.16.0
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot
@@ -85,6 +86,41 @@ For automatic interactive visualisation, please run [NGIAB](https://github.com/C
 * This tool is officially supported on macOS or Ubuntu (tested on 22.04 & 24.04). To use it on Windows, please install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 
 ## Installation and Running
+It is highly recommended to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. It works similarly to pip and conda, and I would also recommend you use it for other python projects as it is so useful.
+
+```bash
+# Install UV
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# It can be installed via pip if that fails
+# pip install uv
+
+# Create a virtual environment in the current directory
+uv venv
+
+# Install the tool in the virtual environment 
+uv pip install ngiab_data_preprocess
+
+# To run the cli
+uv run cli --help
+
+# To run the map 
+uv run map_app
+```
+
+UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
+
+### Running without install
+This package supports pipx and uvx which means you can run the tool without installing it. No virtual environment needed, just UV.
+```bash
+# run this from anywhere 
+uvx --from ngiab_data_preprocess cli --help
+# for the map
+uvx --from ngiab_data_preprocess map_app
+```
+
+## For legacy pip installation
+<details>
+  <summary>Click here to expand</summary>
 
 ```bash
 # If you're installing this on jupyterhub / 2i2c you HAVE TO DEACTIVATE THE CONDA ENV
@@ -102,8 +138,7 @@ pip install 'ngiab_data_preprocess'
 python -m map_app
 # CLI instructions at the bottom of the README
 ```
-
-The first time you run this command, it will download the hydrofabric from Lynker Spatial. If you already have it, place `conus_nextgen.gpkg` into `~/.ngiab/hydrofabric/v2.2/`.
+</details>
 
 ## Development Installation
 
@@ -117,24 +152,23 @@ To install and run the tool, follow these steps:
    git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
    cd NGIAB_data_preprocess
    ```
-2. Create a virtual environment and activate it:
+2. Create a virtual environment:
    ```bash
-   python3 -m venv env
-   source env/bin/activate
+   uv venv
    ```
 3. Install the tool:
    ```bash
-   pip install -e .
+   uv pip install -e .
    ```
 4. Run the map app:
    ```bash
-   python -m map_app
+   uv run map_app
    ```
 </details>
 
 ## Usage
 
-Running the command `python -m map_app` will open the app in a new browser tab.
+Running the command `uv run map_app` will open the app in a new browser tab.
 
 To use the tool:
 1. Select the catchment you're interested in on the map.

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/README.md

@@ -44,6 +44,41 @@ For automatic interactive visualisation, please run [NGIAB](https://github.com/C
 * This tool is officially supported on macOS or Ubuntu (tested on 22.04 & 24.04). To use it on Windows, please install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 
 ## Installation and Running
+It is highly recommended to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. It works similarly to pip and conda, and I would also recommend you use it for other python projects as it is so useful.
+
+```bash
+# Install UV
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# It can be installed via pip if that fails
+# pip install uv
+
+# Create a virtual environment in the current directory
+uv venv
+
+# Install the tool in the virtual environment 
+uv pip install ngiab_data_preprocess
+
+# To run the cli
+uv run cli --help
+
+# To run the map 
+uv run map_app
+```
+
+UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
+
+### Running without install
+This package supports pipx and uvx which means you can run the tool without installing it. No virtual environment needed, just UV.
+```bash
+# run this from anywhere 
+uvx --from ngiab_data_preprocess cli --help
+# for the map
+uvx --from ngiab_data_preprocess map_app
+```
+
+## For legacy pip installation
+<details>
+  <summary>Click here to expand</summary>
 
 ```bash
 # If you're installing this on jupyterhub / 2i2c you HAVE TO DEACTIVATE THE CONDA ENV
@@ -61,8 +96,7 @@ pip install 'ngiab_data_preprocess'
 python -m map_app
 # CLI instructions at the bottom of the README
 ```
-
-The first time you run this command, it will download the hydrofabric from Lynker Spatial. If you already have it, place `conus_nextgen.gpkg` into `~/.ngiab/hydrofabric/v2.2/`.
+</details>
 
 ## Development Installation
 
@@ -76,24 +110,23 @@ To install and run the tool, follow these steps:
    git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
    cd NGIAB_data_preprocess
    ```
-2. Create a virtual environment and activate it:
+2. Create a virtual environment:
    ```bash
-   python3 -m venv env
-   source env/bin/activate
+   uv venv
    ```
 3. Install the tool:
    ```bash
-   pip install -e .
+   uv pip install -e .
    ```
 4. Run the map app:
    ```bash
-   python -m map_app
+   uv run map_app
    ```
 </details>
 
 ## Usage
 
-Running the command `python -m map_app` will open the app in a new browser tab.
+Running the command `uv run map_app` will open the app in a new browser tab.
 
 To use the tool:
 1. Select the catchment you're interested in on the map.

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/data_processing/create_realization.py

@@ -4,6 +4,7 @@ import sqlite3
 from datetime import datetime
 from pathlib import Path
 import shutil
+import requests
 
 import pandas
 import s3fs
@@ -309,18 +310,37 @@ def create_em_realization(cat_id: str, start_time: datetime, end_time: datetime)
     create_partitions(paths)
 
 
-def create_realization(cat_id: str, start_time: datetime, end_time: datetime, use_nwm_gw: bool = False):
+def create_realization(
+    cat_id: str,
+    start_time: datetime,
+    end_time: datetime,
+    use_nwm_gw: bool = False,
+    gage_id: str = None,
+):
     paths = file_paths(cat_id)
 
-    # get approximate groundwater levels from nwm output
     template_path = paths.template_cfe_nowpm_realization_config
-    
+
+    if gage_id is not None:
+        # try and download s3:communityhydrofabric/hydrofabrics/community/gage_parameters/gage_id
+        # if it doesn't exist, use the default
+        try:
+            url = f"https://communityhydrofabric.s3.us-east-1.amazonaws.com/hydrofabrics/community/gage_parameters/{gage_id}.json"
+
+            new_template = requests.get(url).json()
+            template_path = paths.config_dir / "calibrated_params.json"
+            with open(template_path, "w") as f:
+                json.dump(new_template, f)
+        except Exception as e:
+            logger.warning(f"Failed to download gage parameters")
+
     conf_df = get_model_attributes(paths.geopackage_path)
 
     if use_nwm_gw:
         gw_levels = get_approximate_gw_storage(paths, start_time)
     else:
         gw_levels = dict()
+
     make_cfe_config(conf_df, paths, gw_levels)
 
     make_noahowp_config(paths.config_dir, conf_df, start_time, end_time)

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/data_processing/gpkg_utils.py

@@ -317,15 +317,33 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     sql_query = f"SELECT * FROM '{table}' WHERE vpuid IN ({','.join(vpus)})"
     contents = source_db.execute(sql_query).fetchall()
 
+    if table == "network":
+        # Look for the network entry that has a toid not in the flowpath or nexus tables        
+        network_toids = [x[2] for x in contents]
+        print(f"Network toids: {len(network_toids)}")
+        sql = "SELECT id FROM flowpaths"  
+        flowpath_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Flowpath ids: {len(flowpath_ids)}")
+        sql = "SELECT id FROM nexus"
+        nexus_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Nexus ids: {len(nexus_ids)}")
+        bad_ids = set(network_toids) - set(flowpath_ids + nexus_ids)
+        print(bad_ids)
+        print(f"Removing {len(bad_ids)} network entries that are not in flowpaths or nexuses")
+        # id column is second after fid
+        contents = [x for x in contents if x[1] not in bad_ids]
+
     insert_data(dest_db, table, contents)
 
+
     if table in get_feature_tables(file_paths.conus_hydrofabric):
         fids = [str(x[0]) for x in contents]
         copy_rTree_tables(table, fids, source_db, dest_db)
 
     dest_db.commit()
     source_db.close()
-    dest_db.close()
+    dest_db.close()   
+    
 
 def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name: Path) -> None:
     """

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/data_processing/subset.py

@@ -22,13 +22,16 @@ subset_tables = [
     "flowpath-attributes-ml",
     "flowpaths",
     "hydrolocations",
-    "network",
     "nexus",
     "pois",  # requires flowpaths
     "lakes",  # requires pois
+    "network",
 ]
 
-def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False) -> Path:
+
+def create_subset_gpkg(
+    ids: Union[List[str], str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False
+) -> Path:
     # ids is a list of nexus and wb ids, or a single vpu id
     if not isinstance(ids, list):
         ids = [ids]
@@ -42,12 +45,13 @@ def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg
     for table in subset_tables:
         if is_vpu:
             subset_table_by_vpu(table, ids[0], hydrofabric, output_gpkg_path)
-        else:                
+        else:
             subset_table(table, ids, hydrofabric, output_gpkg_path)
 
     add_triggers_to_gpkg(output_gpkg_path)
     update_geopackage_metadata(output_gpkg_path)
 
+
 def subset_vpu(vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_paths.conus_hydrofabric):
 
     if output_gpkg_path.exists():
@@ -77,6 +81,7 @@ def subset(
     logger.info(f"Subset complete for {len(upstream_ids)} features (catchments + nexuses)")
     logger.debug(f"Subset complete for {upstream_ids} catchments")
 
+
 def move_files_to_config_dir(subset_output_dir: str) -> None:
     config_dir = subset_output_dir / "config"
     config_dir.mkdir(parents=True, exist_ok=True)

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/ngiab_data_cli/__main__.py

@@ -169,13 +169,20 @@ def main() -> None:
 
         if args.realization:
             logging.info(f"Creating realization from {args.start_date} to {args.end_date}...")
+            gage_id = None
+            if args.gage:
+                gage_id = args.input_feature
             if args.empirical_model:
                 create_em_realization(
                     output_folder, start_time=args.start_date, end_time=args.end_date
                 )
             else:
                 create_realization(
-                    output_folder, start_time=args.start_date, end_time=args.end_date, use_nwm_gw=args.nwm_gw
+                    output_folder,
+                    start_time=args.start_date,
+                    end_time=args.end_date,
+                    use_nwm_gw=args.nwm_gw,
+                    gage_id=gage_id,
                 )
             logging.info("Realization creation complete.")
 
@@ -198,7 +205,7 @@ def main() -> None:
             except:
                 logging.error("Docker is not running, please start Docker and try again.")
             try:
-                #command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
+                # command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
                 command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {num_partitions} local'
                 subprocess.run(command, shell=True)
                 logging.info("Next Gen run complete.")

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.0.5
+Version: 4.1.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -8,11 +8,11 @@ Project-URL: Issues, https://github.com/CIROH-UA/NGIAB_data_preprocess/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.13,>=3.10
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pyogrio==0.7.2
-Requires-Dist: pyproj==3.6.1
+Requires-Dist: pyogrio>=0.7.2
+Requires-Dist: pyproj>=3.6.1
 Requires-Dist: Flask==3.0.2
 Requires-Dist: geopandas>=1.0.0
 Requires-Dist: requests==2.32.2
@@ -20,19 +20,20 @@ Requires-Dist: igraph==0.11.4
 Requires-Dist: s3fs==2024.3.1
 Requires-Dist: xarray==2024.2.0
 Requires-Dist: zarr==2.17.1
-Requires-Dist: netCDF4==1.6.5
+Requires-Dist: netCDF4>=1.6.5
 Requires-Dist: dask==2024.4.1
 Requires-Dist: dask[distributed]==2024.4.1
 Requires-Dist: black==24.3.0
 Requires-Dist: isort==5.13.2
 Requires-Dist: h5netcdf==1.3.0
 Requires-Dist: exactextract==0.2.0
-Requires-Dist: numpy==1.26.4
+Requires-Dist: numpy>=1.26.4
 Requires-Dist: tqdm==4.66.4
 Requires-Dist: rich==13.7.1
 Requires-Dist: colorama==0.4.6
 Requires-Dist: bokeh==3.5.1
 Requires-Dist: boto3
+Requires-Dist: numcodecs<0.16.0
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot
@@ -85,6 +86,41 @@ For automatic interactive visualisation, please run [NGIAB](https://github.com/C
 * This tool is officially supported on macOS or Ubuntu (tested on 22.04 & 24.04). To use it on Windows, please install [WSL](https://learn.microsoft.com/en-us/windows/wsl/install).
 
 ## Installation and Running
+It is highly recommended to use [Astral UV](https://docs.astral.sh/uv/) to install and run this tool. It works similarly to pip and conda, and I would also recommend you use it for other python projects as it is so useful.
+
+```bash
+# Install UV
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# It can be installed via pip if that fails
+# pip install uv
+
+# Create a virtual environment in the current directory
+uv venv
+
+# Install the tool in the virtual environment 
+uv pip install ngiab_data_preprocess
+
+# To run the cli
+uv run cli --help
+
+# To run the map 
+uv run map_app
+```
+
+UV automatically detects any virtual environments in the current directory and will use them when you use `uv run`.
+
+### Running without install
+This package supports pipx and uvx which means you can run the tool without installing it. No virtual environment needed, just UV.
+```bash
+# run this from anywhere 
+uvx --from ngiab_data_preprocess cli --help
+# for the map
+uvx --from ngiab_data_preprocess map_app
+```
+
+## For legacy pip installation
+<details>
+  <summary>Click here to expand</summary>
 
 ```bash
 # If you're installing this on jupyterhub / 2i2c you HAVE TO DEACTIVATE THE CONDA ENV
@@ -102,8 +138,7 @@ pip install 'ngiab_data_preprocess'
 python -m map_app
 # CLI instructions at the bottom of the README
 ```
-
-The first time you run this command, it will download the hydrofabric from Lynker Spatial. If you already have it, place `conus_nextgen.gpkg` into `~/.ngiab/hydrofabric/v2.2/`.
+</details>
 
 ## Development Installation
 
@@ -117,24 +152,23 @@ To install and run the tool, follow these steps:
    git clone https://github.com/CIROH-UA/NGIAB_data_preprocess
    cd NGIAB_data_preprocess
    ```
-2. Create a virtual environment and activate it:
+2. Create a virtual environment:
    ```bash
-   python3 -m venv env
-   source env/bin/activate
+   uv venv
    ```
 3. Install the tool:
    ```bash
-   pip install -e .
+   uv pip install -e .
    ```
 4. Run the map app:
    ```bash
-   python -m map_app
+   uv run map_app
    ```
 </details>
 
 ## Usage
 
-Running the command `python -m map_app` will open the app in a new browser tab.
+Running the command `uv run map_app` will open the app in a new browser tab.
 
 To use the tool:
 1. Select the catchment you're interested in on the map.

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/modules/ngiab_data_preprocess.egg-info/requires.txt

@@ -1,5 +1,5 @@
-pyogrio==0.7.2
-pyproj==3.6.1
+pyogrio>=0.7.2
+pyproj>=3.6.1
 Flask==3.0.2
 geopandas>=1.0.0
 requests==2.32.2
@@ -7,19 +7,20 @@ igraph==0.11.4
 s3fs==2024.3.1
 xarray==2024.2.0
 zarr==2.17.1
-netCDF4==1.6.5
+netCDF4>=1.6.5
 dask==2024.4.1
 dask[distributed]==2024.4.1
 black==24.3.0
 isort==5.13.2
 h5netcdf==1.3.0
 exactextract==0.2.0
-numpy==1.26.4
+numpy>=1.26.4
 tqdm==4.66.4
 rich==13.7.1
 colorama==0.4.6
 bokeh==3.5.1
 boto3
+numcodecs<0.16.0
 
 [eval]
 ngiab_eval

{ngiab_data_preprocess-4.0.5 → ngiab_data_preprocess-4.1.1}/pyproject.toml

@@ -12,19 +12,19 @@ exclude = ["tests*"]
 
 [project]
 name = "ngiab_data_preprocess"
-version = "v4.0.5"
+version = "v4.1.1"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"
-requires-python = ">=3.10,<3.13"
+requires-python = ">=3.10"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "pyogrio==0.7.2",
-    "pyproj==3.6.1",
+    "pyogrio>=0.7.2",
+    "pyproj>=3.6.1",
     "Flask==3.0.2",
     "geopandas>=1.0.0",
     "requests==2.32.2",
@@ -32,19 +32,20 @@ dependencies = [
     "s3fs==2024.3.1",
     "xarray==2024.2.0",
     "zarr==2.17.1",
-    "netCDF4==1.6.5",
+    "netCDF4>=1.6.5",
     "dask==2024.4.1",
     "dask[distributed]==2024.4.1",
     "black==24.3.0",
     "isort==5.13.2",
     "h5netcdf==1.3.0",
     "exactextract==0.2.0",
-    "numpy==1.26.4",
+    "numpy>=1.26.4",
     "tqdm==4.66.4",
     "rich==13.7.1",
     "colorama==0.4.6",
     "bokeh==3.5.1",
-    "boto3"
+    "boto3",
+    "numcodecs<0.16.0"
 ]
 
 [project.optional-dependencies]