ngiab-data-preprocess 4.0.4__tar.gz → 4.1.0__tar.gz

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.0.4
+Version: 4.1.0
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -37,6 +37,7 @@ Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot
 Requires-Dist: ngiab_eval[plot]; extra == "plot"
+Dynamic: license-file
 
 # NGIAB Data Preprocess
 

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/data_processing/create_realization.py

@@ -4,6 +4,7 @@ import sqlite3
 from datetime import datetime
 from pathlib import Path
 import shutil
+import requests
 
 import pandas
 import s3fs
@@ -309,18 +310,37 @@ def create_em_realization(cat_id: str, start_time: datetime, end_time: datetime)
     create_partitions(paths)
 
 
-def create_realization(cat_id: str, start_time: datetime, end_time: datetime, use_nwm_gw: bool = False):
+def create_realization(
+    cat_id: str,
+    start_time: datetime,
+    end_time: datetime,
+    use_nwm_gw: bool = False,
+    gage_id: str = None,
+):
     paths = file_paths(cat_id)
 
-    # get approximate groundwater levels from nwm output
     template_path = paths.template_cfe_nowpm_realization_config
-    
+
+    if gage_id is not None:
+        # try and download s3:communityhydrofabric/hydrofabrics/community/gage_parameters/gage_id
+        # if it doesn't exist, use the default
+        try:
+            url = f"https://communityhydrofabric.s3.us-east-1.amazonaws.com/hydrofabrics/community/gage_parameters/{gage_id}.json"
+
+            new_template = requests.get(url).json()
+            template_path = paths.config_dir / "calibrated_params.json"
+            with open(template_path, "w") as f:
+                json.dump(new_template, f)
+        except Exception as e:
+            logger.warning(f"Failed to download gage parameters")
+
     conf_df = get_model_attributes(paths.geopackage_path)
 
     if use_nwm_gw:
         gw_levels = get_approximate_gw_storage(paths, start_time)
     else:
         gw_levels = dict()
+
     make_cfe_config(conf_df, paths, gw_levels)
 
     make_noahowp_config(paths.config_dir, conf_df, start_time, end_time)

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/data_processing/gpkg_utils.py

@@ -317,15 +317,33 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     sql_query = f"SELECT * FROM '{table}' WHERE vpuid IN ({','.join(vpus)})"
     contents = source_db.execute(sql_query).fetchall()
 
+    if table == "network":
+        # Look for the network entry that has a toid not in the flowpath or nexus tables        
+        network_toids = [x[2] for x in contents]
+        print(f"Network toids: {len(network_toids)}")
+        sql = "SELECT id FROM flowpaths"  
+        flowpath_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Flowpath ids: {len(flowpath_ids)}")
+        sql = "SELECT id FROM nexus"
+        nexus_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
+        print(f"Nexus ids: {len(nexus_ids)}")
+        bad_ids = set(network_toids) - set(flowpath_ids + nexus_ids)
+        print(bad_ids)
+        print(f"Removing {len(bad_ids)} network entries that are not in flowpaths or nexuses")
+        # id column is second after fid
+        contents = [x for x in contents if x[1] not in bad_ids]
+
     insert_data(dest_db, table, contents)
 
+
     if table in get_feature_tables(file_paths.conus_hydrofabric):
         fids = [str(x[0]) for x in contents]
         copy_rTree_tables(table, fids, source_db, dest_db)
 
     dest_db.commit()
     source_db.close()
-    dest_db.close()
+    dest_db.close()   
+    
 
 def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name: Path) -> None:
     """

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/data_processing/subset.py

@@ -22,13 +22,16 @@ subset_tables = [
     "flowpath-attributes-ml",
     "flowpaths",
     "hydrolocations",
-    "network",
     "nexus",
     "pois",  # requires flowpaths
     "lakes",  # requires pois
+    "network",
 ]
 
-def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False) -> Path:
+
+def create_subset_gpkg(
+    ids: Union[List[str], str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False
+) -> Path:
     # ids is a list of nexus and wb ids, or a single vpu id
     if not isinstance(ids, list):
         ids = [ids]
@@ -42,12 +45,13 @@ def create_subset_gpkg(ids: Union[List[str],str], hydrofabric: Path, output_gpkg
     for table in subset_tables:
         if is_vpu:
             subset_table_by_vpu(table, ids[0], hydrofabric, output_gpkg_path)
-        else:                
+        else:
             subset_table(table, ids, hydrofabric, output_gpkg_path)
 
     add_triggers_to_gpkg(output_gpkg_path)
     update_geopackage_metadata(output_gpkg_path)
 
+
 def subset_vpu(vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_paths.conus_hydrofabric):
 
     if output_gpkg_path.exists():
@@ -77,6 +81,7 @@ def subset(
     logger.info(f"Subset complete for {len(upstream_ids)} features (catchments + nexuses)")
     logger.debug(f"Subset complete for {upstream_ids} catchments")
 
+
 def move_files_to_config_dir(subset_output_dir: str) -> None:
     config_dir = subset_output_dir / "config"
     config_dir.mkdir(parents=True, exist_ok=True)

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/map_app/templates/index.html

@@ -10,9 +10,9 @@
     <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}">
     <link rel="stylesheet" href="{{ url_for('static', filename='css/toggle.css') }}">
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
-    <script src="https://unpkg.com/maplibre-gl@^4.7.1/dist/maplibre-gl.js"></script>
-    <script src="https://unpkg.com/pmtiles@3.0.6/dist/pmtiles.js"></script>
-    <link href="https://unpkg.com/maplibre-gl@^4.7.1/dist/maplibre-gl.css" rel="stylesheet" />
+    <script src="https://unpkg.com/maplibre-gl@^5.2.0/dist/maplibre-gl.js"></script>
+    <script src="https://unpkg.com/pmtiles@latest/dist/pmtiles.js"></script>
+    <link href="https://unpkg.com/maplibre-gl@^5.2.0/dist/maplibre-gl.css" rel="stylesheet" />
 </head>
 
 <body>
@@ -130,4 +130,4 @@
 
 </body>
 
-</html>
+</html>

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/ngiab_data_cli/__main__.py

@@ -169,13 +169,20 @@ def main() -> None:
 
         if args.realization:
             logging.info(f"Creating realization from {args.start_date} to {args.end_date}...")
+            gage_id = None
+            if args.gage:
+                gage_id = args.input_feature
             if args.empirical_model:
                 create_em_realization(
                     output_folder, start_time=args.start_date, end_time=args.end_date
                 )
             else:
                 create_realization(
-                    output_folder, start_time=args.start_date, end_time=args.end_date, use_nwm_gw=args.nwm_gw
+                    output_folder,
+                    start_time=args.start_date,
+                    end_time=args.end_date,
+                    use_nwm_gw=args.nwm_gw,
+                    gage_id=gage_id,
                 )
             logging.info("Realization creation complete.")
 
@@ -198,7 +205,7 @@ def main() -> None:
             except:
                 logging.error("Docker is not running, please start Docker and try again.")
             try:
-                #command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
+                # command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
                 command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {num_partitions} local'
                 subprocess.run(command, shell=True)
                 logging.info("Next Gen run complete.")

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.0.4
+Version: 4.1.0
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -37,6 +37,7 @@ Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot
 Requires-Dist: ngiab_eval[plot]; extra == "plot"
+Dynamic: license-file
 
 # NGIAB Data Preprocess
 

{ngiab_data_preprocess-4.0.4 → ngiab_data_preprocess-4.1.0}/pyproject.toml

@@ -12,7 +12,7 @@ exclude = ["tests*"]
 
 [project]
 name = "ngiab_data_preprocess"
-version = "v4.0.4"
+version = "v4.1.0"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"