ngiab-data-preprocess 4.1.1__tar.gz → 4.2.1__tar.gz

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.1.1
+Version: 4.2.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/data_processing/datasets.py

@@ -85,3 +85,35 @@ def load_aorc_zarr(start_year: int = None, end_year: int = None) -> xr.Dataset:
 
     validate_dataset_format(dataset)
     return dataset
+
+
+def load_swe_zarr() -> xr.Dataset:
+    """Load the swe zarr dataset from S3."""
+    s3_urls = [
+        f"s3://noaa-nwm-retrospective-3-0-pds/CONUS/zarr/ldasout.zarr"
+    ]
+    # default cache is readahead which is detrimental to performance in this case
+    fs = S3ParallelFileSystem(anon=True, default_cache_type="none")  # default_block_size
+    s3_stores = [s3fs.S3Map(url, s3=fs) for url in s3_urls]
+    # the cache option here just holds accessed data in memory to prevent s3 being queried multiple times
+    # most of the data is read once and written to disk but some of the coordinate data is read multiple times
+    dataset = xr.open_mfdataset(s3_stores, parallel=True, engine="zarr", cache=True)
+    
+    # set the crs attribute to conform with the format
+    esri_pe_string = dataset.crs.esri_pe_string
+    dataset = dataset.drop_vars(["crs"])
+    dataset.attrs["crs"] = esri_pe_string
+    # drop everything except SNEQV
+    vars_to_drop = list(dataset.data_vars)
+    vars_to_drop.remove('SNEQV')
+    dataset = dataset.drop_vars(vars_to_drop)
+    dataset.attrs["name"] = "v3_swe_zarr"
+
+    # rename the data vars to work with ngen
+    variables = {
+        "SNEQV": "swe"
+    }
+    dataset = dataset.rename_vars(variables)
+
+    validate_dataset_format(dataset)
+    return dataset

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/data_processing/gpkg_utils.py

@@ -69,7 +69,7 @@ def verify_indices(gpkg: str = file_paths.conus_hydrofabric) -> None:
     con.close()
 
 
-def create_empty_gpkg(gpkg: str) -> None:
+def create_empty_gpkg(gpkg: Path) -> None:
     """
     Create an empty geopackage with the necessary tables and indices.
     """
@@ -80,7 +80,7 @@ def create_empty_gpkg(gpkg: str) -> None:
         conn.executescript(sql_script)
 
 
-def add_triggers_to_gpkg(gpkg: str) -> None:
+def add_triggers_to_gpkg(gpkg: Path) -> None:
     """
     Adds geopackage triggers required to maintain spatial index integrity
     """
@@ -256,7 +256,7 @@ def insert_data(con: sqlite3.Connection, table: str, contents: List[Tuple]) -> N
     con.commit()
 
 
-def update_geopackage_metadata(gpkg: str) -> None:
+def update_geopackage_metadata(gpkg: Path) -> None:
     """
     Update the contents of the gpkg_contents table in the specified geopackage.
     """
@@ -290,6 +290,7 @@ def update_geopackage_metadata(gpkg: str) -> None:
 
     con.close()
 
+
 def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_name: Path) -> None:
     """
     Subset the specified table from the hydrofabric database by vpuid and save it to the subset geopackage.
@@ -306,9 +307,9 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     dest_db = sqlite3.connect(subset_gpkg_name)
 
     if vpu == "03":
-        vpus = ["03N","03S","03W"]
+        vpus = ["03N", "03S", "03W"]
     elif vpu == "10":
-        vpus = ["10L","10U"]
+        vpus = ["10L", "10U"]
     else:
         vpus = [vpu]
 
@@ -318,10 +319,10 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
     contents = source_db.execute(sql_query).fetchall()
 
     if table == "network":
-        # Look for the network entry that has a toid not in the flowpath or nexus tables        
+        # Look for the network entry that has a toid not in the flowpath or nexus tables
         network_toids = [x[2] for x in contents]
         print(f"Network toids: {len(network_toids)}")
-        sql = "SELECT id FROM flowpaths"  
+        sql = "SELECT id FROM flowpaths"
         flowpath_ids = [x[0] for x in dest_db.execute(sql).fetchall()]
         print(f"Flowpath ids: {len(flowpath_ids)}")
         sql = "SELECT id FROM nexus"
@@ -335,15 +336,14 @@ def subset_table_by_vpu(table: str, vpu: str, hydrofabric: Path, subset_gpkg_nam
 
     insert_data(dest_db, table, contents)
 
-
     if table in get_feature_tables(file_paths.conus_hydrofabric):
         fids = [str(x[0]) for x in contents]
         copy_rTree_tables(table, fids, source_db, dest_db)
 
     dest_db.commit()
     source_db.close()
-    dest_db.close()   
-    
+    dest_db.close()
+
 
 def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name: Path) -> None:
     """
@@ -359,7 +359,7 @@ def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name
     source_db = sqlite3.connect(f"file:{hydrofabric}?mode=ro", uri=True)
     dest_db = sqlite3.connect(subset_gpkg_name)
 
-    table_keys = {"divides": "toid", "divide-attributes": "divide_id", "lakes": "poi_id"}
+    table_keys = {"divide-attributes": "divide_id", "lakes": "poi_id"}
 
     if table == "lakes":
         # lakes subset we get from the pois table which was already subset by water body id
@@ -372,12 +372,20 @@ def subset_table(table: str, ids: List[str], hydrofabric: Path, subset_gpkg_name
         sql_query = "SELECT divide_id FROM 'divides'"
         contents = dest_db.execute(sql_query).fetchall()
         ids = [str(x[0]) for x in contents]
+
+    if table == "nexus":
+        # add the nexuses in the toid column from the flowpaths table
+        sql_query = "SELECT toid FROM 'flowpaths'"
+        contents = dest_db.execute(sql_query).fetchall()
+        new_ids = [str(x[0]) for x in contents]
+        ids.extend(new_ids)
+
     ids = [f"'{x}'" for x in ids]
     key_name = "id"
     if table in table_keys:
         key_name = table_keys[table]
     sql_query = f"SELECT * FROM '{table}' WHERE {key_name} IN ({','.join(ids)})"
-    contents = source_db.execute(sql_query).fetchall()    
+    contents = source_db.execute(sql_query).fetchall()
 
     insert_data(dest_db, table, contents)
 
@@ -409,6 +417,7 @@ def get_table_crs_short(gpkg: str, table: str) -> str:
         crs = con.execute(sql_query).fetchone()[0]
     return crs
 
+
 def get_table_crs(gpkg: str, table: str) -> str:
     """
     Get the CRS of the specified table in the specified geopackage.
@@ -429,16 +438,16 @@ def get_table_crs(gpkg: str, table: str) -> str:
 
 def get_cat_from_gage_id(gage_id: str, gpkg: Path = file_paths.conus_hydrofabric) -> str:
     """
-    Get the nexus id of associated with a gage id.
+    Get the catchment id associated with a gage id.
 
     Args:
         gage_id (str): The gage ID.
 
     Returns:
-        str: The nexus id of the watershed containing the gage ID.
+        str: The catchment id of the watershed containing the gage ID.
 
     Raises:
-        IndexError: If nexus is found for the given gage ID.
+        IndexError: If catchment is found for the given gage ID.
 
     """
     gage_id = "".join([x for x in gage_id if x.isdigit()])
@@ -510,7 +519,6 @@ def get_available_tables(gpkg: Path) -> List[str]:
 
 
 def get_cat_to_nhd_feature_id(gpkg: Path = file_paths.conus_hydrofabric) -> dict:
-
     available_tables = get_available_tables(gpkg)
     possible_tables = ["flowpath_edge_list", "network"]
 

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/data_processing/graph_utils.py

@@ -165,6 +165,10 @@ def get_upstream_cats(names: Union[str, List[str]]) -> Set[str]:
         if name in parent_ids:
             continue
         try:
+            if "cat" in name:
+                node_index = graph.vs.find(cat=name).index
+            else:
+                node_index = graph.vs.find(name=name).index
             node_index = graph.vs.find(cat=name).index
             upstream_nodes = graph.subcomponent(node_index, mode="IN")
             for node in upstream_nodes:
@@ -205,7 +209,10 @@ def get_upstream_ids(names: Union[str, List[str]], include_outlet: bool = True)
         if name in parent_ids:
             continue
         try:
-            node_index = graph.vs.find(name=name).index
+            if "cat" in name:
+                node_index = graph.vs.find(cat=name).index
+            else:
+                node_index = graph.vs.find(name=name).index
             upstream_nodes = graph.subcomponent(node_index, mode="IN")
             for node in upstream_nodes:
                 parent_ids.add(graph.vs[node]["name"])

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/data_processing/subset.py

@@ -22,7 +22,7 @@ subset_tables = [
     "flowpath-attributes-ml",
     "flowpaths",
     "hydrolocations",
-    "nexus",
+    "nexus",  # depends on flowpaths in some cases e.g. gage delineation
     "pois",  # requires flowpaths
     "lakes",  # requires pois
     "network",
@@ -31,7 +31,7 @@ subset_tables = [
 
 def create_subset_gpkg(
     ids: Union[List[str], str], hydrofabric: Path, output_gpkg_path: Path, is_vpu: bool = False
-) -> Path:
+):
     # ids is a list of nexus and wb ids, or a single vpu id
     if not isinstance(ids, list):
         ids = [ids]
@@ -52,8 +52,9 @@ def create_subset_gpkg(
     update_geopackage_metadata(output_gpkg_path)
 
 
-def subset_vpu(vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_paths.conus_hydrofabric):
-
+def subset_vpu(
+    vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_paths.conus_hydrofabric
+):
     if output_gpkg_path.exists():
         os.remove(output_gpkg_path)
 
@@ -65,10 +66,11 @@ def subset_vpu(vpu_id: str, output_gpkg_path: Path, hydrofabric: Path = file_pat
 def subset(
     cat_ids: List[str],
     hydrofabric: Path = file_paths.conus_hydrofabric,
-    output_gpkg_path: Path = None,
-) -> str:
-
-    upstream_ids = list(get_upstream_ids(cat_ids))
+    output_gpkg_path: Path = Path(),
+    include_outlet: bool = True,
+):
+    print(cat_ids)
+    upstream_ids = list(get_upstream_ids(cat_ids, include_outlet))
 
     if not output_gpkg_path:
         # if the name isn't provided, use the first upstream id
@@ -80,15 +82,3 @@ def subset(
     create_subset_gpkg(upstream_ids, hydrofabric, output_gpkg_path)
     logger.info(f"Subset complete for {len(upstream_ids)} features (catchments + nexuses)")
     logger.debug(f"Subset complete for {upstream_ids} catchments")
-
-
-def move_files_to_config_dir(subset_output_dir: str) -> None:
-    config_dir = subset_output_dir / "config"
-    config_dir.mkdir(parents=True, exist_ok=True)
-
-    files = [x for x in subset_output_dir.iterdir()]
-    for file in files:
-        if file.suffix in [".csv", ".json", ".geojson"]:
-            if "partitions" in file.name:
-                continue
-            os.system(f"mv {file} {config_dir}")

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/ngiab_data_cli/__main__.py

@@ -147,7 +147,10 @@ def main() -> None:
                 logging.info("Subsetting complete.")
             else:
                 logging.info(f"Subsetting hydrofabric")
-                subset(feature_to_subset, output_gpkg_path=paths.geopackage_path)
+                include_outlet = True
+                if args.gage:
+                    include_outlet = False
+                subset(feature_to_subset, output_gpkg_path=paths.geopackage_path, include_outlet=include_outlet)
                 logging.info("Subsetting complete.")
 
         if args.forcings:
@@ -237,7 +240,7 @@ def main() -> None:
 
         if args.vis:
             try:
-                command = f'docker run --rm -it -p 3000:3000 -v "{str(paths.subset_dir)}:/ngen/ngen/data/" joshcu/ngiab_grafana:v0.2.0'
+                command = f'docker run --rm -it -p 3000:3000 -v "{str(paths.subset_dir)}:/ngen/ngen/data/" joshcu/ngiab_grafana:v0.2.1'
                 subprocess.run(command, shell=True)
             except:
                 logging.error("Failed to launch docker container.")

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.1.1
+Version: 4.2.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-4.1.1 → ngiab_data_preprocess-4.2.1}/pyproject.toml

@@ -12,7 +12,7 @@ exclude = ["tests*"]
 
 [project]
 name = "ngiab_data_preprocess"
-version = "v4.1.1"
+version = "v4.2.1"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"