ngiab-data-preprocess 4.6.8__tar.gz → 4.7.0__tar.gz

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.6.8
+Version: 4.7.0
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -33,6 +33,7 @@ Requires-Dist: bokeh==3.8.2
 Requires-Dist: boto3
 Requires-Dist: numcodecs<0.16.0
 Requires-Dist: scipy>=1.15.3
+Requires-Dist: pyarrow
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/data_processing/create_realization.py

@@ -20,6 +20,7 @@ from data_processing.gpkg_utils import (
     get_cat_to_nhd_feature_id,
     get_table_crs_short,
 )
+from data_sources.source_validation import download_dhbv_attributes
 from pyproj import Transformer
 from tqdm.rich import tqdm
 
@@ -112,19 +113,20 @@ def make_noahowp_config(
             )
 
 
-def get_model_attributes(hydrofabric: Path) -> pandas.DataFrame:
+def get_model_attributes(hydrofabric: Path, layer: str = "divides") -> pandas.DataFrame:
     with sqlite3.connect(hydrofabric) as conn:
         conf_df = pandas.read_sql_query(
             """
             SELECT
             d.areasqkm,
+            d.lengthkm,
             da.*
             FROM divides AS d
             JOIN 'divide-attributes' AS da ON d.divide_id = da.divide_id
             """,
             conn,
         )
-    source_crs = get_table_crs_short(hydrofabric, "divides")
+    source_crs = get_table_crs_short(hydrofabric, layer)
     transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
     lon, lat = transformer.transform(conf_df["centroid_x"].values, conf_df["centroid_y"].values)
     conf_df["longitude"] = lon
@@ -172,6 +174,44 @@ def make_lstm_config(
             )
 
 
+def make_dhbv2_config(
+    hydrofabric: Path,
+    output_dir: Path,
+    start_time: datetime,
+    end_time: datetime,
+    template_path: Path = FilePaths.template_dhbv2_config,
+):
+    divide_conf_df = get_model_attributes(hydrofabric)
+    divide_ids = divide_conf_df["divide_id"].to_list()
+
+    download_dhbv_attributes()
+    dhbv_atts = pandas.read_parquet(FilePaths.dhbv_attributes)
+    atts_df = dhbv_atts.loc[dhbv_atts["divide_id"].isin(divide_ids)]
+
+    cat_config_dir = output_dir / "cat_config" / "dhbv2"
+    if cat_config_dir.exists():
+        shutil.rmtree(cat_config_dir)
+    cat_config_dir.mkdir(parents=True, exist_ok=True)
+
+    with open(template_path, "r") as file:
+        template = file.read()
+
+    for _, row in atts_df.iterrows():
+        divide = row["divide_id"]
+        divide_conf_df_row = divide_conf_df.loc[divide_conf_df["divide_id"] == divide]
+
+        with open(cat_config_dir / f"{divide}.yml", "w") as file:
+            file.write(
+                template.format(
+                    **row,
+                    catchsize=divide_conf_df_row["areasqkm"].values[0],
+                    lengthkm=divide_conf_df_row["lengthkm"].values[0],
+                    start_time=start_time,
+                    end_time=end_time,
+                )
+            )
+
+
 def configure_troute(
     cat_id: str, config_dir: Path, start_time: datetime, end_time: datetime
 ) -> None:
@@ -218,7 +258,10 @@ def configure_troute(
 
 
 def make_ngen_realization_json(
-    config_dir: Path, template_path: Path, start_time: datetime, end_time: datetime
+    config_dir: Path,
+    template_path: Path,
+    start_time: datetime,
+    end_time: datetime,
 ) -> None:
     with open(template_path, "r") as file:
         realization = json.load(file)
@@ -252,7 +295,20 @@ def create_lstm_realization(
         (paths.config_dir / "python_lstm_real.json").rename(realization_path)
 
     make_lstm_config(paths.geopackage_path, paths.config_dir)
-    # create some partitions for parallelization
+    paths.setup_run_folders()
+
+
+def create_dhbv2_realization(cat_id: str, start_time: datetime, end_time: datetime):
+    paths = FilePaths(cat_id)
+    configure_troute(cat_id, paths.config_dir, start_time, end_time)
+
+    make_ngen_realization_json(
+        paths.config_dir,
+        FilePaths.template_dhbv2_realization_config,
+        start_time,
+        end_time,
+    )
+    make_dhbv2_config(paths.geopackage_path, paths.config_dir, start_time, end_time)
     paths.setup_run_folders()
 
 

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/data_processing/file_paths.py

@@ -22,6 +22,7 @@ class FilePaths:
     template_sql = data_sources / "template.sql"
     triggers_sql = data_sources / "triggers.sql"
     conus_hydrofabric = hydrofabric_dir / "conus_nextgen.gpkg"
+    dhbv_attributes = hydrofabric_dir / "dhbv_attrs.parquet"
     hydrofabric_graph = hydrofabric_dir / "conus_igraph_network.gpickle"
     template_nc = data_sources / "forcing_template.nc"
     dev_file = Path(__file__).parent.parent.parent / ".dev"
@@ -32,6 +33,8 @@ class FilePaths:
     template_noahowp_config = data_sources / "noah-owp-modular-init.namelist.input"
     template_cfe_config = data_sources / "cfe-template.ini"
     template_lstm_config = data_sources / "lstm-catchment-template.yml"
+    template_dhbv2_realization_config = data_sources / "dhbv2-realization-template.json"
+    template_dhbv2_config = data_sources / "dhbv2-catchment-template.yaml"
 
     def __init__(self, folder_name: Optional[str] = None, output_dir: Optional[Path] = None):
         """

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/data_processing/forcings.py

@@ -117,6 +117,7 @@ def get_cell_weights(raster: xr.Dataset, gdf: gpd.GeoDataFrame, wkt: str) -> pd.
     )  # type: ignore
     return output.set_index("divide_id")
 
+
 def add_APCP_SURFACE_to_dataset(dataset: xr.Dataset) -> xr.Dataset:
     """Convert precipitation value to correct units."""
     # precip_rate is mm/s
@@ -311,6 +312,7 @@ def get_units(dataset: xr.Dataset) -> dict:
             units[var] = dataset[var].attrs["units"]
     return units
 
+
 def interpolate_nan_values(
     dataset: xr.Dataset,
     dim: str = "time",
@@ -383,9 +385,12 @@ def compute_zonal_stats(
     progress_file = FilePaths(output_dir=forcings_dir.parent).forcing_progress_file
     ex_var_name = list(gridded_data.data_vars)[0]
     example_time_chunks = get_index_chunks(gridded_data[ex_var_name])
-    all_steps = len(example_time_chunks) * len(gridded_data.data_vars)
+
+    data_vars = gridded_data.data_vars
+
+    all_steps = len(example_time_chunks) * len(data_vars)
     logger.info(
-        f"Total steps: {all_steps}, Number of time chunks: {len(example_time_chunks)}, Number of variables: {len(gridded_data.data_vars)}"
+        f"Total steps: {all_steps}, Number of time chunks: {len(example_time_chunks)}, Number of variables: {len(data_vars)}"
     )
     steps_completed = 0
     with open(progress_file, "w") as f:
@@ -405,10 +410,11 @@ def compute_zonal_stats(
 
     timer = time.perf_counter()
     variable_task = progress.add_task(
-        "[cyan]Processing variables...", total=len(gridded_data.data_vars), elapsed=0
+        "[cyan]Processing variables...", total=len(data_vars), elapsed=0
     )
     progress.start()
-    for data_var_name in list(gridded_data.data_vars):
+
+    for data_var_name in list(data_vars):
         data_var_name: str
         progress.update(variable_task, advance=1)
         progress.update(variable_task, description=f"Processing {data_var_name}")
@@ -524,6 +530,7 @@ def write_outputs(forcings_dir: Path, units: dict) -> None:
     # time is stored as unix timestamps, units have to be set
     # add the catchment ids as a 1d data var
     final_ds["ids"] = final_ds["catchment"].astype(str)
+
     # time needs to be a 2d array of the same time array as unix timestamps for every catchment
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -575,5 +582,7 @@ def create_forcings(dataset: xr.Dataset, output_folder_name: str) -> None:
     gdf = gpd.read_file(forcing_paths.geopackage_path, layer="divides")
     logger.debug(f"gdf  bounds: {gdf.total_bounds}")
     gdf = gdf.to_crs(dataset.crs)
-    dataset = dataset.isel(y=slice(None, None, -1)) # Flip y-axis: source data has y ordered from top-to-bottom (as in image arrays), but geospatial operations expect y to increase from bottom-to-top (increasing latitude).
+    dataset = dataset.isel(
+        y=slice(None, None, -1)
+    )  # Flip y-axis: source data has y ordered from top-to-bottom (as in image arrays), but geospatial operations expect y to increase from bottom-to-top (increasing latitude).
     compute_zonal_stats(gdf, dataset, forcing_paths.forcings_dir)

ngiab_data_preprocess-4.7.0/modules/data_sources/dhbv2-catchment-template.yaml

@@ -0,0 +1,46 @@
+### Single-catchment configuration ###
+catchment_id: {divide_id}
+catchment_name: {divide_id}
+
+# Static catchment attributes
+aridity: {aridity}
+meanP: {meanP}
+ETPOT_Hargr: {ETPOT_Hargr}
+NDVI: {NDVI}
+FW: {FW}
+meanslope: {meanslope}
+SoilGrids1km_sand: {SoilGrids1km_sand}
+SoilGrids1km_clay: {SoilGrids1km_clay}
+SoilGrids1km_silt: {SoilGrids1km_silt}
+glaciers: {glaciers}
+HWSD_clay: {HWSD_clay}
+HWSD_gravel: {HWSD_gravel}
+HWSD_sand: {HWSD_sand}
+HWSD_silt: {HWSD_silt}
+meanelevation: {meanelevation}
+meanTa: {meanTa}
+permafrost: {permafrost}
+permeability: {permeability}
+seasonality_P: {seasonality_P}
+seasonality_PET: {seasonality_PET}
+snow_fraction: {snow_fraction}
+snowfall_fraction: {snowfall_fraction}
+T_clay: {T_clay}
+T_gravel: {T_gravel}
+T_sand: {T_sand}
+T_silt: {T_silt}
+Porosity: {Porosity}
+uparea: {uparea}
+catchsize: {catchsize}
+lengthkm: {lengthkm}
+
+model_dir: /ngen/ngen/extern/dhbv2/ngen_resources/data/dhbv_2_mts/model/dhbv_2_mts
+stepwise: True  # True: stepwise inference, False: Single forward simulation on all data in one go
+initial_state: zero  # zero: set initial states of the model to zero
+dtype: float32
+verbose: 1  # 0: no output, 1: print output
+
+# Simulation window
+time_step: 1 hour
+start_time: {start_time}
+end_time: {end_time}

ngiab_data_preprocess-4.7.0/modules/data_sources/dhbv2-realization-template.json

@@ -0,0 +1,56 @@
+{
+  "global": {
+    "formulations": [
+      {
+        "name": "bmi_multi",
+        "params": {
+          "model_type_name": "dhbv2",
+          "forcing_file": "",
+          "init_config": "",
+          "allow_exceed_end_time": true,
+          "main_output_variable": "land_surface_water__runoff_volume_flux",
+          "modules": [
+            {
+              "name": "bmi_python",
+              "params": {
+                "python_type": "dhbv2.mts_bmi.MtsDeltaModelBmi",
+                "model_type_name": "DeltaModelBmi",
+                "init_config": "./config/cat_config/dhbv2/{{id}}.yml",
+                "allow_exceed_end_time": true,
+                "main_output_variable": "land_surface_water__runoff_volume_flux",
+                "fixed_time_step": false,
+                "uses_forcing_file": false,
+                "variables_names_map" : {
+                  "atmosphere_water__liquid_equivalent_precipitation_rate":"precip_rate",
+                  "land_surface_air__temperature":"TMP_2maboveground",
+                  "atmosphere_air_water~vapor__relative_saturation":"SPFH_2maboveground",
+                  "land_surface_radiation~incoming~longwave__energy_flux":"DLWRF_surface",
+                  "land_surface_radiation~incoming~shortwave__energy_flux":"DSWRF_surface",
+                  "land_surface_air__pressure":"PRES_surface",
+                  "land_surface_wind__x_component_of_velocity":"UGRD_10maboveground",
+                  "land_surface_wind__y_component_of_velocity":"VGRD_10maboveground",
+                  "land_surface_water__runoff_volume_flux":"streamflow_cms"
+
+                }
+
+              }
+            }
+          ]
+        }
+      }
+    ],
+    "forcing": {
+      "path": "./forcings/forcings.nc",
+      "provider": "NetCDF"
+    }
+  },
+  "time": {
+    "start_time": "2000-01-01 00:00:00",
+    "end_time": "2005-12-30 23:00:00",
+    "output_interval": 3600
+  },
+  "routing": {
+    "t_route_config_file_with_path": "./config/troute.yaml"
+  },
+  "output_root": "./outputs/ngen"
+}

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/data_sources/source_validation.py

@@ -126,16 +126,40 @@ def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
         return False
 
 
-def get_headers():
+def get_headers(url: str = hydrofabric_url):
     # for versioning
     # Useful Headers: { 'Last-Modified': 'Wed, 20 Nov 2024 18:45:59 GMT', 'ETag': '"cc1452838886a7ab3065a61073fa991b-207"'}
     try:
-        response = requests.head(hydrofabric_url)
+        response = requests.head(url)
     except requests.exceptions.ConnectionError:
         return 500, {}
     return response.status_code, response.headers
 
 
+def download_dhbv_attributes():
+    s3_key = "hydrofabrics/community/resources/dhbv_attrs.parquet"
+    attributes_url = f"https://{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com/{s3_key}"
+
+    status, headers = get_headers(attributes_url)
+    download_log = FilePaths.dhbv_attributes.with_suffix(".log")
+    if download_log.exists():
+        with open(download_log, "r") as f:
+            local_headers = json.load(f)
+    else:
+        local_headers = {}
+
+    if not FilePaths.dhbv_attributes.exists() or headers.get("ETag", "") != local_headers.get(
+        "ETag", ""
+    ):
+        download_from_s3(
+            FilePaths.dhbv_attributes,
+            bucket=S3_BUCKET,
+            key=s3_key,
+        )
+        with open(FilePaths.dhbv_attributes.with_suffix(".log"), "w") as f:
+            json.dump(dict(headers), f)
+
+
 def download_and_update_hf():
     if FilePaths.conus_hydrofabric.is_file():
         console.print(

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/ngiab_data_cli/__main__.py

@@ -12,7 +12,11 @@ with rich.status.Status("loading") as status:
     from pathlib import Path
 
     import geopandas as gpd
-    from data_processing.create_realization import create_lstm_realization, create_realization
+    from data_processing.create_realization import (
+        create_dhbv2_realization,
+        create_lstm_realization,
+        create_realization,
+    )
     from data_processing.dask_utils import shutdown_cluster
     from data_processing.dataset_utils import save_and_clip_dataset
     from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
@@ -217,6 +221,12 @@ def main() -> None:
                     end_time=args.end_date,
                     use_rust=args.lstm_rust,
                 )
+            if args.dhbv2:
+                create_dhbv2_realization(
+                    output_folder,
+                    start_time=args.start_date,
+                    end_time=args.end_date,
+                )
             else:
                 create_realization(
                     output_folder,

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/ngiab_data_cli/arguments.py

@@ -92,13 +92,17 @@ def parse_arguments() -> argparse.Namespace:
     parser.add_argument(
         "--start_date",
         "--start",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT)
+        if len(s) == 10
+        else datetime.strptime(s, DATE_FORMAT2),
         help=f"Start date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
         "--end_date",
         "--end",
-        type=lambda s: datetime.strptime(s, DATE_FORMAT) if len(s) == 10 else datetime.strptime(s, DATE_FORMAT2),
+        type=lambda s: datetime.strptime(s, DATE_FORMAT)
+        if len(s) == 10
+        else datetime.strptime(s, DATE_FORMAT2),
         help=f"End date for forcings/realization (format {DATE_FORMAT_HINT})",
     )
     parser.add_argument(
@@ -124,6 +128,13 @@ def parse_arguments() -> argparse.Namespace:
         action="store_true",
         help="enable experimental high speed Rust bindings of LSTM model realization and forcings",
     )
+
+    parser.add_argument(
+        "--dhbv2",
+        action="store_true",
+        help="enable dHBV2 model realization and forcings",
+    )
+
     parser.add_argument(
         "--nwm_gw",
         action="store_true",

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.6.8
+Version: 4.7.0
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -33,6 +33,7 @@ Requires-Dist: bokeh==3.8.2
 Requires-Dist: boto3
 Requires-Dist: numcodecs<0.16.0
 Requires-Dist: scipy>=1.15.3
+Requires-Dist: pyarrow
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/ngiab_data_preprocess.egg-info/SOURCES.txt

@@ -18,6 +18,8 @@ modules/data_processing/s3fs_utils.py
 modules/data_processing/subset.py
 modules/data_sources/cfe-nowpm-realization-template.json
 modules/data_sources/cfe-template.ini
+modules/data_sources/dhbv2-catchment-template.yaml
+modules/data_sources/dhbv2-realization-template.json
 modules/data_sources/forcing_template.nc
 modules/data_sources/lstm-catchment-template.yml
 modules/data_sources/lstm-realization-template.json

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/modules/ngiab_data_preprocess.egg-info/requires.txt

@@ -20,6 +20,7 @@ bokeh==3.8.2
 boto3
 numcodecs<0.16.0
 scipy>=1.15.3
+pyarrow
 
 [eval]
 ngiab_eval

{ngiab_data_preprocess-4.6.8 → ngiab_data_preprocess-4.7.0}/pyproject.toml

@@ -19,7 +19,7 @@ filterwarnings = [
 ]
 [project]
 name = "ngiab_data_preprocess"
-version = "v4.6.8"
+version = "v4.7.0"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"
@@ -52,6 +52,7 @@ dependencies = [
     "boto3",
     "numcodecs<0.16.0",
     "scipy>=1.15.3",
+    "pyarrow",
 ]
 
 [project.optional-dependencies]