ngiab-data-preprocess 4.4.0__py3-none-any.whl → 4.5.1__py3-none-any.whl

data_processing/create_realization.py

@@ -1,23 +1,22 @@
 import json
 import logging
 import multiprocessing
+import os
 import shutil
 import sqlite3
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional
-import psutil
-import os
 
+import numpy as np
 import pandas
+import psutil
 import requests
 import s3fs
 import xarray as xr
 from data_processing.dask_utils import temp_cluster
 from data_processing.file_paths import file_paths
 from data_processing.gpkg_utils import (
-    GeoPackage,
-    get_cat_to_nex_flowpairs,
     get_cat_to_nhd_feature_id,
     get_table_crs_short,
 )
@@ -91,7 +90,6 @@ def make_cfe_config(
 def make_noahowp_config(
     base_dir: Path, divide_conf_df: pandas.DataFrame, start_time: datetime, end_time: datetime
 ) -> None:
-    divide_conf_df.set_index("divide_id", inplace=True)
     start_datetime = start_time.strftime("%Y%m%d%H%M")
     end_datetime = end_time.strftime("%Y%m%d%H%M")
     with open(file_paths.template_noahowp_config, "r") as file:
@@ -100,155 +98,78 @@ def make_noahowp_config(
     cat_config_dir = base_dir / "cat_config" / "NOAH-OWP-M"
     cat_config_dir.mkdir(parents=True, exist_ok=True)
 
-    for divide in divide_conf_df.index:
-        with open(cat_config_dir / f"{divide}.input", "w") as file:
+    for _, row in divide_conf_df.iterrows():
+        with open(cat_config_dir / f"{row['divide_id']}.input", "w") as file:
             file.write(
                 template.format(
                     start_datetime=start_datetime,
                     end_datetime=end_datetime,
-                    lat=divide_conf_df.loc[divide, "latitude"],
-                    lon=divide_conf_df.loc[divide, "longitude"],
-                    terrain_slope=divide_conf_df.loc[divide, "mean.slope_1km"],
-                    azimuth=divide_conf_df.loc[divide, "circ_mean.aspect"],
-                    ISLTYP=int(divide_conf_df.loc[divide, "mode.ISLTYP"]),  # type: ignore
-                    IVGTYP=int(divide_conf_df.loc[divide, "mode.IVGTYP"]),  # type: ignore
+                    lat=row["latitude"],
+                    lon=row["longitude"],
+                    terrain_slope=row["mean.slope_1km"],
+                    azimuth=row["circ_mean.aspect"],
+                    ISLTYP=int(row["mode.ISLTYP"]),  # type: ignore
+                    IVGTYP=int(row["mode.IVGTYP"]),  # type: ignore
                 )
             )
 
 
-def get_model_attributes_modspatialite(hydrofabric: Path) -> pandas.DataFrame:
-    # modspatialite is faster than pyproj but can't be added as a pip dependency
-    # This incantation took a while
-    with GeoPackage(hydrofabric) as conn:
-        sql = """WITH source_crs AS (
-        SELECT organization || ':' || organization_coordsys_id AS crs_string
-        FROM gpkg_spatial_ref_sys
-        WHERE srs_id = (
-            SELECT srs_id
-            FROM gpkg_geometry_columns
-            WHERE table_name = 'divides'
-        )
-        )
-        SELECT
-        d.divide_id,
-        d.areasqkm,
-        da."mean.slope",
-        da."mean.slope_1km",
-        da."mean.elevation",
-        ST_X(Transform(MakePoint(da.centroid_x, da.centroid_y), 4326, NULL,
-            (SELECT crs_string FROM source_crs), 'EPSG:4326')) AS longitude,
-        ST_Y(Transform(MakePoint(da.centroid_x, da.centroid_y), 4326, NULL,
-            (SELECT crs_string FROM source_crs), 'EPSG:4326')) AS latitude
-        FROM divides AS d
-        JOIN 'divide-attributes' AS da ON d.divide_id = da.divide_id
-        """
-        divide_conf_df = pandas.read_sql_query(sql, conn)
-    divide_conf_df.set_index("divide_id", inplace=True)
-    return divide_conf_df
-
-
-def get_model_attributes_pyproj(hydrofabric: Path) -> pandas.DataFrame:
-    # if modspatialite is not available, use pyproj
+def get_model_attributes(hydrofabric: Path) -> pandas.DataFrame:
     with sqlite3.connect(hydrofabric) as conn:
-        sql = """
-        SELECT
-        d.divide_id,
-        d.areasqkm,
-        da."mean.slope",
-        da."mean.slope_1km",
-        da."mean.elevation",
-        da.centroid_x,
-        da.centroid_y
-        FROM divides AS d
-        JOIN 'divide-attributes' AS da ON d.divide_id = da.divide_id
-        """
-        divide_conf_df = pandas.read_sql_query(sql, conn)
-
+        conf_df = pandas.read_sql_query(
+            """
+            SELECT
+            d.areasqkm,
+            da.*
+            FROM divides AS d
+            JOIN 'divide-attributes' AS da ON d.divide_id = da.divide_id
+            """,
+            conn,
+        )
     source_crs = get_table_crs_short(hydrofabric, "divides")
-
     transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
-
-    lon, lat = transformer.transform(
-        divide_conf_df["centroid_x"].values, divide_conf_df["centroid_y"].values
-    )
-
-    divide_conf_df["longitude"] = lon
-    divide_conf_df["latitude"] = lat
-
-    divide_conf_df.drop(columns=["centroid_x", "centroid_y"], axis=1, inplace=True)
-    divide_conf_df.set_index("divide_id", inplace=True)
-
-    return divide_conf_df
-
-
-def get_model_attributes(hydrofabric: Path) -> pandas.DataFrame:
-    try:
-        with GeoPackage(hydrofabric) as conn:
-            conf_df = pandas.read_sql_query(
-                """WITH source_crs AS (
-            SELECT organization || ':' || organization_coordsys_id AS crs_string
-            FROM gpkg_spatial_ref_sys
-            WHERE srs_id = (
-                SELECT srs_id
-                FROM gpkg_geometry_columns
-                WHERE table_name = 'divides'
-            )
-            )
-            SELECT
-            *,
-            ST_X(Transform(MakePoint(centroid_x, centroid_y), 4326, NULL,
-                (SELECT crs_string FROM source_crs), 'EPSG:4326')) AS longitude,
-            ST_Y(Transform(MakePoint(centroid_x, centroid_y), 4326, NULL,
-                (SELECT crs_string FROM source_crs), 'EPSG:4326')) AS latitude FROM 'divide-attributes';""",
-                conn,
-            )
-    except sqlite3.OperationalError:
-        with sqlite3.connect(hydrofabric) as conn:
-            conf_df = pandas.read_sql_query(
-                "SELECT* FROM 'divide-attributes';",
-                conn,
-            )
-        source_crs = get_table_crs_short(hydrofabric, "divides")
-        transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
-        lon, lat = transformer.transform(conf_df["centroid_x"].values, conf_df["centroid_y"].values)
-        conf_df["longitude"] = lon
-        conf_df["latitude"] = lat
-
-        conf_df.drop(columns=["centroid_x", "centroid_y"], axis=1, inplace=True)
+    lon, lat = transformer.transform(conf_df["centroid_x"].values, conf_df["centroid_y"].values)
+    conf_df["longitude"] = lon
+    conf_df["latitude"] = lat
     return conf_df
 
 
-def make_em_config(
+def make_lstm_config(
     hydrofabric: Path,
     output_dir: Path,
-    template_path: Path = file_paths.template_em_config,
+    template_path: Path = file_paths.template_lstm_config,
 ):
     # test if modspatialite is available
-    try:
-        divide_conf_df = get_model_attributes_modspatialite(hydrofabric)
-    except Exception as e:
-        logger.warning(f"mod_spatialite not available, using pyproj instead: {e}")
-        logger.warning("Install mod_spatialite for improved performance")
-        divide_conf_df = get_model_attributes_pyproj(hydrofabric)
-
-    cat_config_dir = output_dir / "cat_config" / "empirical_model"
+
+    divide_conf_df = get_model_attributes(hydrofabric)
+
+    cat_config_dir = output_dir / "cat_config" / "lstm"
     if cat_config_dir.exists():
         shutil.rmtree(cat_config_dir)
     cat_config_dir.mkdir(parents=True, exist_ok=True)
 
+    # convert the mean.slope from degrees 0-90 where 90 is flat and 0 is vertical to m/km
+    # flip 0 and 90 degree values
+    divide_conf_df["flipped_mean_slope"] = abs(divide_conf_df["mean.slope"] - 90)
+    # Convert degrees to meters per kmmeter
+    divide_conf_df["mean_slope_mpkm"] = (
+        np.tan(np.radians(divide_conf_df["flipped_mean_slope"])) * 1000
+    )
+
     with open(template_path, "r") as file:
         template = file.read()
 
-    for divide in divide_conf_df.index:
+    for _, row in divide_conf_df.iterrows():
+        divide = row["divide_id"]
         with open(cat_config_dir / f"{divide}.yml", "w") as file:
             file.write(
                 template.format(
-                    area_sqkm=divide_conf_df.loc[divide, "areasqkm"],
+                    area_sqkm=row["areasqkm"],
                     divide_id=divide,
-                    lat=divide_conf_df.loc[divide, "latitude"],
-                    lon=divide_conf_df.loc[divide, "longitude"],
-                    slope_mean=divide_conf_df.loc[divide, "mean.slope"],
-                    elevation_mean=divide_conf_df.loc[divide, "mean.slope"],
+                    lat=row["latitude"],
+                    lon=row["longitude"],
+                    slope_mean=row["mean_slope_mpkm"],
+                    elevation_mean=row["mean.elevation"] / 100,  # convert cm in hf to m
                 )
             )
 
@@ -259,14 +180,16 @@ def configure_troute(
     with open(file_paths.template_troute_config, "r") as file:
         troute_template = file.read()
     time_step_size = 300
-    gpkg_file_path=f"{config_dir}/{cat_id}_subset.gpkg"
+    gpkg_file_path = f"{config_dir}/{cat_id}_subset.gpkg"
     nts = (end_time - start_time).total_seconds() / time_step_size
     with sqlite3.connect(gpkg_file_path) as conn:
         ncats_df = pandas.read_sql_query("SELECT COUNT(id) FROM 'divides';", conn)
-        ncats = ncats_df['COUNT(id)'][0]
+        ncats = ncats_df["COUNT(id)"][0]
 
-    est_bytes_required = nts * ncats * 45 # extremely rough calculation based on about 3 tests :)
-    local_ram_available = 0.8 * psutil.virtual_memory().available # buffer to not accidentally explode machine
+    est_bytes_required = nts * ncats * 45  # extremely rough calculation based on about 3 tests :)
+    local_ram_available = (
+        0.8 * psutil.virtual_memory().available
+    )  # buffer to not accidentally explode machine
 
     if est_bytes_required > local_ram_available:
         max_loop_size = nts // (est_bytes_required // local_ram_available)
@@ -289,7 +212,7 @@ def configure_troute(
         start_datetime=start_time.strftime("%Y-%m-%d %H:%M:%S"),
         nts=nts,
         max_loop_size=max_loop_size,
-        binary_nexus_file_folder_comment=binary_nexus_file_folder_comment 
+        binary_nexus_file_folder_comment=binary_nexus_file_folder_comment,
     )
 
     with open(config_dir / "troute.yaml", "w") as file:
@@ -310,22 +233,14 @@ def make_ngen_realization_json(
         json.dump(realization, file, indent=4)
 
 
-def create_em_realization(cat_id: str, start_time: datetime, end_time: datetime):
+def create_lstm_realization(cat_id: str, start_time: datetime, end_time: datetime):
     paths = file_paths(cat_id)
-    template_path = file_paths.template_em_realization_config
-    em_config = file_paths.template_em_model_config
-    # move em_config to paths.config_dir
-    with open(em_config, "r") as f:
-        em_config = f.read()
-    with open(paths.config_dir / "em-config.yml", "w") as f:
-        f.write(em_config)
-
+    template_path = file_paths.template_lstm_realization_config
     configure_troute(cat_id, paths.config_dir, start_time, end_time)
     make_ngen_realization_json(paths.config_dir, template_path, start_time, end_time)
-    make_em_config(paths.geopackage_path, paths.config_dir)
+    make_lstm_config(paths.geopackage_path, paths.config_dir)
     # create some partitions for parallelization
     paths.setup_run_folders()
-    create_partitions(paths)
 
 
 def create_realization(
@@ -368,48 +283,3 @@ def create_realization(
 
     # create some partitions for parallelization
     paths.setup_run_folders()
-    create_partitions(paths)
-
-
-def create_partitions(paths: file_paths, num_partitions: Optional[int] = None) -> None:
-    if num_partitions is None:
-        num_partitions = multiprocessing.cpu_count()
-
-    cat_to_nex_pairs = get_cat_to_nex_flowpairs(hydrofabric=paths.geopackage_path)
-    # nexus = defaultdict(list)
-
-    # for cat, nex in cat_to_nex_pairs:
-    #     nexus[nex].append(cat)
-
-    num_partitions = min(num_partitions, len(cat_to_nex_pairs))
-    # partition_size = ceil(len(nexus) / num_partitions)
-    # num_nexus = len(nexus)
-    # nexus = list(nexus.items())
-    # partitions = []
-    # for i in range(0, num_nexus, partition_size):
-    #     part = {}
-    #     part["id"] = i // partition_size
-    #     part["cat-ids"] = []
-    #     part["nex-ids"] = []
-    #     part["remote-connections"] = []
-    #     for j in range(i, i + partition_size):
-    #         if j < num_nexus:
-    #             part["cat-ids"].extend(nexus[j][1])
-    #             part["nex-ids"].append(nexus[j][0])
-    #     partitions.append(part)
-
-    # with open(paths.subset_dir / f"partitions_{num_partitions}.json", "w") as f:
-    #     f.write(json.dumps({"partitions": partitions}, indent=4))
-
-    # write this to a metadata file to save on repeated file io to recalculate
-    with open(paths.metadata_dir / "num_partitions", "w") as f:
-        f.write(str(num_partitions))
-
-
-if __name__ == "__main__":
-    cat_id = "cat-1643991"
-    start_time = datetime(2010, 1, 1, 0, 0, 0)
-    end_time = datetime(2010, 1, 2, 0, 0, 0)
-    # output_interval = 3600
-    # nts = 2592
-    create_realization(cat_id, start_time, end_time)

data_processing/file_paths.py

@@ -1,6 +1,7 @@
+from datetime import datetime
 from pathlib import Path
 from typing import Optional
-from datetime import datetime
+
 
 class file_paths:
     """
@@ -27,11 +28,10 @@ class file_paths:
     dev_file = Path(__file__).parent.parent.parent / ".dev"
     template_troute_config = data_sources / "ngen-routing-template.yaml"
     template_cfe_nowpm_realization_config = data_sources / "cfe-nowpm-realization-template.json"
-    template_em_realization_config = data_sources / "em-realization-template.json"
+    template_lstm_realization_config = data_sources / "lstm-realization-template.json"
     template_noahowp_config = data_sources / "noah-owp-modular-init.namelist.input"
     template_cfe_config = data_sources / "cfe-template.ini"
-    template_em_config = data_sources / "em-catchment-template.yml"
-    template_em_model_config = data_sources / "em-config.yml"
+    template_lstm_config = data_sources / "lstm-catchment-template.yml"
 
     def __init__(self, folder_name: Optional[str] = None, output_dir: Optional[Path] = None):
         """

data_sources/lstm-catchment-template.yml

@@ -0,0 +1,17 @@
+time_step: "1 hour"
+area_sqkm: {area_sqkm} # areasqkm
+basin_id: {divide_id}
+basin_name: {divide_id}
+elev_mean: {elevation_mean} # mean.elevation
+initial_state: zero
+lat: {lat} # needs calulating
+lon: {lon} # needs calulating
+slope_mean: {slope_mean} # mean.slope
+train_cfg_file:
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_25yr_1210_112435_7/config.yml
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_25yr_1210_112435_8/config.yml
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_25yr_1210_112435_9/config.yml
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_25yr_seq999_seed101_0701_143442/config.yml
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_25yr_seq999_seed103_2701_171540/config.yml
+  - /ngen/ngen/extern/lstm/trained_neuralhydrology_models/nh_AORC_hourly_slope_elev_precip_temp_seq999_seed101_2801_191806/config.yml
+verbose: 0

data_sources/{em-realization-template.json → lstm-realization-template.json}

@@ -5,25 +5,22 @@
         "name": "bmi_multi",
         "params": {
           "name": "bmi_multi",
-          "model_type_name": "empirical_model",
+          "model_type_name": "lstm",
           "forcing_file": "",
           "init_config": "",
           "allow_exceed_end_time": true,
           "main_output_variable": "land_surface_water__runoff_depth",
-          "modules": [            
+          "modules": [
             {
               "name": "bmi_python",
               "params": {
                 "name": "bmi_python",
                 "python_type": "lstm.bmi_lstm.bmi_LSTM",
-                "model_type_name": "bmi_empirical_model",
-                "init_config": "./config/cat_config/empirical_model/{{id}}.yml",
+                "model_type_name": "bmi_lstm",
+                "init_config": "./config/cat_config/lstm/{{id}}.yml",
                 "allow_exceed_end_time": true,
                 "main_output_variable": "land_surface_water__runoff_depth",
-                "uses_forcing_file": false,
-                "variables_names_map": {
-                  "atmosphere_water__liquid_equivalent_precipitation_rate": "APCP_surface"
-                }
+                "uses_forcing_file": false
               }
             }
           ]

ngiab_data_cli/__main__.py

@@ -8,9 +8,10 @@ with rich.status.Status("loading") as status:
     import logging
     import subprocess
     import time
+    from multiprocessing import cpu_count
 
     import geopandas as gpd
-    from data_processing.create_realization import create_em_realization, create_realization
+    from data_processing.create_realization import create_lstm_realization, create_realization
     from data_processing.dask_utils import shutdown_cluster
     from data_processing.dataset_utils import save_and_clip_dataset
     from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
@@ -22,6 +23,7 @@ with rich.status.Status("loading") as status:
     from data_sources.source_validation import validate_hydrofabric, validate_output_dir
     from ngiab_data_cli.arguments import parse_arguments
     from ngiab_data_cli.custom_logging import set_logging_to_critical_only, setup_logging
+    
 
 
 def validate_input(args: argparse.Namespace) -> Tuple[str, str]:
@@ -184,8 +186,8 @@ def main() -> None:
             gage_id = None
             if args.gage:
                 gage_id = args.input_feature
-            if args.empirical_model:
-                create_em_realization(
+            if args.lstm:
+                create_lstm_realization(
                     output_folder, start_time=args.start_date, end_time=args.end_date
                 )
             else:
@@ -200,17 +202,13 @@ def main() -> None:
 
         if args.run:
             logging.info("Running Next Gen using NGIAB...")
-            # open the partitions.json file and get the number of partitions
-            with open(paths.metadata_dir / "num_partitions", "r") as f:
-                num_partitions = int(f.read())
-
+            
             try:
                 subprocess.run("docker pull awiciroh/ciroh-ngen-image:latest", shell=True)
             except:
                 logging.error("Docker is not running, please start Docker and try again.")
             try:
-                # command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
-                command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {num_partitions} local'
+                command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {cpu_count()} local'
                 subprocess.run(command, shell=True)
                 logging.info("Next Gen run complete.")
             except:

ngiab_data_cli/arguments.py

@@ -106,10 +106,9 @@ def parse_arguments() -> argparse.Namespace:
         help="enable debug logging",
     )
     parser.add_argument(
-        "--empirical_model",
-        "--em",
+        "--lstm",
         action="store_true",
-        help="enable empirical model realization and forcings",
+        help="enable LSTM model realization and forcings",
     )
     parser.add_argument(
         "--nwm_gw",

{ngiab_data_preprocess-4.4.0.dist-info → ngiab_data_preprocess-4.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ngiab_data_preprocess
-Version: 4.4.0
+Version: 4.5.1
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-4.4.0.dist-info → ngiab_data_preprocess-4.5.1.dist-info}/RECORD

@@ -1,8 +1,8 @@
-data_processing/create_realization.py,sha256=mdse8W2DgPg5Lj2_ErUsLJh-touTmShKwQrrOWO0jlY,15958
+data_processing/create_realization.py,sha256=deRP3p4zfV9PF_vUikYKRiKA994bgj2GtPGCjQDSsTE,10795
 data_processing/dask_utils.py,sha256=A2IP94WAz8W9nek3etXKEKTOxGPf0NWSFLh8cZ5S-xU,2454
 data_processing/dataset_utils.py,sha256=AJOxE2nRfZnWYon_qqGcfkpRZuRW8Yy8YI86SxVDU3M,11168
 data_processing/datasets.py,sha256=_EJ1uZSWTU1HWpvF7TQSikneJqWZFikTrdo9usCV8A0,4665
-data_processing/file_paths.py,sha256=l2iCUFt_pk-jjzl7OS7npROAnQxwqFfZ7b2wRjViqiU,4720
+data_processing/file_paths.py,sha256=MFUShBB1g9IGi9MaJwrl6fKIcsrhbmcYEdTHtmnphZo,4667
 data_processing/forcings.py,sha256=k-JhBncTnXcdjSieam1Q2cDx5Xt9hH5Aywv0gDY4O2U,19010
 data_processing/gpkg_utils.py,sha256=tSSIMlHeqqgxTJQyF3X9tPmunQTJYx0xrCNHqUBQxkg,20590
 data_processing/graph_utils.py,sha256=qvHw6JlzQxLi--eMsGgC_rUBP4nDatl6X9mSa03Xxyo,8306
@@ -10,10 +10,9 @@ data_processing/s3fs_utils.py,sha256=ki1EmA0ezV0r26re6dRWIGzL5FudGdwF9Qw1eVLR0Bc
 data_processing/subset.py,sha256=XoojOgWCwxOi5Q4KXHXARNQeoZlobJp-mqhIIvTRtTw,3793
 data_sources/cfe-nowpm-realization-template.json,sha256=8an6q1drWD8wU1ocvdPab-GvZDvlQ-0di_-NommH3QI,3528
 data_sources/cfe-template.ini,sha256=6e5-usqjWtm3MWVvtm8CTeZTJJMxO1ZswkOXq0L9mnc,2033
-data_sources/em-catchment-template.yml,sha256=M08ixazEUHYI2PNavtI0xPZeSzcQ9bg2g0XzNT-8_u4,292
-data_sources/em-config.yml,sha256=y0J8kEA70rxLWXJjz-CQ7sawcVyhQcayofeLlq4Svbo,1330
-data_sources/em-realization-template.json,sha256=DJvB7N8lCeS2vLFenmbTzysBDR-xPaJ09XA8heu1ijY,1466
 data_sources/forcing_template.nc,sha256=uRuVAqX3ngdlougZINavtwl_wC2VLD8fHqG7_CLim1s,85284
+data_sources/lstm-catchment-template.yml,sha256=LtknqvxbWrtLLZIXxFgTfbQmM4x8XnHBDFvRIh2EIFI,965
+data_sources/lstm-realization-template.json,sha256=ndz3h5NGhtUSnsZwscgNuXYBG9mlAuz7Lxx7iCw22UY,1270
 data_sources/ngen-routing-template.yaml,sha256=wM5v6jj0kwcJBVatLFuy2big6g8nlSXxzc8a23nwI5s,4655
 data_sources/noah-owp-modular-init.namelist.input,sha256=Vb7mp40hFpJogruOrXrDHwVW1bKi9h1ciDNyDvTzn20,3045
 data_sources/source_validation.py,sha256=RmvyPLjuDetpuNOUqCclgDfe8zd_Ojr7pfbUoUya2pQ,9498
@@ -31,13 +30,13 @@ map_app/static/js/main.js,sha256=_Yq1tuzyREqWU24rFQJSh5zIaXtAXEGlfZPo36QLHvI,969
 map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
 map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
 map_app/templates/index.html,sha256=Jy2k1Ob2_et--BPpfmTYO22Yin3vrG6IOeNlwzUoEqY,7878
-ngiab_data_cli/__main__.py,sha256=13W3RnD73weQNYZdq6munx_0oMBgzc-yzluKEm5nSxg,10570
-ngiab_data_cli/arguments.py,sha256=yBULJnFgUvgP4YZmZ5HhR7g0EfdMtBCdQuDkDuYSXCQ,4322
+ngiab_data_cli/__main__.py,sha256=io9YbZY65tQC66gpcP02ECRnGpM-fnjLxQHa1EKDKzc,10269
+ngiab_data_cli/arguments.py,sha256=qS8RupcT3Ax7ZRT0uKKzFdUvkDdVugBlYyuzljY_bxo,4290
 ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
 ngiab_data_cli/forcing_cli.py,sha256=eIWRxRWUwPqR16fihFDEIV4VzGlNuvcD6lJW5VYjkPU,3635
-ngiab_data_preprocess-4.4.0.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
-ngiab_data_preprocess-4.4.0.dist-info/METADATA,sha256=8PlfoGwOJIpuKhFwtfWmfxdMaDeXBfFRz9CAeZ3sZKk,13344
-ngiab_data_preprocess-4.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ngiab_data_preprocess-4.4.0.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
-ngiab_data_preprocess-4.4.0.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
-ngiab_data_preprocess-4.4.0.dist-info/RECORD,,
+ngiab_data_preprocess-4.5.1.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
+ngiab_data_preprocess-4.5.1.dist-info/METADATA,sha256=Sa-C8mvOi-1Qn5u27X2Dz5wYeMPLnjle-nuC_VJ2kZo,13344
+ngiab_data_preprocess-4.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ngiab_data_preprocess-4.5.1.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
+ngiab_data_preprocess-4.5.1.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
+ngiab_data_preprocess-4.5.1.dist-info/RECORD,,

data_sources/em-catchment-template.yml

@@ -1,10 +0,0 @@
-area_sqkm: {area_sqkm} # areasqkm
-basin_id: {divide_id}
-basin_name: {divide_id}
-elev_mean: {elevation_mean} # mean.elevation
-initial_state: zero
-lat: {lat} # needs calulating
-lon: {lon} # needs calulating
-slope_mean: {slope_mean} # mean.slope
-train_cfg_file: ./config/em-config.yml
-verbose: 0

data_sources/em-config.yml

@@ -1,60 +0,0 @@
-batch_size: 256
-clip_gradient_norm: 1
-clip_targets_to_zero:
-- QObs(mm/d)
-data_dir: ./data/
-dataset: hourly_camels_us
-device: cpu
-dynamic_inputs:
-- total_precipitation
-- temperature
-epochs: 9
-experiment_name: hourly_slope_mean_precip_temp
-forcings: nldas_hourly
-head: regression
-hidden_size: 64
-img_log_dir: /ngen/ngen/extern/lstm/trained_neuralhydrology_models/hourly_slope_mean_precip_temp/img_log
-initial_forget_bias: 3
-learning_rate:
-  0: 0.0005
-  10: 0.0001
-  25: 5e-05
-log_interval: 5
-log_n_figures: 0
-log_tensorboard: false
-loss: NSE
-mass_inputs:
-metrics:
-- NSE
-- KGE
-- Alpha-NSE
-- Beta-NSE
-model: cudalstm
-num_workers: 8
-number_of_basins: 516
-optimizer: Adam
-output_activation: linear
-output_dropout: 0.4
-package_version: 1.0.0-beta1
-predict_last_n: 24
-run_dir: /ngen/ngen/extern/lstm/trained_neuralhydrology_models/hourly_slope_mean_precip_temp
-save_weights_every: 1
-seed: 102
-seq_length: 336
-static_attributes:
-- elev_mean
-- slope_mean
-target_variables:
-- QObs(mm/d)
-test_basin_file: 516_basins.txt
-test_end_date: 31/12/2002
-test_start_date: 01/01/2000
-train_basin_file: 516_basins.txt
-train_dir: trained_elsewhere
-train_end_date: 30/09/2018
-train_start_date: 01/10/1980
-validate_every: 1
-validate_n_random_basins: 15
-validation_basin_file: 516_basins.txt
-validation_end_date: 30/09/2018
-validation_start_date: 01/10/1980