fimserve 0.1.6__py3-none-any.whl

fimserve/FIMsubset/__init__.py

@@ -0,0 +1,7 @@
+import warnings
+
+warnings.simplefilter("ignore")
+
+from .xycoord import subsetFIM
+
+__all__ = ["subsetFIM"]

fimserve/FIMsubset/shpsubset.py

@@ -0,0 +1,61 @@
+import os
+import glob
+import rasterio
+from rasterio.mask import mask
+from pathlib import Path
+import geopandas as gpd
+
+from ..datadownload import setup_directories
+
+
+def checkSHP(input_file):
+    if isinstance(input_file, Path):
+        input_file = str(input_file)
+
+    if input_file.endswith(".shp"):
+        gdf = gpd.read_file(input_file)
+    elif input_file.endswith(".gpkg"):
+        gdf = gpd.read_file(input_file)
+    elif input_file.endswith(".geojson"):
+        gdf = gpd.read_file(input_file)
+    elif input_file.endswith(".kml"):
+        gdf = gpd.read_file(input_file)
+    else:
+        raise ValueError(
+            "Unsupported file format. Please provide a shapefile GeoPackage, GeoJSON, or KML."
+        )
+    return gdf
+
+
+def clipFIMforboundary(inundation_raster, shapefile_geom, output_directory, huc):
+    with rasterio.open(inundation_raster) as src:
+        geometries = shapefile_geom.geometry.tolist()
+        if src.crs != shapefile_geom.crs:
+            print("CRS mismatch. Reprojecting geometry to match raster.")
+            geometries = [geom.to_crs(src.crs) for geom in geometries]
+
+        out_image, out_transform = mask(src, geometries, crop=True)
+        out_meta = src.meta.copy()
+        out_meta.update(
+            {
+                "driver": "GTiff",
+                "count": 1,
+                "crs": src.crs,
+                "transform": out_transform,
+                "width": out_image.shape[2],
+                "height": out_image.shape[1],
+            }
+        )
+
+        output_directory = os.path.join(output_directory, "subsetFIM")
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+
+        file_basename = os.path.basename(inundation_raster).split("_")[0]
+        output_file = os.path.join(output_directory, f"{file_basename}_subsetFIM.tif")
+
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+        with rasterio.open(output_file, "w", **out_meta) as dest:
+            dest.write(out_image)
+        print(f"Clipped raster saved to {output_file}")

fimserve/FIMsubset/xycoord.py

@@ -0,0 +1,102 @@
+import os
+import glob
+import geopandas as gpd
+from shapely.geometry import Point
+from pyproj import CRS
+import rasterio
+from rasterio.mask import mask
+
+from ..datadownload import setup_directories
+from .shpsubset import checkSHP, clipFIMforboundary
+
+
+def checkifWGS(x, y):
+    """Check if coordinates are in WGS84 (EPSG:4326) range."""
+    if -180 <= x <= 180 and -90 <= y <= 90:
+        return True
+    else:
+        return False
+
+
+def reproject_coordinates(x, y, target_crs="EPSG:5070"):
+    """Reproject coordinates to target CRS."""
+    if checkifWGS(x, y):
+        from pyproj import CRS, Transformer
+
+        crs_wgs84 = CRS("EPSG:4326")
+        crs_target = CRS(target_crs)
+        transformer = Transformer.from_crs(crs_wgs84, crs_target, always_xy=True)
+        x_new, y_new = transformer.transform(x, y)
+        return x_new, y_new
+    else:
+        return x, y
+
+
+def clipFIM(inundation_raster, shapefile_geom, output_directory, huc):
+    with rasterio.open(inundation_raster) as src:
+        out_image, out_transform = mask(src, [shapefile_geom], crop=True)
+        out_meta = src.meta.copy()
+        out_meta.update(
+            {
+                "driver": "GTiff",
+                "count": 1,
+                "crs": src.crs,
+                "transform": out_transform,
+                "width": out_image.shape[2],
+                "height": out_image.shape[1],
+            }
+        )
+        file_basename = os.path.basename(inundation_raster).split("_")[0]
+        output_file = os.path.join(output_directory, f"{file_basename}_subsetFIM.tif")
+
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+
+        with rasterio.open(output_file, "w", **out_meta) as dest:
+            dest.write(out_image)
+
+        print(f"Clipped raster saved to {output_file}")
+        return out_image, out_transform
+
+
+def withininWatershed(x, y, geopackage_path, inundation_raster, huc, output_directory):
+    point = Point(x, y)
+    watersheds = gpd.read_file(geopackage_path)
+
+    # Watershed containing the point
+    watershed_containing_point = None
+    for _, watershed in watersheds.iterrows():
+        if watershed["geometry"].contains(point):
+            watershed_containing_point = watershed
+            break
+    output_directory = os.path.join(output_directory, "subsetFIM")
+    if not os.path.exists(output_directory):
+        os.makedirs(output_directory)
+
+    if watershed_containing_point is None:
+        raise ValueError(f"Point is not within any watershed boundary for {huc}")
+    return clipFIM(
+        inundation_raster, watershed_containing_point["geometry"], output_directory, huc
+    )
+
+
+def subsetFIM(location, huc, method):
+    code_dir, data_dir, output_dir = setup_directories()
+    gpkg_path = os.path.join(
+        output_dir, f"flood_{huc}", huc, "nwm_catchments_proj_subset.gpkg"
+    )
+    out_dir = os.path.join(output_dir, f"flood_{huc}", f"{huc}_inundation")
+    inundation_rasters = os.path.join(out_dir, "*_inundation.tif")
+    print(inundation_rasters)
+    files = glob.glob(inundation_rasters)
+    if method == "xy":
+        x, y = location
+        print("point", x, y)
+        x, y = reproject_coordinates(x, y)
+        for file in files:
+            print(f"Clipping {file} to watershed containing point {x}, {y}")
+            withininWatershed(x, y, gpkg_path, file, huc, out_dir)
+    elif method == "boundary":
+        shapefile = checkSHP(location)
+        for file in files:
+            clipFIMforboundary(file, shapefile, out_dir, huc)

fimserve/__init__.py

@@ -0,0 +1,29 @@
+import warnings
+
+warnings.simplefilter("ignore")
+
+from .datadownload import DownloadHUC8
+from .streamflowdata.nwmretrospective import getNWMretrospectivedata
+from .runFIM import runOWPHANDFIM
+
+from .streamflowdata.forecasteddata import getNWMForecasteddata
+
+# plots
+from .plot.nwmfid import plotNWMStreamflow
+from .plot.usgs import getUSGSsitedata
+from .plot.comparestreamflow import CompareNWMnUSGSStreamflow
+from .plot.usgs import plotUSGSStreamflow
+from .plot.src import plotSRC
+
+# For table
+from .plot.usgsandfid import GetUSGSIDandCorrFID
+
+# subsetting
+from .FIMsubset.xycoord import subsetFIM
+
+# Fim visualization
+from .vizualizationFIM import vizualizeFIM
+
+
+# Statistics
+from .statistics.calculatestatistics import CalculateStatistics

fimserve/datadownload.py

@@ -0,0 +1,112 @@
+import os
+import csv
+import pandas as pd
+import subprocess
+
+
+def setup_directories():
+    parent_dir = os.getcwd()
+    code_dir = os.path.join(parent_dir, "code", "inunation-mapping")
+    data_dir = os.path.join(parent_dir, "data", "inputs")
+    output_dir = os.path.join(parent_dir, "output")
+
+    # Create the directories if not exist
+    os.makedirs(code_dir, exist_ok=True)
+    os.makedirs(data_dir, exist_ok=True)
+    os.makedirs(output_dir, exist_ok=True)
+
+    return code_dir, data_dir, output_dir
+
+
+def clone_repository(code_dir):
+    repo_path = os.path.join(code_dir)
+
+    # Check if repository folder exists and has files in it
+    if os.path.exists(repo_path) and os.listdir(repo_path):
+        print(
+            f"Repository already exists at {repo_path} and contains files. Skipping clone."
+        )
+    else:
+        # Clone the repository if it doesn't exist or is empty
+        repo_url = "https://github.com/NOAA-OWP/inundation-mapping.git"
+        subprocess.run(["git", "clone", repo_url, repo_path], check=True)
+        print(f"Repository cloned into: {repo_path}")
+
+
+def download_data(huc_number, base_dir):
+    output_dir = os.path.join(base_dir, f"flood_{huc_number}", str(huc_number))
+
+    # Create the directory structure if it doesn't exist
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # For the CIROH
+    cmd = f"aws s3 sync s3://ciroh-owp-hand-fim/hand_fim_4_5_2_11/{huc_number}/ {output_dir} --no-sign-request "
+
+    # Run the AWS CLI command
+    os.system(cmd)
+    print(f"Data for HUC {huc_number}")
+
+    # Now copying the hydrotable.csv to the outside of directory as fim_inputs.csv
+    hydrotable_path = os.path.join(output_dir, "branch_ids.csv")
+    fim_inputs_path = os.path.join(base_dir, f"flood_{huc_number}", "fim_inputs.csv")
+
+    # Read the first row from branch_ids.csv
+    with open(hydrotable_path, "r") as infile:
+        reader = csv.reader(infile)
+        first_row = next(reader)  # Get the first row
+
+    # Write the first row to fim_inputs.csv
+    with open(fim_inputs_path, "w", newline="") as outfile:
+        writer = csv.writer(outfile)
+        writer.writerow(first_row)
+
+    print(
+        f"Copied the first row of {hydrotable_path} to {fim_inputs_path} as fim_inputs.csv."
+    )
+
+
+def uniqueFID(hydrotable, fid_dir, stream_order=None):
+    hydrotable_df = pd.read_csv(hydrotable)
+    if stream_order:
+        stream_order = [str(order) for order in stream_order]
+        hydrotable_df["order_"] = hydrotable_df["order_"].astype(str)
+        hydrotable_df_filtered = hydrotable_df[
+            hydrotable_df["order_"].isin(stream_order)
+        ]
+        unique_FIDs = hydrotable_df_filtered["feature_id"].drop_duplicates()
+    else:
+        unique_FIDs = hydrotable_df["feature_id"].drop_duplicates()
+    unique_FIDs_df = pd.DataFrame(unique_FIDs, columns=["feature_id"])
+    unique_FIDs_df.to_csv(fid_dir, index=False)
+    print(f"Unique feature IDs saved to {fid_dir}.")
+
+
+# IN the replacement of Docker, the environement variables are set in the .env file
+def EnvFile(code_dir):
+    env_dir = os.path.join(code_dir, ".env")
+    env_content = """
+    inputsDir=inputs
+    outputsDir=output
+    """
+    os.makedirs(os.path.dirname(env_dir), exist_ok=True)
+
+    with open(env_dir, "w") as f:
+        f.write(env_content)
+
+
+def DownloadHUC8(huc, stream_order=None):
+    code_dir, data_dir, output_dir = setup_directories()
+    clone_repository(code_dir)
+
+    # if huc is not str:
+    huc = str(huc)
+    download_data(huc, output_dir)
+    EnvFile(code_dir)
+    HUC_dir = os.path.join(output_dir, f"flood_{huc}")
+    hydrotable_dir = os.path.join(HUC_dir, str(huc), "hydrotable.csv")
+    featureID_dir = os.path.join(HUC_dir, f"feature_IDs.csv")
+    if stream_order is None:
+        uniqueFID(hydrotable_dir, featureID_dir)
+    else:
+        uniqueFID(hydrotable_dir, featureID_dir, stream_order)

fimserve/plot/__init__.py

@@ -0,0 +1,19 @@
+import warnings
+
+warnings.simplefilter("ignore")
+
+from .nwmfid import plotNWMStreamflow
+from .usgs import getUSGSsitedata
+from .usgs import plotUSGSStreamflow
+from .comparestreamflow import CompareNWMnUSGSStreamflow
+from .src import plotSRC
+from .usgsandfid import GetUSGSIDandCorrFID
+
+__all__ = [
+    "plotNWMStreamflow",
+    "getUSGSsitedata",
+    "plotUSGSStreamflow",
+    "CompareNWMnUSGSStreamflow",
+    "plotSRC",
+    "GetUSGSIDandCorrFID",
+]

fimserve/plot/comparestreamflow.py

@@ -0,0 +1,69 @@
+import os
+import matplotlib.pyplot as plt
+
+from .nwmfid import getFIDdata
+from .usgs import getUSGSdata
+from ..datadownload import setup_directories
+
+
+def plotcomparision(
+    data_dir_nwm, data_dir_usgs, feature_id, usgs_site, output_dir, start_date, end_date
+):
+    nwm_data = getFIDdata(data_dir_nwm, feature_id, start_date, end_date)
+    usgs_data = getUSGSdata(data_dir_usgs, usgs_site, start_date, end_date)
+
+    plt.figure(figsize=(10, 5))
+    # Plot NWM data with solid line
+    plt.plot(
+        nwm_data["Date"],
+        nwm_data["Discharge"],
+        label=f"NWM Streamflow of feature ID {feature_id}",
+        linestyle="solid",
+        color="#167693",
+        linewidth=2,
+    )
+
+    # Plot USGS data with dashed line
+    plt.plot(
+        usgs_data["Date"],
+        usgs_data["Discharge"],
+        label=f"USGS Streamflow of gauged site {usgs_site}",
+        linestyle="dashed",
+        color="#BF4037",
+        linewidth=2,
+    )
+
+    plt.xlabel("Date (Hourly)", fontsize=14)
+    plt.ylabel("Discharge (m³/s)", fontsize=14)
+    plt.title(f"Discharge comparison between USGS and NWM streamflow", fontsize=16)
+    plt.legend()
+    plt.xticks(rotation=45, fontsize=12)
+    plt.yticks(fontsize=12)
+    plt.tight_layout()
+    # Save dir
+    plt_dir = os.path.join(output_dir, "Plots")
+    os.makedirs(plt_dir, exist_ok=True)
+    plot_dir = os.path.join(plt_dir, f"NWMvsUSGS_{usgs_site}.png")
+    plt.grid(True, which="both", linestyle="-", linewidth=0.3)
+    plt.savefig(plot_dir, dpi=500, bbox_inches="tight")
+    plt.show()
+
+
+def CompareNWMnUSGSStreamflow(huc, feature_id, usgs_site, start_date, end_date):
+    code_dir, data_dir, output_dir = setup_directories()
+    discharge_dir_nwm = os.path.join(
+        output_dir, f"flood_{huc}", "discharge", "nwm30_retrospective"
+    )
+    discharge_dir_usgs = os.path.join(
+        output_dir, f"flood_{huc}", "discharge", "usgs_streamflow"
+    )
+    HUC_dir = os.path.join(output_dir, f"flood_{huc}")
+    plotcomparision(
+        discharge_dir_nwm,
+        discharge_dir_usgs,
+        feature_id,
+        usgs_site,
+        HUC_dir,
+        start_date,
+        end_date,
+    )

fimserve/plot/nwmfid.py

@@ -0,0 +1,123 @@
+import pandas as pd
+import os
+import matplotlib.pyplot as plt
+
+from ..datadownload import setup_directories
+
+
+def getFIDdata(data_dir, feature_id, start_date, end_date):
+    location_id = f"nwm30-{feature_id}"
+
+    if isinstance(start_date, str):
+        start_date = pd.to_datetime(start_date)
+    if isinstance(end_date, str):
+        end_date = pd.to_datetime(end_date)
+
+    start_dateSTR = start_date.strftime("%Y%m%d")
+    end_dateSTR = end_date.strftime("%Y%m%d")
+    target_file = f"{start_dateSTR}_{end_dateSTR}.parquet"
+
+    target_dir = os.path.join(data_dir, target_file)
+
+    if not os.path.exists(target_dir):
+        raise FileNotFoundError(
+            f"No NWM data found for the date range: {target_file} in {data_dir}. "
+            "Please check the date range that you downloaded for that HUC."
+        )
+
+    df = pd.read_parquet(target_dir)
+    matched_rows = df[df["location_id"] == location_id]
+
+    # Prepare the filtered data with desired columns
+    filtered_data = matched_rows[["value_time", "value"]].copy()
+    filtered_data.rename(
+        columns={"value_time": "Date", "value": "Discharge"}, inplace=True
+    )
+
+    return filtered_data
+
+
+# For the default feature_id
+def getFeatureWithMaxDischarge(data_dir, start_date, end_date):
+    start_dateSTR = pd.to_datetime(start_date).strftime("%Y%m%d")
+    end_dateSTR = pd.to_datetime(end_date).strftime("%Y%m%d")
+    target_file = f"{start_dateSTR}_{end_dateSTR}.parquet"
+
+    target_dir = os.path.join(data_dir, target_file)
+
+    if not os.path.exists(target_dir):
+        raise FileNotFoundError(
+            f"No NWM data found for the date range: {target_file} in {data_dir}. "
+            "Please check the date range that you downloaded for that HUC."
+        )
+
+    df = pd.read_parquet(target_dir)
+    max_discharge_row = df.loc[df["value"].idxmax()]
+    max_feature_id = max_discharge_row["location_id"].split("-")[1]
+    return max_feature_id
+
+
+def plotNWMStreamflowData(dischargedata, feature_ids, output_dir, start_date, end_date):
+    plt.figure(figsize=(10, 5))
+    missing_ids = []
+    plotted_ids = []
+
+    for feature_id in feature_ids:
+        try:
+            data = getFIDdata(dischargedata, feature_id, start_date, end_date)
+            if data.empty:
+                raise ValueError(f"No data for feature ID: {feature_id}")
+
+            # Plot data only if it exists
+            plt.plot(
+                data["Date"],
+                data["Discharge"],
+                label=f"NWM streamflow for feature ID: {feature_id}",
+                linewidth=2,
+            )
+            plotted_ids.append(feature_id)
+        except (FileNotFoundError, ValueError):
+            missing_ids.append(feature_id)
+
+    plt.xlabel("Date (Hourly)", fontsize=14)
+    plt.ylabel("Discharge (m³/s)", fontsize=14)
+    plt.title("NWM hourly streamflow", fontsize=16)
+    plt.xticks(rotation=45, fontsize=12)
+    plt.yticks(fontsize=12)
+    plt.grid(True, which="both", linestyle="-", linewidth=0.3)
+    plt.tight_layout()
+
+    if plotted_ids:
+        plt.legend()
+        # Save the plot
+        plt_dir = os.path.join(output_dir, "Plots")
+        os.makedirs(plt_dir, exist_ok=True)
+        plot_dir = os.path.join(plt_dir, "NWMStreamflow.png")
+        plt.savefig(plot_dir, dpi=500, bbox_inches="tight")
+        plt.show()
+    else:
+        print(
+            "\033[1m****No valid data found for any provided feature IDs. Plot not generated.****\033[0m"
+        )
+
+    if missing_ids:
+        print(
+            f"\033[1m****Data not found for the following NWM feature IDs: {', '.join(map(str, missing_ids))}****\033[0m"
+        )
+
+
+# Main function to drive the process
+def plotNWMStreamflow(huc, start_date, end_date, feature_ids=None):
+    code_dir, data_dir, output_dir = setup_directories()
+    huc_dir = os.path.join(output_dir, f"flood_{huc}")
+    discharge_dir = os.path.join(
+        output_dir, f"flood_{huc}", "discharge", "nwm30_retrospective"
+    )
+    if feature_ids is None or not feature_ids:
+        max_feature_id = getFeatureWithMaxDischarge(discharge_dir, start_date, end_date)
+        print(
+            f"*****No feature_id provided. Using the feature with max discharge: {max_feature_id}******"
+        )
+        feature_ids = [max_feature_id]
+
+    plotNWMStreamflowData(discharge_dir, feature_ids, huc_dir, start_date, end_date)

fimserve/plot/src.py

@@ -0,0 +1,93 @@
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from ..datadownload import setup_directories
+
+
+def filterhydroID(file_path, hydro_ids, branch_ids):
+    df = pd.read_csv(file_path)
+
+    df["HydroID"] = df["HydroID"].astype(str)
+    df["branch_id"] = df["branch_id"].astype(str)
+
+    filtered_dfs = []
+    for hydro_id, branch_id in zip(hydro_ids, branch_ids):
+        filtered_df = df[(df["HydroID"] == hydro_id) & (df["branch_id"] == branch_id)]
+        result_df = filtered_df[["stage", "default_discharge_cms"]]
+        filtered_dfs.append(result_df)
+    return filtered_dfs
+
+
+def plotsrc(file, hydro_ids, branch_ids, output_dir, discharge_value=None):
+    data_list = filterhydroID(file, hydro_ids, branch_ids)
+    plt.figure(figsize=(5, 4))
+    cmap = plt.get_cmap("tab10" if len(data_list) <= 10 else "hsv")
+    colors = [cmap(i / len(data_list)) for i in range(len(data_list))]
+
+    # Fill colors for the markers
+    marker_fill_colors = ["pink", "lightblue", "lightgreen", "lightcoral", "yellow"]
+
+    for i, data in enumerate(data_list):
+        plt.plot(
+            data["default_discharge_cms"],
+            data["stage"],
+            label=f"Synthetic rating curve for HydroID: {hydro_ids[i]}",
+            linestyle="-",
+            linewidth=2.5,
+            color=colors[i],
+        )
+
+        if discharge_value:
+            stage_at_discharge = data["stage"].iloc[
+                (data["default_discharge_cms"] - discharge_value).abs().argmin()
+            ]
+
+            plt.plot(
+                [discharge_value, discharge_value],
+                [0, stage_at_discharge],
+                linestyle="--",
+                color=colors[i],
+                linewidth=2,
+            )
+            plt.plot(
+                [0, discharge_value],
+                [stage_at_discharge, stage_at_discharge],
+                linestyle="--",
+                color=colors[i],
+                linewidth=2,
+            )
+            # Marker at the intersection
+            plt.scatter(
+                discharge_value,
+                stage_at_discharge,
+                color=marker_fill_colors[i % len(marker_fill_colors)],
+                edgecolor="black",
+                s=200,
+                marker="*",
+                zorder=5,
+                label=f"For discharge {discharge_value} cms, stage: {stage_at_discharge:.2f} m",
+            )
+
+    plt.xlabel("Discharge (m³/s)", fontsize=14)
+    plt.ylabel("Stage (in m)", fontsize=14)
+    plt.title("Synthetic Rating Curves", fontsize=16)
+    plt.legend()
+    plt.xticks(fontsize=12)
+    plt.yticks(fontsize=12)
+    plt.tight_layout()
+    plt.grid(True, which="both", linestyle="-", linewidth=0.3)
+
+    # Save dir
+    plt_dir = os.path.join(output_dir, "Plots")
+    os.makedirs(plt_dir, exist_ok=True)
+    plot_dir = os.path.join(plt_dir, f"SRC_{hydro_ids[0]}.png")
+    plt.savefig(plot_dir, dpi=500, bbox_inches="tight")
+    plt.show()
+
+
+def plotSRC(huc, hydro_ids, branch_ids, discharge_value=None):
+    code_dir, data_dir, output_dir = setup_directories()
+    HUC_dir = os.path.join(output_dir, f"flood_{huc}")
+    hydrotable_dir = os.path.join(output_dir, f"flood_{huc}", huc, "hydrotable.csv")
+    plotsrc(hydrotable_dir, hydro_ids, branch_ids, HUC_dir, discharge_value)