fimeval 0.1.55__py3-none-any.whl → 0.1.57__py3-none-any.whl

fimeval/BenchFIMQuery/utilis.py

@@ -0,0 +1,269 @@
+"""
+This utility function contains how to retrieve all the necessary metadata of benchmark FIM
+from the s3 bucket during benchmark FIM querying.
+
+Authors: Supath Dhital, sdhital@crimson.ua.edu
+Updated date: 25 Nov, 2025
+"""
+
+from __future__ import annotations
+import os, re, json, datetime as dt
+from typing import List, Dict, Any, Optional
+
+import urllib.parse
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
+
+# constants
+BUCKET = "sdmlab"
+CATALOG_KEY = (
+    "FIM_Database/FIM_Viz/catalog_core.json"  # Path of the json file in the s3 bucket
+)
+
+# s3 client
+_S3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+
+# helpers for direct S3 file links
+def s3_http_url(bucket: str, key: str) -> str:
+    """Build a public-style S3 HTTPS URL."""
+    return f"https://{bucket}.s3.amazonaws.com/{urllib.parse.quote(key, safe='/')}"
+
+
+# utils
+_YMD_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+_YMD_COMPACT_RE = re.compile(r"^\d{8}$")
+_YMDH_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}$")
+_YMDHMS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2})?$")
+
+
+def _normalize_user_dt(s: str) -> str:
+    s = s.strip()
+    s = s.replace("/", "-")
+    s = re.sub(r"\s+", " ", s)
+    return s
+
+
+def _to_date(s: str) -> dt.date:
+    s = _normalize_user_dt(s)
+    if _YMD_COMPACT_RE.match(s):
+        return dt.datetime.strptime(s, "%Y%m%d").date()
+    if _YMD_RE.match(s):
+        return dt.date.fromisoformat(s)
+    try:
+        return dt.datetime.fromisoformat(s).date()
+    except Exception:
+        m = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2})$", s)
+        if m:
+            return dt.datetime.fromisoformat(f"{m.group(1)} {m.group(2)}:00:00").date()
+        raise ValueError(f"Bad date format: {s}")
+
+
+def _to_hour_or_none(s: str) -> Optional[int]:
+    s = _normalize_user_dt(s)
+    if _YMD_RE.match(s) or _YMD_COMPACT_RE.match(s):
+        return None
+    m = re.match(r"^\d{4}-\d{2}-\d{2}[ T](\d{2})$", s)
+    if m:
+        return int(m.group(1))
+    try:
+        dt_obj = dt.datetime.fromisoformat(s)
+        return dt_obj.hour
+    except Exception:
+        m2 = re.match(r"^\d{4}-\d{2}-\d{2}T(\d{2})$", s)
+        if m2:
+            return int(m2.group(1))
+        return None
+
+
+def _record_day(rec: Dict[str, Any]) -> Optional[dt.date]:
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str):
+        try:
+            return dt.date.fromisoformat(ymd)
+        except Exception:
+            pass
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and len(raw) >= 8:
+        try:
+            return dt.datetime.strptime(raw[:8], "%Y%m%d").date()
+        except Exception:
+            return None
+    return None
+
+
+def _record_hour_or_none(rec: Dict[str, Any]) -> Optional[int]:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        try:
+            return int(raw.split("T", 1)[1][:2])
+        except Exception:
+            return None
+    return None
+
+
+# Printing helpers
+def _pretty_date_for_print(rec: Dict[str, Any]) -> str:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}T{raw.split('T',1)[1][:2]}"
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str) and _YMD_RE.match(ymd):
+        return ymd
+    if isinstance(raw, str) and len(raw) >= 8:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}"
+    return "unknown"
+
+
+def _context_str(
+    huc8: Optional[str] = None,
+    date_input: Optional[str] = None,
+    file_name: Optional[str] = None,
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+) -> str:
+    """
+    Builds a readable context summary for printing headers.
+    Example outputs:
+      - "HUC 12090301"
+      - "HUC 12090301, date '2017-08-30'"
+      - "HUC 12090301, range 2017-08-30 to 2017-09-01"
+      - "HUC 12090301, file 'PSS_3_0m_20170830T162251_BM.tif'"
+    """
+    parts = []
+    if huc8:
+        parts.append(f"HUC {huc8}")
+    if date_input:
+        parts.append(f"date '{date_input}'")
+    if start_date or end_date:
+        if start_date and end_date:
+            parts.append(f"range {start_date} to {end_date}")
+        elif start_date:
+            parts.append(f"from {start_date}")
+        elif end_date:
+            parts.append(f"until {end_date}")
+    if file_name:
+        parts.append(f"file '{file_name}'")
+
+    return ", ".join(parts) if parts else "your filters"
+
+
+def format_records_for_print(
+    records: List[Dict[str, Any]], context: Optional[str] = None
+) -> str:
+    if not records:
+        ctx = context or "your filters"
+        return f"Benchmark FIMs were not matched for {ctx}."
+
+    header = (
+        f"Following are the available benchmark data for {context}:\n"
+        if context
+        else ""
+    )
+
+    def _is_synthetic_tier_local(r: Dict[str, Any]) -> bool:
+        t = str(r.get("tier") or r.get("quality") or "").lower()
+        return "tier_4" in t or t.strip() == "4"
+
+    def _return_period_text_local(r: Dict[str, Any]) -> str:
+        rp = (
+            r.get("return_period")
+            or r.get("return_period_yr")
+            or r.get("rp")
+            or r.get("rp_years")
+        )
+        if rp is None:
+            return "synthetic flow (return period unknown)"
+        try:
+            rp_int = int(float(str(rp).strip().replace("yr", "").replace("-year", "")))
+            return f"{rp_int}-year synthetic flow"
+        except Exception:
+            return f"{rp} synthetic flow"
+
+    blocks: List[str] = []
+    for r in records:
+        tier = r.get("tier") or r.get("quality") or "Unknown"
+        res = r.get("resolution_m")
+        res_txt = f"{res}m" if res is not None else "NA"
+        fname = r.get("file_name") or "NA"
+
+        # Build lines with Tier-aware event text
+        lines = [f"Data Tier: {tier}"]
+        if _is_synthetic_tier_local(r):
+            lines.append(f"Return Period: {_return_period_text_local(r)}")
+        else:
+            date_str = _pretty_date_for_print(r)
+            lines.append(f"Benchmark FIM date: {date_str}")
+
+        lines.extend([
+            f"Spatial Resolution: {res_txt}",
+            f"Benchmark FIM raster name in DB: {fname}",
+        ])
+        blocks.append("\n".join(lines))
+
+    return (header + "\n\n".join(blocks)).strip()
+
+# S3 and json catalog
+def load_catalog_core() -> Dict[str, Any]:
+    obj = _S3.get_object(Bucket=BUCKET, Key=CATALOG_KEY)
+    return json.loads(obj["Body"].read().decode("utf-8", "replace"))
+
+
+def _list_prefix(prefix: str) -> List[str]:
+    keys: List[str] = []
+    paginator = _S3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=BUCKET, Prefix=prefix):
+        for obj in page.get("Contents", []) or []:
+            keys.append(obj["Key"])
+    return keys
+
+
+def _download(bucket: str, key: str, dest_path: str) -> str:
+    os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+    _S3.download_file(bucket, key, dest_path)
+    return dest_path
+
+# Get the files from s3 bucket
+def _folder_from_record(rec: Dict[str, Any]) -> str:
+    s3_key = rec.get("s3_key")
+    if not s3_key or "/" not in s3_key:
+        raise ValueError("Record lacks s3_key to derive folder")
+    return s3_key.rsplit("/", 1)[0] + "/"
+
+
+def _tif_key_from_record(rec: Dict[str, Any]) -> Optional[str]:
+    tif_url = rec.get("tif_url")
+    if isinstance(tif_url, str) and ".amazonaws.com/" in tif_url:
+        return tif_url.split(".amazonaws.com/", 1)[1]
+    fname = rec.get("file_name")
+    if not fname:
+        return None
+    return _folder_from_record(rec) + fname
+
+#Download that tif and the boundary file --> need to add building footprint automation as well.
+def download_fim_assets(record: Dict[str, Any], dest_dir: str) -> Dict[str, Any]:
+    """
+    Download the .tif (if present) and any .gpkg from the record's folder to dest_dir.
+    """
+    os.makedirs(dest_dir, exist_ok=True)
+    out = {"tif": None, "gpkg_files": []}
+
+    # TIF
+    tif_key = _tif_key_from_record(record)
+    if tif_key:
+        local = os.path.join(dest_dir, os.path.basename(tif_key))
+        if not os.path.exists(local):
+            _download(BUCKET, tif_key, local)
+        out["tif"] = local
+
+    # GPKGs (list folder)
+    folder = _folder_from_record(record)
+    for key in _list_prefix(folder):
+        if key.lower().endswith(".gpkg"):
+            local = os.path.join(dest_dir, os.path.basename(key))
+            if not os.path.exists(local):
+                _download(BUCKET, key, local)
+            out["gpkg_files"].append(local)
+
+    return out

fimeval/BuildingFootprint/evaluationwithBF.py

@@ -8,6 +8,7 @@ from plotly.subplots import make_subplots
 import plotly.graph_objects as go
 import seaborn as sns
 import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
 
 
 def Changeintogpkg(input_path, output_dir, layer_name):
@@ -21,8 +22,8 @@ def Changeintogpkg(input_path, output_dir, layer_name):
         output_gpkg = os.path.join(output_dir, f"{layer_name}.gpkg")
         gdf.to_file(output_gpkg, driver="GPKG")
         return output_gpkg
-    
-    
+
+
 def GetFloodedBuildingCountInfo(
     building_fp_path,
     study_area_path,
@@ -76,8 +77,12 @@ def GetFloodedBuildingCountInfo(
 
     count_centroids_in_contingency(contingency_map)
 
-    centroid_counts["Candidate"] = centroid_counts["True Positive"] + centroid_counts["False Positive"]
-    centroid_counts["Benchmark"] = centroid_counts["True Positive"] + centroid_counts["False Negative"]
+    centroid_counts["Candidate"] = (
+        centroid_counts["True Positive"] + centroid_counts["False Positive"]
+    )
+    centroid_counts["Benchmark"] = (
+        centroid_counts["True Positive"] + centroid_counts["False Negative"]
+    )
 
     total_buildings = len(clipped_buildings)
     percentages = {
@@ -93,7 +98,9 @@ def GetFloodedBuildingCountInfo(
     FAR = FP / (TP + FP) if (TP + FP) > 0 else 0
     POD = TP / (TP + FN) if (TP + FN) > 0 else 0
     if centroid_counts["Benchmark"] > 0:
-        BDR = (centroid_counts["Candidate"] - centroid_counts["Benchmark"]) / centroid_counts["Benchmark"]
+        BDR = (
+            centroid_counts["Candidate"] - centroid_counts["Benchmark"]
+        ) / centroid_counts["Benchmark"]
     else:
         BDR = 0
 
@@ -122,7 +129,9 @@ def GetFloodedBuildingCountInfo(
         ],
     }
     counts_df = pd.DataFrame(counts_data)
-    csv_file_path = os.path.join(save_dir, "EvaluationMetrics", f"BuildingCounts_{basename}.csv")
+    csv_file_path = os.path.join(
+        save_dir, "EvaluationMetrics", f"BuildingCounts_{basename}.csv"
+    )
     os.makedirs(os.path.dirname(csv_file_path), exist_ok=True)
     counts_df.to_csv(csv_file_path, index=False)
 
@@ -135,98 +144,146 @@ def GetFloodedBuildingCountInfo(
     ]
 
     fig = make_subplots(
-        rows=1, cols=2,
-        subplot_titles=("Building Counts on Different FIMs", "Contingency Flooded Building Counts"),
+        rows=1,
+        cols=2,
+        subplot_titles=(
+            "Building Counts on Different FIMs",
+            "Contingency Flooded Building Counts",
+        ),
     )
 
     fig.add_trace(
         go.Bar(
-            x=["Candidate"], y=[centroid_counts["Candidate"]],
-            text=[f"{centroid_counts['Candidate']}"], textposition="auto",
-            marker_color="#1c83eb", marker_line_color="black", marker_line_width=1,
+            x=["Candidate"],
+            y=[centroid_counts["Candidate"]],
+            text=[f"{centroid_counts['Candidate']}"],
+            textposition="auto",
+            marker_color="#1c83eb",
+            marker_line_color="black",
+            marker_line_width=1,
             name=f"Candidate ({percentages['Candidate']:.2f}%)",
         ),
-        row=1, col=1,
+        row=1,
+        col=1,
     )
     fig.add_trace(
         go.Bar(
-            x=["Benchmark"], y=[centroid_counts["Benchmark"]],
-            text=[f"{centroid_counts['Benchmark']}"], textposition="auto",
-            marker_color="#a4490e", marker_line_color="black", marker_line_width=1,
+            x=["Benchmark"],
+            y=[centroid_counts["Benchmark"]],
+            text=[f"{centroid_counts['Benchmark']}"],
+            textposition="auto",
+            marker_color="#a4490e",
+            marker_line_color="black",
+            marker_line_width=1,
             name=f"Benchmark ({percentages['Benchmark']:.2f}%)",
         ),
-        row=1, col=1,
+        row=1,
+        col=1,
     )
 
     for i, label in enumerate(third_raster_labels):
         fig.add_trace(
             go.Bar(
-                x=[label], y=[third_raster_counts[i]],
-                text=[f"{third_raster_counts[i]}"], textposition="auto",
+                x=[label],
+                y=[third_raster_counts[i]],
+                text=[f"{third_raster_counts[i]}"],
+                textposition="auto",
                 marker_color=["#ff5733", "#ffc300", "#28a745"][i],
-                marker_line_color="black", marker_line_width=1,
+                marker_line_color="black",
+                marker_line_width=1,
                 name=f"{label} ({percentages[label]:.2f}%)",
             ),
-            row=1, col=2,
+            row=1,
+            col=2,
         )
 
     fig.update_layout(
         title="Flooded Building Counts",
         xaxis_title="Inundation Surface",
         yaxis_title="Flooded Building Counts",
-        width=1100, height=400,
-        plot_bgcolor="rgba(0,0,0,0)", paper_bgcolor="rgba(0,0,0,0)",
-        showlegend=True, font=dict(family="Arial", size=18, color="black"),
+        width=1100,
+        height=400,
+        plot_bgcolor="rgba(0,0,0,0)",
+        paper_bgcolor="rgba(0,0,0,0)",
+        showlegend=True,
+        font=dict(family="Arial", size=18, color="black"),
     )
     fig.show()
 
-    # Seaborn for static PNG saving only
-    df_left = pd.DataFrame({
-        "Category": ["Candidate", "Benchmark"],
-        "Count": [centroid_counts["Candidate"], centroid_counts["Benchmark"]],
-    })
-    df_right = pd.DataFrame({
-        "Category": third_raster_labels,
-        "Count": third_raster_counts,
-    })
+    # Seaborn for static PNG
+    df_left = pd.DataFrame(
+        {
+            "Category": ["Candidate", "Benchmark"],
+            "Count": [centroid_counts["Candidate"], centroid_counts["Benchmark"]],
+        }
+    )
+    df_right = pd.DataFrame(
+        {
+            "Category": third_raster_labels,
+            "Count": third_raster_counts,
+        }
+    )
 
     sns.set_theme(style="whitegrid")
-    fig_sb, axes = plt.subplots(1, 2, figsize=(8, 3), constrained_layout=True)
+
+    fig_sb = plt.figure(figsize=(10, 3), constrained_layout=True)
+    gs = gridspec.GridSpec(1, 3, figure=fig_sb, width_ratios=[1, 1, 0.4])
+
+    ax0 = fig_sb.add_subplot(gs[0, 0])
+    ax1 = fig_sb.add_subplot(gs[0, 1])
+    ax_leg = fig_sb.add_subplot(gs[0, 2])
+    ax_leg.axis("off")
 
     def style_axes(ax, title_text, xlab, show_ylabel: bool):
-        # Adding a bit of padding so bar labels don’t overlap with the title
-        ax.set_title(title_text, fontsize=16, pad=20)
-        ax.set_xlabel(xlab, fontsize=14, color="black")
+        ax.set_title(title_text, fontsize=14, pad=15)
+        ax.set_xlabel(xlab, fontsize=13, color="black")
         if show_ylabel:
-            ax.set_ylabel("Flooded Building Counts", fontsize=14, color="black")
+            ax.set_ylabel("Flooded Building Counts", fontsize=13, color="black")
         else:
             ax.set_ylabel("")
 
-        # Thicker black left/bottom spines
         for spine in ("left", "bottom"):
             ax.spines[spine].set_linewidth(1.5)
             ax.spines[spine].set_color("black")
 
         sns.despine(ax=ax, right=True, top=True)
-        ax.tick_params(axis="x", labelsize=12, colors="black")
-        ax.tick_params(axis="y", labelsize=12, colors="black")
+        ax.tick_params(axis="x", labelsize=11, colors="black")
+        ax.tick_params(axis="y", labelsize=11, colors="black")
 
     # Left panel
-    ax0 = axes[0]
-    sns.barplot(data=df_left, x="Category", y="Count", ax=ax0,
-                palette=["#1c83eb", "#a4490e"])
-    style_axes(ax0, "Building Counts on Different FIMs", "Inundation Surface", show_ylabel=True)
+    colors_left = ["#1c83eb", "#a4490e"]
+    sns.barplot(data=df_left, x="Category", y="Count", ax=ax0, palette=colors_left)
+    style_axes(ax0, "Building Counts on Different FIMs", "Inundation Surface", True)
     for c in ax0.containers:
-        ax0.bar_label(c, fmt="%.0f", label_type="edge", padding=3, fontsize=14, color="black")
+        ax0.bar_label(
+            c, fmt="%.0f", label_type="edge", padding=3, fontsize=12, color="black"
+        )
 
     # Right panel
-    ax1 = axes[1]
-    sns.barplot(data=df_right, x="Category", y="Count", ax=ax1,
-                palette=["#ff5733", "#ffc300", "#28a745"])
-    style_axes(ax1, "Contingency Flooded Building Counts", "Category", show_ylabel=False)
+    colors_right = ["#ff5733", "#ffc300", "#28a745"]
+    sns.barplot(data=df_right, x="Category", y="Count", ax=ax1, palette=colors_right)
+    style_axes(ax1, "Contingency Flooded Building Counts", "Category", False)
     for c in ax1.containers:
-        ax1.bar_label(c, fmt="%.0f", label_type="edge", padding=3, fontsize=14, color="black")
+        ax1.bar_label(
+            c, fmt="%.0f", label_type="edge", padding=3, fontsize=12, color="black"
+        )
 
+    # Combined legend
+    all_labels = ["Candidate", "Benchmark"] + third_raster_labels
+    all_colors = colors_left + colors_right
+    legend_handles = [
+        plt.Line2D(
+            [0],
+            [0],
+            marker="s",
+            color="w",
+            markerfacecolor=all_colors[i],
+            markersize=12,
+            label=f"{all_labels[i]} ({percentages[all_labels[i]]:.2f}%)",
+        )
+        for i in range(len(all_labels))
+    ]
+    ax_leg.legend(handles=legend_handles, fontsize=12, loc="center left", frameon=True)
     plot_dir = os.path.join(save_dir, "FinalPlots")
     os.makedirs(plot_dir, exist_ok=True)
     output_path = os.path.join(plot_dir, f"BuildingCounts_{basename}.png")
@@ -236,8 +293,6 @@ def GetFloodedBuildingCountInfo(
     print(f"PNG were saved in : {output_path}")
 
 
-    
-
 def process_TIFF(
     tif_files, contingency_files, building_footprint, boundary, method_path
 ):

fimeval/BuildingFootprint/microsoftBF.py

@@ -130,3 +130,5 @@ def BuildingFootprintwithISO(countryISO, ROI, out_dir, geeprojectID=None):
     getBuildingFootprintSpark(
         countryISO, ROI, out_dir, tile_size=0.05, projectID=geeprojectID
     )
+
+BuildingFootprintwithISO("USA", "/Users/supath/Downloads/S1A_9_6m_20190530T23573_910244W430506N_AOI.gpkg", "/Users/supath/Downloads/AOI", geeprojectID="supathdh")