fimeval 0.1.56__py3-none-any.whl → 0.1.57__py3-none-any.whl

fimeval/BenchFIMQuery/utilis.py

@@ -0,0 +1,269 @@
+"""
+This utility function contains how to retrieve all the necessary metadata of benchmark FIM
+from the s3 bucket during benchmark FIM querying.
+
+Authors: Supath Dhital, sdhital@crimson.ua.edu
+Updated date: 25 Nov, 2025
+"""
+
+from __future__ import annotations
+import os, re, json, datetime as dt
+from typing import List, Dict, Any, Optional
+
+import urllib.parse
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
+
+# constants
+BUCKET = "sdmlab"
+CATALOG_KEY = (
+    "FIM_Database/FIM_Viz/catalog_core.json"  # Path of the json file in the s3 bucket
+)
+
+# s3 client
+_S3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+
+# helpers for direct S3 file links
+def s3_http_url(bucket: str, key: str) -> str:
+    """Build a public-style S3 HTTPS URL."""
+    return f"https://{bucket}.s3.amazonaws.com/{urllib.parse.quote(key, safe='/')}"
+
+
+# utils
+_YMD_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+_YMD_COMPACT_RE = re.compile(r"^\d{8}$")
+_YMDH_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}$")
+_YMDHMS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2})?$")
+
+
+def _normalize_user_dt(s: str) -> str:
+    s = s.strip()
+    s = s.replace("/", "-")
+    s = re.sub(r"\s+", " ", s)
+    return s
+
+
+def _to_date(s: str) -> dt.date:
+    s = _normalize_user_dt(s)
+    if _YMD_COMPACT_RE.match(s):
+        return dt.datetime.strptime(s, "%Y%m%d").date()
+    if _YMD_RE.match(s):
+        return dt.date.fromisoformat(s)
+    try:
+        return dt.datetime.fromisoformat(s).date()
+    except Exception:
+        m = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2})$", s)
+        if m:
+            return dt.datetime.fromisoformat(f"{m.group(1)} {m.group(2)}:00:00").date()
+        raise ValueError(f"Bad date format: {s}")
+
+
+def _to_hour_or_none(s: str) -> Optional[int]:
+    s = _normalize_user_dt(s)
+    if _YMD_RE.match(s) or _YMD_COMPACT_RE.match(s):
+        return None
+    m = re.match(r"^\d{4}-\d{2}-\d{2}[ T](\d{2})$", s)
+    if m:
+        return int(m.group(1))
+    try:
+        dt_obj = dt.datetime.fromisoformat(s)
+        return dt_obj.hour
+    except Exception:
+        m2 = re.match(r"^\d{4}-\d{2}-\d{2}T(\d{2})$", s)
+        if m2:
+            return int(m2.group(1))
+        return None
+
+
+def _record_day(rec: Dict[str, Any]) -> Optional[dt.date]:
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str):
+        try:
+            return dt.date.fromisoformat(ymd)
+        except Exception:
+            pass
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and len(raw) >= 8:
+        try:
+            return dt.datetime.strptime(raw[:8], "%Y%m%d").date()
+        except Exception:
+            return None
+    return None
+
+
+def _record_hour_or_none(rec: Dict[str, Any]) -> Optional[int]:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        try:
+            return int(raw.split("T", 1)[1][:2])
+        except Exception:
+            return None
+    return None
+
+
+# Printing helpers
+def _pretty_date_for_print(rec: Dict[str, Any]) -> str:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}T{raw.split('T',1)[1][:2]}"
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str) and _YMD_RE.match(ymd):
+        return ymd
+    if isinstance(raw, str) and len(raw) >= 8:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}"
+    return "unknown"
+
+
+def _context_str(
+    huc8: Optional[str] = None,
+    date_input: Optional[str] = None,
+    file_name: Optional[str] = None,
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+) -> str:
+    """
+    Builds a readable context summary for printing headers.
+    Example outputs:
+      - "HUC 12090301"
+      - "HUC 12090301, date '2017-08-30'"
+      - "HUC 12090301, range 2017-08-30 to 2017-09-01"
+      - "HUC 12090301, file 'PSS_3_0m_20170830T162251_BM.tif'"
+    """
+    parts = []
+    if huc8:
+        parts.append(f"HUC {huc8}")
+    if date_input:
+        parts.append(f"date '{date_input}'")
+    if start_date or end_date:
+        if start_date and end_date:
+            parts.append(f"range {start_date} to {end_date}")
+        elif start_date:
+            parts.append(f"from {start_date}")
+        elif end_date:
+            parts.append(f"until {end_date}")
+    if file_name:
+        parts.append(f"file '{file_name}'")
+
+    return ", ".join(parts) if parts else "your filters"
+
+
+def format_records_for_print(
+    records: List[Dict[str, Any]], context: Optional[str] = None
+) -> str:
+    if not records:
+        ctx = context or "your filters"
+        return f"Benchmark FIMs were not matched for {ctx}."
+
+    header = (
+        f"Following are the available benchmark data for {context}:\n"
+        if context
+        else ""
+    )
+
+    def _is_synthetic_tier_local(r: Dict[str, Any]) -> bool:
+        t = str(r.get("tier") or r.get("quality") or "").lower()
+        return "tier_4" in t or t.strip() == "4"
+
+    def _return_period_text_local(r: Dict[str, Any]) -> str:
+        rp = (
+            r.get("return_period")
+            or r.get("return_period_yr")
+            or r.get("rp")
+            or r.get("rp_years")
+        )
+        if rp is None:
+            return "synthetic flow (return period unknown)"
+        try:
+            rp_int = int(float(str(rp).strip().replace("yr", "").replace("-year", "")))
+            return f"{rp_int}-year synthetic flow"
+        except Exception:
+            return f"{rp} synthetic flow"
+
+    blocks: List[str] = []
+    for r in records:
+        tier = r.get("tier") or r.get("quality") or "Unknown"
+        res = r.get("resolution_m")
+        res_txt = f"{res}m" if res is not None else "NA"
+        fname = r.get("file_name") or "NA"
+
+        # Build lines with Tier-aware event text
+        lines = [f"Data Tier: {tier}"]
+        if _is_synthetic_tier_local(r):
+            lines.append(f"Return Period: {_return_period_text_local(r)}")
+        else:
+            date_str = _pretty_date_for_print(r)
+            lines.append(f"Benchmark FIM date: {date_str}")
+
+        lines.extend([
+            f"Spatial Resolution: {res_txt}",
+            f"Benchmark FIM raster name in DB: {fname}",
+        ])
+        blocks.append("\n".join(lines))
+
+    return (header + "\n\n".join(blocks)).strip()
+
+# S3 and json catalog
+def load_catalog_core() -> Dict[str, Any]:
+    obj = _S3.get_object(Bucket=BUCKET, Key=CATALOG_KEY)
+    return json.loads(obj["Body"].read().decode("utf-8", "replace"))
+
+
+def _list_prefix(prefix: str) -> List[str]:
+    keys: List[str] = []
+    paginator = _S3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=BUCKET, Prefix=prefix):
+        for obj in page.get("Contents", []) or []:
+            keys.append(obj["Key"])
+    return keys
+
+
+def _download(bucket: str, key: str, dest_path: str) -> str:
+    os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+    _S3.download_file(bucket, key, dest_path)
+    return dest_path
+
+# Get the files from s3 bucket
+def _folder_from_record(rec: Dict[str, Any]) -> str:
+    s3_key = rec.get("s3_key")
+    if not s3_key or "/" not in s3_key:
+        raise ValueError("Record lacks s3_key to derive folder")
+    return s3_key.rsplit("/", 1)[0] + "/"
+
+
+def _tif_key_from_record(rec: Dict[str, Any]) -> Optional[str]:
+    tif_url = rec.get("tif_url")
+    if isinstance(tif_url, str) and ".amazonaws.com/" in tif_url:
+        return tif_url.split(".amazonaws.com/", 1)[1]
+    fname = rec.get("file_name")
+    if not fname:
+        return None
+    return _folder_from_record(rec) + fname
+
+#Download that tif and the boundary file --> need to add building footprint automation as well.
+def download_fim_assets(record: Dict[str, Any], dest_dir: str) -> Dict[str, Any]:
+    """
+    Download the .tif (if present) and any .gpkg from the record's folder to dest_dir.
+    """
+    os.makedirs(dest_dir, exist_ok=True)
+    out = {"tif": None, "gpkg_files": []}
+
+    # TIF
+    tif_key = _tif_key_from_record(record)
+    if tif_key:
+        local = os.path.join(dest_dir, os.path.basename(tif_key))
+        if not os.path.exists(local):
+            _download(BUCKET, tif_key, local)
+        out["tif"] = local
+
+    # GPKGs (list folder)
+    folder = _folder_from_record(record)
+    for key in _list_prefix(folder):
+        if key.lower().endswith(".gpkg"):
+            local = os.path.join(dest_dir, os.path.basename(key))
+            if not os.path.exists(local):
+                _download(BUCKET, key, local)
+            out["gpkg_files"].append(local)
+
+    return out

fimeval/BuildingFootprint/microsoftBF.py

@@ -130,3 +130,5 @@ def BuildingFootprintwithISO(countryISO, ROI, out_dir, geeprojectID=None):
     getBuildingFootprintSpark(
         countryISO, ROI, out_dir, tile_size=0.05, projectID=geeprojectID
     )
+
+BuildingFootprintwithISO("USA", "/Users/supath/Downloads/S1A_9_6m_20190530T23573_910244W430506N_AOI.gpkg", "/Users/supath/Downloads/AOI", geeprojectID="supathdh")

fimeval/ContingencyMap/evaluationFIM.py

@@ -1,4 +1,5 @@
 import os
+import re
 import numpy as np
 from pathlib import Path
 import geopandas as gpd
@@ -12,6 +13,8 @@ from rasterio.io import MemoryFile
 from rasterio import features
 from rasterio.mask import mask
 
+os.environ["CHECK_DISK_FREE_SPACE"] = "NO"
+
 import warnings
 
 warnings.filterwarnings("ignore", category=rasterio.errors.ShapeSkipWarning)
@@ -19,7 +22,8 @@ warnings.filterwarnings("ignore", category=rasterio.errors.ShapeSkipWarning)
 from .methods import AOI, smallest_extent, convex_hull, get_smallest_raster_path
 from .metrics import evaluationmetrics
 from .PWBs3 import get_PWB
-from ..utilis import MakeFIMsUniform
+from ..utilis import MakeFIMsUniform, benchmark_name, find_best_boundary
+from ..setup_benchFIM import ensure_benchmark
 
 
 # giving the permission to the folder
@@ -98,20 +102,18 @@ def evaluateFIM(
 
     # If method is AOI, and direct shapefile directory is not provided, then it will search for the shapefile in the folder
     if method.__name__ == "AOI":
-        # If shapefile is not provided, search in the folder
+        # Ubest-matching boundary file, prefer .gpkg from benchFIM downloads
         if shapefile is None:
-            for ext in (".shp", ".gpkg", ".geojson", ".kml"):
-                for file in os.listdir(folder):
-                    if file.lower().endswith(ext):
-                        shapefile = os.path.join(folder, file)
-                        print(f"Auto-detected shapefile: {shapefile}")
-                        break
-                if shapefile:
-                    break
-            if shapefile is None:
+            shapefile_path = find_best_boundary(Path(folder), Path(benchmark_path))
+            if shapefile_path is None:
                 raise FileNotFoundError(
-                    "No shapefile (.shp, .gpkg, .geojson, .kml) found in the folder and none provided. Either provide a shapefile directory or put shapefile inside folder directory."
+                    f"No boundary file (.gpkg, .shp, .geojson, .kml) found in {folder}. "
+                    "Either provide a shapefile path or place a boundary file in the folder."
                 )
+            shapefile = str(shapefile_path)
+        else:
+            shapefile = str(shapefile)
+
         # Run AOI with the found or provided shapefile
         bounding_geom = AOI(benchmark_path, shapefile, save_dir)
 
@@ -277,8 +279,8 @@ def evaluateFIM(
                                 out_transform1,
                             )
                             merged = out_image1 + out_image2_resized
-                            merged[merged==7] = 5 
-                            
+                            merged[merged == 7] = 5
+
             # Get Evaluation Metrics
             (
                 unique_values,
@@ -392,13 +394,17 @@ def safe_delete_folder(folder_path):
 
 def EvaluateFIM(
     main_dir,
-    method_name,
-    output_dir,
+    method_name=None,  
+    output_dir=None,    
     PWB_dir=None,
     shapefile_dir=None,
     target_crs=None,
     target_resolution=None,
+    benchmark_dict=None,
 ):
+    if output_dir is None: 
+        output_dir = os.path.join(os.getcwd(), "Evaluation_Results")
+
     main_dir = Path(main_dir)
     # Read the permanent water bodies
     if PWB_dir is None:
@@ -414,32 +420,46 @@ def EvaluateFIM(
         benchmark_path = None
         candidate_path = []
 
-        if len(tif_files) == 2:
-            for tif_file in tif_files:
-                if "benchmark" in tif_file.name.lower() or "BM" in tif_file.name:
-                    benchmark_path = tif_file
-                else:
-                    candidate_path.append(tif_file)
+        for tif_file in tif_files:
+            if benchmark_name(tif_file):
+                benchmark_path = tif_file
+            else:
+                candidate_path.append(tif_file)
 
-        elif len(tif_files) > 2:
-            for tif_file in tif_files:
-                if "benchmark" in tif_file.name.lower() or "BM" in tif_file.name:
-                    benchmark_path = tif_file
+        if benchmark_path and candidate_path:
+            if method_name is None:
+                local_method = "AOI"
+                
+                #For single case, if user have explicitly send boundary, use that, else use the boundary from the benchmark FIM evaluation
+                if shapefile_dir is not None:
+                    local_shapefile = shapefile_dir
                 else:
-                    candidate_path.append(tif_file)
+                    boundary = find_best_boundary(folder_dir, benchmark_path)
+                    if boundary is None:
+                        print(
+                            f"Skipping {folder_dir.name}: no boundary file found "
+                            f"and method_name is None (auto-AOI)."
+                        )
+                        return
+                    local_shapefile = str(boundary)
+            else:
+                local_method = method_name
+                local_shapefile = shapefile_dir 
 
-        if benchmark_path and candidate_path:
             print(f"**Flood Inundation Evaluation of {folder_dir.name}**")
-            Metrics = evaluateFIM(
-                benchmark_path,
-                candidate_path,
-                gdf,
-                folder_dir,
-                method_name,
-                output_dir,
-                shapefile_dir,
-            )
-            print("\n", Metrics, "\n")
+            try:
+                Metrics = evaluateFIM(
+                    benchmark_path,
+                    candidate_path,
+                    gdf,
+                    folder_dir,
+                    local_method,
+                    output_dir,
+                    shapefile=local_shapefile,  
+                )
+                print("\n", Metrics, "\n")
+            except Exception as e:
+                print(f"Error evaluating {folder_dir.name}: {e}")
         else:
             print(
                 f"Skipping {folder_dir.name} as it doesn't have a valid benchmark and candidate configuration."
@@ -448,34 +468,54 @@ def EvaluateFIM(
     # Check if main_dir directly contains tif files
     TIFFfiles_main_dir = list(main_dir.glob("*.tif"))
     if TIFFfiles_main_dir:
-        MakeFIMsUniform(
-            main_dir, target_crs=target_crs, target_resolution=target_resolution
+
+        # Ensure benchmark is present if needed
+        TIFFfiles_main_dir = ensure_benchmark( 
+            main_dir, TIFFfiles_main_dir, benchmark_dict
         )
 
-        # processing folder
         processing_folder = main_dir / "processing"
-        TIFFfiles = list(processing_folder.glob("*.tif"))
+        try:
+            MakeFIMsUniform(
+                main_dir, target_crs=target_crs, target_resolution=target_resolution
+            )
 
-        process_TIFF(TIFFfiles, main_dir)
-        safe_delete_folder(processing_folder)
+            # processing folder
+            TIFFfiles = list(processing_folder.glob("*.tif"))
+
+            process_TIFF(TIFFfiles, main_dir)
+        except Exception as e:
+            print(f"Error processing {main_dir}: {e}")
+        finally:
+            safe_delete_folder(processing_folder)
     else:
         for folder in main_dir.iterdir():
             if folder.is_dir():
                 tif_files = list(folder.glob("*.tif"))
 
                 if tif_files:
-                    MakeFIMsUniform(
-                        folder,
-                        target_crs=target_crs,
-                        target_resolution=target_resolution,
-                    )
+                    processing_folder = folder / "processing"  
+                    try:
+                        # Ensure benchmark is present if needed
+                        tif_files = ensure_benchmark(  
+                            folder, tif_files, benchmark_dict
+                        )
+
+                        MakeFIMsUniform(
+                            folder,
+                            target_crs=target_crs,
+                            target_resolution=target_resolution,
+                        )
 
-                    processing_folder = folder / "processing"
-                    TIFFfiles = list(processing_folder.glob("*.tif"))
+                        TIFFfiles = list(processing_folder.glob("*.tif"))
 
-                    process_TIFF(TIFFfiles, folder)
-                    safe_delete_folder(processing_folder)
+                        process_TIFF(TIFFfiles, folder)
+                    except Exception as e:
+                        print(f"Error processing folder {folder.name}: {e}")
+                    finally:
+                        safe_delete_folder(processing_folder)
                 else:
                     print(
                         f"Skipping {folder.name} as it doesn't contain any tif files."
                     )
+

fimeval/__init__.py

@@ -10,6 +10,9 @@ from .utilis import compress_tif_lzw
 # Evaluation with Building foorprint module
 from .BuildingFootprint.evaluationwithBF import EvaluationWithBuildingFootprint
 
+#Access benchmark FIM module
+from .BenchFIMQuery.access_benchfim import benchFIMquery
+
 __all__ = [
     "EvaluateFIM",
     "PrintContingencyMap",
@@ -17,4 +20,5 @@ __all__ = [
     "get_PWB",
     "EvaluationWithBuildingFootprint",
     "compress_tif_lzw",
+    "benchFIMquery",
 ]

fimeval/setup_benchFIM.py

@@ -0,0 +1,39 @@
+"""
+This code setup all the case folders whether it has valid benchmark FIM/ which benchmark need to access from catalog and so on.
+Basically It will do everything before going into the actual evaluation process.
+Author: Supath Dhital
+Date updated: 25 Nov, 2025
+"""
+from pathlib import Path
+
+from .BenchFIMQuery.access_benchfim import benchFIMquery
+from .utilis import benchmark_name
+
+def ensure_benchmark(folder_dir, tif_files, benchmark_map):
+    """
+    If no local benchmark is found in `tif_files`, and `folder_dir.name`
+    exists in `benchmark_map`, download it into this folder using benchFIMquery.
+    Returns an updated list of tif files.
+    """
+    folder_dir = Path(folder_dir)
+
+    # If a benchmark/BM tif is already present, just use existing files
+    has_benchmark = any(benchmark_name(f) for f in tif_files)
+    if has_benchmark or not benchmark_map:
+        return tif_files
+
+    # If folder not in mapping, do nothing
+    folder_key = folder_dir.name
+    file_name = benchmark_map.get(folder_key)
+    if not file_name:
+        return tif_files
+
+    # Download benchmark FIM by filename into this folder
+    benchFIMquery(
+        file_name=file_name,
+        download=True,
+        out_dir=str(folder_dir),
+    )
+
+    # Return refreshed tif list
+    return list(folder_dir.glob("*.tif"))

fimeval/utilis.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 import pyproj
 import rasterio
@@ -182,3 +183,51 @@ def MakeFIMsUniform(fim_dir, target_crs=None, target_resolution=None):
             resample_to_resolution(str(src_path), coarsest_x, coarsest_y)
     else:
         print("All rasters already have the same resolution. No resampling needed.")
+
+#Function to find the best boundary file in the folder if multiple boundary files are present
+def find_best_boundary(folder: Path, benchmark_path: Path):  
+    """
+    Choose the best boundary file in `folder`:
+    - prefer .gpkg (from benchFIM downloads),
+    - otherwise, pick the file with the most name tokens in common with the benchmark.
+    """
+    exts = [".gpkg", ".shp", ".geojson", ".kml"]
+    candidates = []
+    for ext in exts:
+        candidates.extend(folder.glob(f"*{ext}"))
+
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        print(f"Auto-detected boundary: {candidates[0]}")
+        return candidates[0]
+
+    bench_tokens = set(
+        t for t in re.split(r"[_\-\.\s]+", benchmark_path.stem.lower()) if t
+    )
+
+    def score(path: Path):
+        name_tokens = set(
+            t for t in re.split(r"[_\-\.\s]+", path.stem.lower()) if t
+        )
+        common = len(bench_tokens & name_tokens)
+        bonus = 1 if path.suffix.lower() == ".gpkg" else 0 
+        return (common, bonus)
+
+    best = max(candidates, key=score)
+    print(f"Auto-detected boundary (best match to benchmark): {best}")
+    return best
+
+
+#To test whether the tif is benchmark or not
+def benchmark_name(f: Path) -> bool:
+    name = f.stem.lower()
+
+    # Explicit word
+    if "benchmark" in name:
+        return True
+
+    # Treating underscores/dashes/dots as separators and look for a 'bm' token
+    tokens = re.split(r"[_\-\.\s]+", name)
+    return "bm" in tokens
+