fimeval 0.1.56__py3-none-any.whl → 0.1.58__py3-none-any.whl

fimeval/ContingencyMap/evaluationFIM.py

@@ -1,4 +1,9 @@
+"""
+Author: Supath Dhital
+Date Updated: January 2026
+"""
 import os
+import re
 import numpy as np
 from pathlib import Path
 import geopandas as gpd
@@ -10,16 +15,20 @@ import pandas as pd
 from rasterio.warp import reproject, Resampling
 from rasterio.io import MemoryFile
 from rasterio import features
+from shapely.geometry import shape
 from rasterio.mask import mask
 
+os.environ["CHECK_DISK_FREE_SPACE"] = "NO"
+
 import warnings
 
 warnings.filterwarnings("ignore", category=rasterio.errors.ShapeSkipWarning)
 
-from .methods import AOI, smallest_extent, convex_hull, get_smallest_raster_path
+from .methods import AOI, convex_hull, smallest_extent, get_smallest_raster_path
 from .metrics import evaluationmetrics
-from .PWBs3 import get_PWB
-from ..utilis import MakeFIMsUniform
+from .water_bodies import ExtractPWB
+from ..utilis import MakeFIMsUniform, benchmark_name, find_best_boundary
+from ..setup_benchFIM import ensure_benchmark
 
 
 # giving the permission to the folder
@@ -64,7 +73,7 @@ def fix_permissions(path):
 
 # Function for the evalution of the model
 def evaluateFIM(
-    benchmark_path, candidate_paths, gdf, folder, method, output_dir, shapefile=None
+    benchmark_path, candidate_paths, PWB_Dir, folder, method, output_dir, shapefile=None
 ):
     # Lists to store evaluation metrics
     csi_values = []
@@ -98,20 +107,18 @@ def evaluateFIM(
 
     # If method is AOI, and direct shapefile directory is not provided, then it will search for the shapefile in the folder
     if method.__name__ == "AOI":
-        # If shapefile is not provided, search in the folder
+        # Ubest-matching boundary file, prefer .gpkg from benchFIM downloads
         if shapefile is None:
-            for ext in (".shp", ".gpkg", ".geojson", ".kml"):
-                for file in os.listdir(folder):
-                    if file.lower().endswith(ext):
-                        shapefile = os.path.join(folder, file)
-                        print(f"Auto-detected shapefile: {shapefile}")
-                        break
-                if shapefile:
-                    break
-            if shapefile is None:
+            shapefile_path = find_best_boundary(Path(folder), Path(benchmark_path))
+            if shapefile_path is None:
                 raise FileNotFoundError(
-                    "No shapefile (.shp, .gpkg, .geojson, .kml) found in the folder and none provided. Either provide a shapefile directory or put shapefile inside folder directory."
+                    f"No boundary file (.gpkg, .shp, .geojson, .kml) found in {folder}. "
+                    "Either provide a shapefile path or place a boundary file in the folder."
                 )
+            shapefile = str(shapefile_path)
+        else:
+            shapefile = str(shapefile)
+
         # Run AOI with the found or provided shapefile
         bounding_geom = AOI(benchmark_path, shapefile, save_dir)
 
@@ -127,8 +134,23 @@ def evaluateFIM(
         benchmark_nodata = src1.nodata
         benchmark_crs = src1.crs
         b_profile = src1.profile
+
+        #Getting the correct geometry shape and crs to extract PWB
+        boundary_shape = shape(bounding_geom[0])
+        boundary_gdf = gpd.GeoDataFrame(geometry=[boundary_shape], crs=benchmark_crs)
+
+        #Proceed the masking
         out_image1[out_image1 == benchmark_nodata] = 0
         out_image1 = np.where(out_image1 > 0, 2, 0).astype(np.float32)
+
+        #If PWB_Dir is provided, use the local PWB shapefile, else download from ArcGIS API
+        if PWB_Dir is not None:
+            gdf = gpd.read_file(PWB_Dir)
+        else:
+            #Get the permanent water bodies from ArcGIS REST API
+            pwb_obj = ExtractPWB(boundary = boundary_gdf, save = False)
+            gdf = pwb_obj.gdf
+
         gdf = gdf.to_crs(benchmark_crs)
         shapes1 = [
             geom for geom in gdf.geometry if geom is not None and not geom.is_empty
@@ -277,8 +299,8 @@ def evaluateFIM(
                                 out_transform1,
                             )
                             merged = out_image1 + out_image2_resized
-                            merged[merged==7] = 5 
-                            
+                            merged[merged == 7] = 5
+
             # Get Evaluation Metrics
             (
                 unique_values,
@@ -392,19 +414,18 @@ def safe_delete_folder(folder_path):
 
 def EvaluateFIM(
     main_dir,
-    method_name,
-    output_dir,
+    method_name=None,
+    output_dir=None,
     PWB_dir=None,
     shapefile_dir=None,
     target_crs=None,
     target_resolution=None,
+    benchmark_dict=None,
 ):
+    if output_dir is None:
+        output_dir = os.path.join(os.getcwd(), "Evaluation_Results")
+
     main_dir = Path(main_dir)
-    # Read the permanent water bodies
-    if PWB_dir is None:
-        gdf = get_PWB()
-    else:
-        gdf = gpd.read_file(PWB_dir)
 
     # Grant the permission to the main directory
     fix_permissions(main_dir)
@@ -414,32 +435,55 @@ def EvaluateFIM(
         benchmark_path = None
         candidate_path = []
 
-        if len(tif_files) == 2:
-            for tif_file in tif_files:
-                if "benchmark" in tif_file.name.lower() or "BM" in tif_file.name:
-                    benchmark_path = tif_file
-                else:
-                    candidate_path.append(tif_file)
+        for tif_file in tif_files:
+            if benchmark_name(tif_file):
+                benchmark_path = tif_file
+            else:
+                candidate_path.append(tif_file)
+
+        if benchmark_path and candidate_path:
+            if method_name is None:
+                local_method = "AOI"
 
-        elif len(tif_files) > 2:
-            for tif_file in tif_files:
-                if "benchmark" in tif_file.name.lower() or "BM" in tif_file.name:
-                    benchmark_path = tif_file
+                # For single case, if user have explicitly send boundary, use that, else use the boundary from the benchmark FIM evaluation
+                if shapefile_dir is not None:
+                    local_shapefile = shapefile_dir
                 else:
-                    candidate_path.append(tif_file)
+                    boundary = find_best_boundary(folder_dir, benchmark_path)
+                    if boundary is None:
+                        print(
+                            f"Skipping {folder_dir.name}: no boundary file found "
+                            f"and method_name is None (auto-AOI)."
+                        )
+                        return
+                    local_shapefile = str(boundary)
+            else:
+                local_method = method_name
+                local_shapefile = shapefile_dir
 
-        if benchmark_path and candidate_path:
             print(f"**Flood Inundation Evaluation of {folder_dir.name}**")
-            Metrics = evaluateFIM(
-                benchmark_path,
-                candidate_path,
-                gdf,
-                folder_dir,
-                method_name,
-                output_dir,
-                shapefile_dir,
-            )
-            print("\n", Metrics, "\n")
+            try:
+                Metrics = evaluateFIM(
+                    benchmark_path,
+                    candidate_path,
+                    PWB_dir,
+                    folder_dir,
+                    local_method,
+                    output_dir,
+                    shapefile=local_shapefile,
+                )
+                
+                # Print results in structured table format with 3 decimal points
+                candidate_names = [os.path.splitext(os.path.basename(path))[0] for path in candidate_path]
+                df_display = pd.DataFrame.from_dict(Metrics, orient='index')
+                df_display.columns = candidate_names
+                df_display.reset_index(inplace=True)
+                df_display.rename(columns={'index': 'Metrics'}, inplace=True)
+                print("\n")
+                print(df_display.to_string(index=False, float_format='%.3f'))
+                print("\n")
+            except Exception as e:
+                print(f"Error evaluating {folder_dir.name}: {e}")
         else:
             print(
                 f"Skipping {folder_dir.name} as it doesn't have a valid benchmark and candidate configuration."
@@ -448,34 +492,51 @@ def EvaluateFIM(
     # Check if main_dir directly contains tif files
     TIFFfiles_main_dir = list(main_dir.glob("*.tif"))
     if TIFFfiles_main_dir:
-        MakeFIMsUniform(
-            main_dir, target_crs=target_crs, target_resolution=target_resolution
+
+        # Ensure benchmark is present if needed
+        TIFFfiles_main_dir = ensure_benchmark(
+            main_dir, TIFFfiles_main_dir, benchmark_dict
         )
 
-        # processing folder
         processing_folder = main_dir / "processing"
-        TIFFfiles = list(processing_folder.glob("*.tif"))
+        try:
+            MakeFIMsUniform(
+                main_dir, target_crs=target_crs, target_resolution=target_resolution
+            )
 
-        process_TIFF(TIFFfiles, main_dir)
-        safe_delete_folder(processing_folder)
+            # processing folder
+            TIFFfiles = list(processing_folder.glob("*.tif"))
+
+            process_TIFF(TIFFfiles, main_dir)
+        except Exception as e:
+            print(f"Error processing {main_dir}: {e}")
+        finally:
+            safe_delete_folder(processing_folder)
     else:
         for folder in main_dir.iterdir():
             if folder.is_dir():
                 tif_files = list(folder.glob("*.tif"))
 
                 if tif_files:
-                    MakeFIMsUniform(
-                        folder,
-                        target_crs=target_crs,
-                        target_resolution=target_resolution,
-                    )
-
                     processing_folder = folder / "processing"
-                    TIFFfiles = list(processing_folder.glob("*.tif"))
+                    try:
+                        # Ensure benchmark is present if needed
+                        tif_files = ensure_benchmark(folder, tif_files, benchmark_dict)
+
+                        MakeFIMsUniform(
+                            folder,
+                            target_crs=target_crs,
+                            target_resolution=target_resolution,
+                        )
+
+                        TIFFfiles = list(processing_folder.glob("*.tif"))
 
-                    process_TIFF(TIFFfiles, folder)
-                    safe_delete_folder(processing_folder)
+                        process_TIFF(TIFFfiles, folder)
+                    except Exception as e:
+                        print(f"Error processing folder {folder.name}: {e}")
+                    finally:
+                        safe_delete_folder(processing_folder)
                 else:
                     print(
                         f"Skipping {folder.name} as it doesn't contain any tif files."
-                    )
+                    )

fimeval/ContingencyMap/printcontingency.py

@@ -101,7 +101,9 @@ def getContingencyMap(raster_path, method_path):
     base_name = os.path.basename(raster_path).split(".")[0]
     output_path = os.path.join(plot_dir, f"{base_name}.png")
     plt.savefig(output_path, dpi=500, bbox_inches="tight")
-    plt.show()
+    plt.show(block=False) 
+    plt.pause(5.0)  
+    plt.close()
 
 
 def PrintContingencyMap(main_dir, method_name, out_dir):

fimeval/ContingencyMap/water_bodies.py

@@ -0,0 +1,175 @@
+"""
+Author: Supath Dhital
+Date Created: January 2026
+
+Description: This module extracts permanent water bodies
+using the ArcGIS REST API  and AWS S3 for a given boundary file. The mechanism is using AWS, it retrieve all the US permanent water bodies shapefile from S3 bucket and then clip on the boundary.
+
+This FIMeval module now uses the ArcGIS REST API to extract water bodies within a specified boundary. As it query data for only specified boundary, it is more efficient and faster than downloading the entire US water bodies dataset.
+"""
+
+# import Libraries
+import geopandas as gpd
+import boto3
+import botocore
+import os
+import tempfile
+import requests
+import pandas as pd
+import numpy as np
+import json
+from pathlib import Path
+from typing import Union, Optional
+from shapely.geometry import box
+
+#USING ANONYMOUS S3 CLIENT TO ACCESS PUBLIC DATA
+# Initialize an anonymous S3 client
+s3 = boto3.client(
+    "s3", config=botocore.config.Config(signature_version=botocore.UNSIGNED)
+)
+
+bucket_name = "sdmlab"
+pwb_folder = "PWB/"
+
+
+def PWB_inS3(s3_client, bucket, prefix):
+    """Download all components of a shapefile from S3 into a temporary directory."""
+    tmp_dir = tempfile.mkdtemp()
+    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
+    if "Contents" not in response:
+        raise ValueError("No files found in the specified S3 folder.")
+
+    for obj in response["Contents"]:
+        file_key = obj["Key"]
+        file_name = os.path.basename(file_key)
+        if file_name.endswith((".shp", ".shx", ".dbf", ".prj", ".cpg")):
+            local_path = os.path.join(tmp_dir, file_name)
+            s3_client.download_file(bucket, file_key, local_path)
+
+    shp_files = [f for f in os.listdir(tmp_dir) if f.endswith(".shp")]
+    if not shp_files:
+        raise ValueError("No .shp file found after download.")
+
+    shp_path = os.path.join(tmp_dir, shp_files[0])
+    return shp_path
+
+
+def get_PWB():
+    shp_path = PWB_inS3(s3, bucket_name, pwb_folder)
+    pwb = gpd.read_file(shp_path)
+    return pwb
+
+
+#USING ARCGIS REST TO ACCESS PUBLIC DATA- More fast
+class ExtractPWB:
+    SERVICE_URL = "https://services.arcgis.com/P3ePLMYs2RVChkJx/arcgis/rest/services/USA_Detailed_Water_Bodies/FeatureServer/0"
+
+    def __init__(
+        self,
+        boundary: Union[str, Path, gpd.GeoDataFrame],
+        layer: Optional[str] = None,
+        output_dir: Optional[Union[str, Path]] = None,
+        save: bool = True,
+        output_filename: str = "permanent_water.gpkg"
+    ):
+        self.boundary_gdf = self._load_boundary(boundary, layer)
+        self.output_dir = Path(output_dir) if output_dir else Path.cwd() / "PWBOutputs"
+        
+        # We store the final result in self.gdf so it can be accessed after init
+        self.gdf = self.extract(save=save, output_filename=output_filename)
+
+    def _load_boundary(self, boundary, layer):
+        if isinstance(boundary, gpd.GeoDataFrame):
+            gdf = boundary.copy()
+        else:
+            kwargs = {"layer": layer} if layer else {}
+            gdf = gpd.read_file(boundary, **kwargs)
+        return gdf.to_crs("EPSG:4326") if gdf.crs != "EPSG:4326" else gdf
+
+    def _get_query_envelopes(self, threshold=1.0):
+        xmin, ymin, xmax, ymax = self.boundary_gdf.total_bounds
+        cols = list(np.arange(xmin, xmax, threshold)) + [xmax]
+        rows = list(np.arange(ymin, ymax, threshold)) + [ymax]
+        
+        grid = []
+        for i in range(len(cols)-1):
+            for j in range(len(rows)-1):
+                grid.append({
+                    "xmin": cols[i], "ymin": rows[j], 
+                    "xmax": cols[i+1], "ymax": rows[j+1],
+                    "spatialReference": {"wkid": 4326}
+                })
+        return grid
+
+    def extract(self, save: bool = True, output_filename: str = "permanent_water.gpkg", verbose: bool = True) -> gpd.GeoDataFrame:
+        all_features = []
+        query_url = f"{self.SERVICE_URL}/query"
+        envelopes = self._get_query_envelopes()
+        
+        permanent_filter = "FTYPE IN ('Lake/Pond', 'Stream/River', 'Reservoir', 'Canal/Ditch')"
+
+        for env_idx, env in enumerate(envelopes):
+            offset = 0
+            limit = 1000 
+            
+            while True:
+                payload = {
+                    "f": "geojson",
+                    "where": permanent_filter,
+                    "geometry": json.dumps(env),
+                    "geometryType": "esriGeometryEnvelope",
+                    "inSR": "4326",
+                    "spatialRel": "esriSpatialRelIntersects",
+                    "outFields": "NAME,FTYPE,FCODE,SQKM",
+                    "returnGeometry": "true",
+                    "outSR": "4326",
+                    "resultOffset": offset,
+                    "resultRecordCount": limit
+                }
+
+                try:
+                    response = requests.post(query_url, data=payload, timeout=60)
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    features = data.get("features", [])
+                    if not features:
+                        break 
+                        
+                    batch_gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326")
+                    all_features.append(batch_gdf)
+                    
+                    if verbose and offset > 0:
+                        print(f"  Grid {env_idx}: Paginated to offset {offset}...")
+
+                    if len(features) < limit:
+                        break
+                    
+                    offset += limit
+                    
+                except Exception as e:
+                    print(f"Error at grid {env_idx}, offset {offset}: {e}")
+                    break
+
+        if not all_features:
+            print("No water bodies found.")
+            return gpd.GeoDataFrame()
+
+        # Combine and Deduplicate
+        full_gdf = pd.concat(all_features, ignore_index=True)
+        full_gdf = gpd.GeoDataFrame(full_gdf, crs="EPSG:4326")
+        full_gdf = full_gdf.loc[full_gdf.geometry.to_wkt().drop_duplicates().index]
+        
+        # Clip to exact AOI
+        final_gdf = gpd.clip(full_gdf, self.boundary_gdf)
+
+        # Conditional Saving
+        if save:
+            self.output_dir.mkdir(parents=True, exist_ok=True)
+            output_path = self.output_dir / output_filename
+            final_gdf.to_file(output_path, driver="GPKG")
+            if verbose: print(f"Saved {len(final_gdf)} features to {output_path}")
+        else:
+            if verbose: print(f"PWB Extraction complete.")
+
+        return final_gdf

fimeval/__init__.py

@@ -2,7 +2,7 @@
 from .ContingencyMap.evaluationFIM import EvaluateFIM
 from .ContingencyMap.printcontingency import PrintContingencyMap
 from .ContingencyMap.plotevaluationmetrics import PlotEvaluationMetrics
-from .ContingencyMap.PWBs3 import get_PWB
+from .ContingencyMap.water_bodies import get_PWB, ExtractPWB
 
 # Utility modules
 from .utilis import compress_tif_lzw
@@ -10,6 +10,12 @@ from .utilis import compress_tif_lzw
 # Evaluation with Building foorprint module
 from .BuildingFootprint.evaluationwithBF import EvaluationWithBuildingFootprint
 
+# Access benchmark FIM module
+from .BenchFIMQuery.access_benchfim import benchFIMquery
+
+# Building Footprint module
+from .BuildingFootprint.arcgis_API import getBuildingFootprint
+
 __all__ = [
     "EvaluateFIM",
     "PrintContingencyMap",
@@ -17,4 +23,7 @@ __all__ = [
     "get_PWB",
     "EvaluationWithBuildingFootprint",
     "compress_tif_lzw",
+    "benchFIMquery",
+    "getBuildingFootprint",
+    "ExtractPWB",
 ]

fimeval/setup_benchFIM.py

@@ -0,0 +1,41 @@
+"""
+This code setup all the case folders whether it has valid benchmark FIM/ which benchmark need to access from catalog and so on.
+Basically It will do everything before going into the actual evaluation process.
+Author: Supath Dhital
+Date updated: 25 Nov, 2025
+"""
+
+from pathlib import Path
+
+from .BenchFIMQuery.access_benchfim import benchFIMquery
+from .utilis import benchmark_name
+
+
+def ensure_benchmark(folder_dir, tif_files, benchmark_map):
+    """
+    If no local benchmark is found in `tif_files`, and `folder_dir.name`
+    exists in `benchmark_map`, download it into this folder using benchFIMquery.
+    Returns an updated list of tif files.
+    """
+    folder_dir = Path(folder_dir)
+
+    # If a benchmark/BM tif is already present, just use existing files
+    has_benchmark = any(benchmark_name(f) for f in tif_files)
+    if has_benchmark or not benchmark_map:
+        return tif_files
+
+    # If folder not in mapping, do nothing
+    folder_key = folder_dir.name
+    file_name = benchmark_map.get(folder_key)
+    if not file_name:
+        return tif_files
+
+    # Download benchmark FIM by filename into this folder
+    benchFIMquery(
+        file_name=file_name,
+        download=True,
+        out_dir=str(folder_dir),
+    )
+
+    # Return refreshed tif list
+    return list(folder_dir.glob("*.tif"))

fimeval/utilis.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 import pyproj
 import rasterio
@@ -182,3 +183,49 @@ def MakeFIMsUniform(fim_dir, target_crs=None, target_resolution=None):
             resample_to_resolution(str(src_path), coarsest_x, coarsest_y)
     else:
         print("All rasters already have the same resolution. No resampling needed.")
+
+
+# Function to find the best boundary file in the folder if multiple boundary files are present
+def find_best_boundary(folder: Path, benchmark_path: Path):
+    """
+    Choose the best boundary file in `folder`:
+    - prefer .gpkg (from benchFIM downloads),
+    - otherwise, pick the file with the most name tokens in common with the benchmark.
+    """
+    exts = [".gpkg", ".shp", ".geojson", ".kml"]
+    candidates = []
+    for ext in exts:
+        candidates.extend(folder.glob(f"*{ext}"))
+
+    if not candidates:
+        return None
+    if len(candidates) == 1:
+        print(f"Auto-detected boundary: {candidates[0]}")
+        return candidates[0]
+
+    bench_tokens = set(
+        t for t in re.split(r"[_\-\.\s]+", benchmark_path.stem.lower()) if t
+    )
+
+    def score(path: Path):
+        name_tokens = set(t for t in re.split(r"[_\-\.\s]+", path.stem.lower()) if t)
+        common = len(bench_tokens & name_tokens)
+        bonus = 1 if path.suffix.lower() == ".gpkg" else 0
+        return (common, bonus)
+
+    best = max(candidates, key=score)
+    print(f"Auto-detected boundary (best match to benchmark): {best}")
+    return best
+
+
+# To test whether the tif is benchmark or not
+def benchmark_name(f: Path) -> bool:
+    name = f.stem.lower()
+
+    # Explicit word
+    if "benchmark" in name:
+        return True
+
+    # Treating underscores/dashes/dots as separators and look for a 'bm' token
+    tokens = re.split(r"[_\-\.\s]+", name)
+    return "bm" in tokens