fimeval 0.1.57__py3-none-any.whl → 0.1.58__py3-none-any.whl

fimeval/BuildingFootprint/arcgis_API.py

@@ -0,0 +1,195 @@
+"""
+Author: Supath Dhital
+Date CreatedL January 2026
+
+Description: This will extract Microsoft Building Footprints using ArcGIS REST API for a given boundary.
+"""
+
+import geopandas as gpd
+import requests
+import pandas as pd
+from pathlib import Path
+from typing import Union, Optional
+
+
+# Main class
+class getBuildingFootprint:
+    """Extract Microsoft Building Footprints within a boundary using spatial queries."""
+
+    MSBFP_URL = "https://services.arcgis.com/P3ePLMYs2RVChkJx/arcgis/rest/services/MSBFP2/FeatureServer/0"
+
+    def __init__(
+        self,
+        boundary: Union[str, Path, gpd.GeoDataFrame],
+        layer: Optional[str] = None,
+        output_dir: Optional[Union[str, Path]] = None,
+        service_url: Optional[str] = None,
+    ):
+        """
+        Parameters
+        ----------
+        boundary : str, Path, or GeoDataFrame
+            Boundary as file path or GeoDataFrame
+        layer : str, optional
+            Layer name if boundary is a geopackage with multiple layers
+        service_url : str, optional
+            Custom ArcGIS feature service URL
+        """
+        self.boundary = self._load_boundary(boundary, layer)
+        self.service_url = service_url or self.MSBFP_URL
+
+        # Setup output directory
+        if output_dir is None:
+            output_dir = Path.cwd() / "BFOutputs"
+        else:
+            output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Run the code
+        self.extract(output_dir=output_dir)
+
+    def _load_boundary(
+        self, boundary: Union[str, Path, gpd.GeoDataFrame], layer: Optional[str]
+    ) -> gpd.GeoDataFrame:
+        """Load and validate boundary."""
+        if isinstance(boundary, gpd.GeoDataFrame):
+            gdf = boundary.copy()
+        else:
+            kwargs = {"layer": layer} if layer else {}
+            gdf = gpd.read_file(boundary, **kwargs)
+
+        # Ensure WGS84
+        if gdf.crs != "EPSG:4326":
+            gdf = gdf.to_crs("EPSG:4326")
+
+        return gdf
+
+    def extract(
+        self,
+        output_dir: Optional[Union[str, Path]] = None,
+        output_filename: str = "building_footprints.gpkg",
+        batch_size: int = 2000,
+        timeout: int = 60,
+        verbose: bool = True,
+    ) -> gpd.GeoDataFrame:
+        """
+        Extract building footprints within the boundary.
+
+        Parameters
+        ----------
+        output_dir : str or Path, optional
+            Output directory (defaults to cwd/building_footprints)
+        output_filename : str, default="building_footprints.gpkg"
+            Output filename
+        batch_size : int, default=2000
+            Number of features to retrieve per request
+        timeout : int, default=60
+            Request timeout in seconds
+        verbose : bool, default=True
+            Print progress messages
+
+        Returns
+        -------
+        GeoDataFrame
+            Extracted building footprints
+        """
+
+        # Get bounding box
+        xmin, ymin, xmax, ymax = self.boundary.total_bounds
+
+        if verbose:
+            print(f"Querying {self.service_url}...")
+
+        # Query the service
+        all_features = []
+        offset = 0
+        query_url = f"{self.service_url}/query"
+
+        while True:
+            params = {
+                "f": "geojson",
+                "where": "1=1",
+                "geometry": f"{xmin},{ymin},{xmax},{ymax}",
+                "geometryType": "esriGeometryEnvelope",
+                "inSR": "4326",
+                "spatialRel": "esriSpatialRelIntersects",
+                "outFields": "*",
+                "returnGeometry": "true",
+                "outSR": "4326",
+                "resultOffset": offset,
+                "resultRecordCount": batch_size,
+            }
+
+            try:
+                response = requests.get(query_url, params=params, timeout=timeout)
+
+                if response.status_code != 200:
+                    if verbose:
+                        print(f"Error {response.status_code}")
+                    break
+
+                data = response.json()
+
+                if "error" in data:
+                    if verbose:
+                        print(f"Server error: {data['error'].get('message')}")
+                    break
+
+                if "features" in data and data["features"]:
+                    batch_gdf = gpd.GeoDataFrame.from_features(
+                        data["features"], crs="EPSG:4326"
+                    )
+                    all_features.append(batch_gdf)
+
+                    if verbose:
+                        total = sum(len(gdf) for gdf in all_features)
+
+                    if len(data["features"]) < batch_size:
+                        break
+                    offset += batch_size
+                else:
+                    break
+
+            except requests.exceptions.Timeout:
+                if verbose:
+                    print("  Request timed out, retrying...")
+                continue
+            except Exception as e:
+                if verbose:
+                    print(f"  Error: {e}")
+                break
+
+        if not all_features:
+            if verbose:
+                print("No features found.")
+            return gpd.GeoDataFrame()
+
+        # Combine and process
+        gdf = pd.concat(all_features, ignore_index=True)
+        gdf = gpd.GeoDataFrame(gdf, crs="EPSG:4326")
+
+        # Remove duplicates
+        for id_field in ["OBJECTID", "FID", "ID"]:
+            if id_field in gdf.columns:
+                initial = len(gdf)
+                gdf = gdf.drop_duplicates(subset=[id_field])
+                if verbose and (initial - len(gdf)) > 0:
+                    print(f"Removed {initial - len(gdf)} duplicates")
+                break
+
+        # Clip to exact boundary
+        if verbose:
+            print(f"Clipping {len(gdf)} features to boundary...")
+        gdf = gpd.clip(gdf, self.boundary)
+
+        # Save
+        output_path = output_dir / output_filename
+        gdf.to_file(output_path, driver="GPKG")
+
+        if verbose:
+            print(f"\n{'='*60}")
+            print(f"SUCCESS: Saved {len(gdf)} buildings to:")
+            print(f"  {output_path}")
+            print(f"{'='*60}")
+
+        return gdf

fimeval/BuildingFootprint/evaluationwithBF.py

@@ -10,6 +10,9 @@ import seaborn as sns
 import matplotlib.pyplot as plt
 import matplotlib.gridspec as gridspec
 
+# Import building footprint module
+from .arcgis_API import getBuildingFootprint
+
 
 def Changeintogpkg(input_path, output_dir, layer_name):
     input_path = str(input_path)
@@ -355,38 +358,12 @@ def detect_shapefile(folder):
     return None
 
 
-def ensure_pyspark(version: str | None = "3.5.4") -> None:
-    """Install pyspark at runtime via `uv pip` into this env (no-op if present)."""
-    import importlib, shutil, subprocess, sys, re
-
-    try:
-        import importlib.util
-
-        if importlib.util.find_spec("pyspark"):
-            return
-    except Exception:
-        pass
-    uv = shutil.which("uv")
-    if not uv:
-        raise RuntimeError(
-            "`uv` not found on PATH. Please install uv or add it to PATH."
-        )
-    if version is None:
-        spec = "pyspark"
-    else:
-        v = version.strip()
-        spec = f"pyspark{v}" if re.match(r"^[<>=!~]", v) else f"pyspark=={v}"
-    subprocess.check_call([uv, "pip", "install", "--python", sys.executable, spec])
-
-
 def EvaluationWithBuildingFootprint(
     main_dir,
     method_name,
     output_dir,
-    country=None,
     building_footprint=None,
     shapefile_dir=None,
-    geeprojectID=None,
 ):
     tif_files_main = glob.glob(os.path.join(main_dir, "*.tif"))
     if tif_files_main:
@@ -410,31 +387,23 @@ def EvaluationWithBuildingFootprint(
 
                 building_footprintMS = building_footprint
 
+                # If no building footprint provided, extract using ArcGIS API
                 if building_footprintMS is None:
-                    ensure_pyspark()
-                    from .microsoftBF import BuildingFootprintwithISO
-
                     out_dir = os.path.join(method_path, "BuildingFootprint")
                     if not os.path.exists(out_dir):
                         os.makedirs(out_dir)
                     EX_building_footprint = find_existing_footprint(out_dir)
-                    if not EX_building_footprint:
+                    if EX_building_footprint:
+                        building_footprintMS = EX_building_footprint
+                    else:
                         boundary_dir = shapefile_dir if shapefile_dir else boundary
-
-                        if geeprojectID:
-                            BuildingFootprintwithISO(
-                                country,
-                                boundary_dir,
-                                out_dir,
-                                geeprojectID=geeprojectID,
-                            )
-                        else:
-                            BuildingFootprintwithISO(country, boundary_dir, out_dir)
-                        building_footprintMS = os.path.join(
-                            out_dir, f"building_footprint.gpkg"
+                        getBuildingFootprint(
+                            boundary=boundary_dir,
+                            output_dir=out_dir,
                         )
-                    else:
-                        building_footprintMS = EX_building_footprint
+                        # After downloading, find the newly created footprint
+                        building_footprintMS = find_existing_footprint(out_dir)
+
                 process_TIFF(
                     tif_files,
                     contingency_files,
@@ -471,33 +440,22 @@ def EvaluationWithBuildingFootprint(
                         building_footprintMS = building_footprint
 
                         if building_footprintMS is None:
-                            ensure_pyspark()
-                            from .microsoftBF import BuildingFootprintwithISO
-
                             out_dir = os.path.join(method_path, "BuildingFootprint")
                             if not os.path.exists(out_dir):
                                 os.makedirs(out_dir)
                             EX_building_footprint = find_existing_footprint(out_dir)
-                            if not EX_building_footprint:
+                            if EX_building_footprint:
+                                building_footprintMS = EX_building_footprint
+                            else:
                                 boundary_dir = (
                                     shapefile_dir if shapefile_dir else boundary
                                 )
-                                if geeprojectID:
-                                    BuildingFootprintwithISO(
-                                        country,
-                                        boundary_dir,
-                                        out_dir,
-                                        geeprojectID=geeprojectID,
-                                    )
-                                else:
-                                    BuildingFootprintwithISO(
-                                        country, boundary_dir, out_dir
-                                    )
-                                building_footprintMS = os.path.join(
-                                    out_dir, f"building_footprint.gpkg"
+                                getBuildingFootprint(
+                                    boundary=boundary_dir,
+                                    output_dir=out_dir,
                                 )
-                            else:
-                                building_footprintMS = EX_building_footprint
+                                # After downloading, find the newly created footprint
+                                building_footprintMS = find_existing_footprint(out_dir)
 
                         process_TIFF(
                             tif_files,

fimeval/ContingencyMap/__init__.py

@@ -1,6 +1,6 @@
 from .evaluationFIM import EvaluateFIM
 from .printcontingency import PrintContingencyMap
 from .plotevaluationmetrics import PlotEvaluationMetrics
-from .PWBs3 import get_PWB
+from .water_bodies import get_PWB, ExtractPWB
 
-__all__ = ["EvaluateFIM", "PrintContingencyMap", "PlotEvaluationMetrics", "get_PWB"]
+__all__ = ["EvaluateFIM", "PrintContingencyMap", "PlotEvaluationMetrics", "get_PWB", "ExtractPWB"]

fimeval/ContingencyMap/evaluationFIM.py

@@ -1,3 +1,7 @@
+"""
+Author: Supath Dhital
+Date Updated: January 2026
+"""
 import os
 import re
 import numpy as np
@@ -11,6 +15,7 @@ import pandas as pd
 from rasterio.warp import reproject, Resampling
 from rasterio.io import MemoryFile
 from rasterio import features
+from shapely.geometry import shape
 from rasterio.mask import mask
 
 os.environ["CHECK_DISK_FREE_SPACE"] = "NO"
@@ -19,9 +24,9 @@ import warnings
 
 warnings.filterwarnings("ignore", category=rasterio.errors.ShapeSkipWarning)
 
-from .methods import AOI, smallest_extent, convex_hull, get_smallest_raster_path
+from .methods import AOI, convex_hull, smallest_extent, get_smallest_raster_path
 from .metrics import evaluationmetrics
-from .PWBs3 import get_PWB
+from .water_bodies import ExtractPWB
 from ..utilis import MakeFIMsUniform, benchmark_name, find_best_boundary
 from ..setup_benchFIM import ensure_benchmark
 
@@ -68,7 +73,7 @@ def fix_permissions(path):
 
 # Function for the evalution of the model
 def evaluateFIM(
-    benchmark_path, candidate_paths, gdf, folder, method, output_dir, shapefile=None
+    benchmark_path, candidate_paths, PWB_Dir, folder, method, output_dir, shapefile=None
 ):
     # Lists to store evaluation metrics
     csi_values = []
@@ -129,8 +134,23 @@ def evaluateFIM(
         benchmark_nodata = src1.nodata
         benchmark_crs = src1.crs
         b_profile = src1.profile
+
+        #Getting the correct geometry shape and crs to extract PWB
+        boundary_shape = shape(bounding_geom[0])
+        boundary_gdf = gpd.GeoDataFrame(geometry=[boundary_shape], crs=benchmark_crs)
+
+        #Proceed the masking
         out_image1[out_image1 == benchmark_nodata] = 0
         out_image1 = np.where(out_image1 > 0, 2, 0).astype(np.float32)
+
+        #If PWB_Dir is provided, use the local PWB shapefile, else download from ArcGIS API
+        if PWB_Dir is not None:
+            gdf = gpd.read_file(PWB_Dir)
+        else:
+            #Get the permanent water bodies from ArcGIS REST API
+            pwb_obj = ExtractPWB(boundary = boundary_gdf, save = False)
+            gdf = pwb_obj.gdf
+
         gdf = gdf.to_crs(benchmark_crs)
         shapes1 = [
             geom for geom in gdf.geometry if geom is not None and not geom.is_empty
@@ -394,23 +414,18 @@ def safe_delete_folder(folder_path):
 
 def EvaluateFIM(
     main_dir,
-    method_name=None,  
-    output_dir=None,    
+    method_name=None,
+    output_dir=None,
     PWB_dir=None,
     shapefile_dir=None,
     target_crs=None,
     target_resolution=None,
     benchmark_dict=None,
 ):
-    if output_dir is None: 
+    if output_dir is None:
         output_dir = os.path.join(os.getcwd(), "Evaluation_Results")
 
     main_dir = Path(main_dir)
-    # Read the permanent water bodies
-    if PWB_dir is None:
-        gdf = get_PWB()
-    else:
-        gdf = gpd.read_file(PWB_dir)
 
     # Grant the permission to the main directory
     fix_permissions(main_dir)
@@ -429,8 +444,8 @@ def EvaluateFIM(
         if benchmark_path and candidate_path:
             if method_name is None:
                 local_method = "AOI"
-                
-                #For single case, if user have explicitly send boundary, use that, else use the boundary from the benchmark FIM evaluation
+
+                # For single case, if user have explicitly send boundary, use that, else use the boundary from the benchmark FIM evaluation
                 if shapefile_dir is not None:
                     local_shapefile = shapefile_dir
                 else:
@@ -444,20 +459,29 @@ def EvaluateFIM(
                     local_shapefile = str(boundary)
             else:
                 local_method = method_name
-                local_shapefile = shapefile_dir 
+                local_shapefile = shapefile_dir
 
             print(f"**Flood Inundation Evaluation of {folder_dir.name}**")
             try:
                 Metrics = evaluateFIM(
                     benchmark_path,
                     candidate_path,
-                    gdf,
+                    PWB_dir,
                     folder_dir,
                     local_method,
                     output_dir,
-                    shapefile=local_shapefile,  
+                    shapefile=local_shapefile,
                 )
-                print("\n", Metrics, "\n")
+                
+                # Print results in structured table format with 3 decimal points
+                candidate_names = [os.path.splitext(os.path.basename(path))[0] for path in candidate_path]
+                df_display = pd.DataFrame.from_dict(Metrics, orient='index')
+                df_display.columns = candidate_names
+                df_display.reset_index(inplace=True)
+                df_display.rename(columns={'index': 'Metrics'}, inplace=True)
+                print("\n")
+                print(df_display.to_string(index=False, float_format='%.3f'))
+                print("\n")
             except Exception as e:
                 print(f"Error evaluating {folder_dir.name}: {e}")
         else:
@@ -470,7 +494,7 @@ def EvaluateFIM(
     if TIFFfiles_main_dir:
 
         # Ensure benchmark is present if needed
-        TIFFfiles_main_dir = ensure_benchmark( 
+        TIFFfiles_main_dir = ensure_benchmark(
             main_dir, TIFFfiles_main_dir, benchmark_dict
         )
 
@@ -494,12 +518,10 @@ def EvaluateFIM(
                 tif_files = list(folder.glob("*.tif"))
 
                 if tif_files:
-                    processing_folder = folder / "processing"  
+                    processing_folder = folder / "processing"
                     try:
                         # Ensure benchmark is present if needed
-                        tif_files = ensure_benchmark(  
-                            folder, tif_files, benchmark_dict
-                        )
+                        tif_files = ensure_benchmark(folder, tif_files, benchmark_dict)
 
                         MakeFIMsUniform(
                             folder,
@@ -517,5 +539,4 @@ def EvaluateFIM(
                 else:
                     print(
                         f"Skipping {folder.name} as it doesn't contain any tif files."
-                    )
-
+                    )

fimeval/ContingencyMap/printcontingency.py

@@ -101,7 +101,9 @@ def getContingencyMap(raster_path, method_path):
     base_name = os.path.basename(raster_path).split(".")[0]
     output_path = os.path.join(plot_dir, f"{base_name}.png")
     plt.savefig(output_path, dpi=500, bbox_inches="tight")
-    plt.show()
+    plt.show(block=False) 
+    plt.pause(5.0)  
+    plt.close()
 
 
 def PrintContingencyMap(main_dir, method_name, out_dir):

fimeval/ContingencyMap/water_bodies.py

@@ -0,0 +1,175 @@
+"""
+Author: Supath Dhital
+Date Created: January 2026
+
+Description: This module extracts permanent water bodies
+using the ArcGIS REST API  and AWS S3 for a given boundary file. The mechanism is using AWS, it retrieve all the US permanent water bodies shapefile from S3 bucket and then clip on the boundary.
+
+This FIMeval module now uses the ArcGIS REST API to extract water bodies within a specified boundary. As it query data for only specified boundary, it is more efficient and faster than downloading the entire US water bodies dataset.
+"""
+
+# import Libraries
+import geopandas as gpd
+import boto3
+import botocore
+import os
+import tempfile
+import requests
+import pandas as pd
+import numpy as np
+import json
+from pathlib import Path
+from typing import Union, Optional
+from shapely.geometry import box
+
+#USING ANONYMOUS S3 CLIENT TO ACCESS PUBLIC DATA
+# Initialize an anonymous S3 client
+s3 = boto3.client(
+    "s3", config=botocore.config.Config(signature_version=botocore.UNSIGNED)
+)
+
+bucket_name = "sdmlab"
+pwb_folder = "PWB/"
+
+
+def PWB_inS3(s3_client, bucket, prefix):
+    """Download all components of a shapefile from S3 into a temporary directory."""
+    tmp_dir = tempfile.mkdtemp()
+    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
+    if "Contents" not in response:
+        raise ValueError("No files found in the specified S3 folder.")
+
+    for obj in response["Contents"]:
+        file_key = obj["Key"]
+        file_name = os.path.basename(file_key)
+        if file_name.endswith((".shp", ".shx", ".dbf", ".prj", ".cpg")):
+            local_path = os.path.join(tmp_dir, file_name)
+            s3_client.download_file(bucket, file_key, local_path)
+
+    shp_files = [f for f in os.listdir(tmp_dir) if f.endswith(".shp")]
+    if not shp_files:
+        raise ValueError("No .shp file found after download.")
+
+    shp_path = os.path.join(tmp_dir, shp_files[0])
+    return shp_path
+
+
+def get_PWB():
+    shp_path = PWB_inS3(s3, bucket_name, pwb_folder)
+    pwb = gpd.read_file(shp_path)
+    return pwb
+
+
+#USING ARCGIS REST TO ACCESS PUBLIC DATA- More fast
+class ExtractPWB:
+    SERVICE_URL = "https://services.arcgis.com/P3ePLMYs2RVChkJx/arcgis/rest/services/USA_Detailed_Water_Bodies/FeatureServer/0"
+
+    def __init__(
+        self,
+        boundary: Union[str, Path, gpd.GeoDataFrame],
+        layer: Optional[str] = None,
+        output_dir: Optional[Union[str, Path]] = None,
+        save: bool = True,
+        output_filename: str = "permanent_water.gpkg"
+    ):
+        self.boundary_gdf = self._load_boundary(boundary, layer)
+        self.output_dir = Path(output_dir) if output_dir else Path.cwd() / "PWBOutputs"
+        
+        # We store the final result in self.gdf so it can be accessed after init
+        self.gdf = self.extract(save=save, output_filename=output_filename)
+
+    def _load_boundary(self, boundary, layer):
+        if isinstance(boundary, gpd.GeoDataFrame):
+            gdf = boundary.copy()
+        else:
+            kwargs = {"layer": layer} if layer else {}
+            gdf = gpd.read_file(boundary, **kwargs)
+        return gdf.to_crs("EPSG:4326") if gdf.crs != "EPSG:4326" else gdf
+
+    def _get_query_envelopes(self, threshold=1.0):
+        xmin, ymin, xmax, ymax = self.boundary_gdf.total_bounds
+        cols = list(np.arange(xmin, xmax, threshold)) + [xmax]
+        rows = list(np.arange(ymin, ymax, threshold)) + [ymax]
+        
+        grid = []
+        for i in range(len(cols)-1):
+            for j in range(len(rows)-1):
+                grid.append({
+                    "xmin": cols[i], "ymin": rows[j], 
+                    "xmax": cols[i+1], "ymax": rows[j+1],
+                    "spatialReference": {"wkid": 4326}
+                })
+        return grid
+
+    def extract(self, save: bool = True, output_filename: str = "permanent_water.gpkg", verbose: bool = True) -> gpd.GeoDataFrame:
+        all_features = []
+        query_url = f"{self.SERVICE_URL}/query"
+        envelopes = self._get_query_envelopes()
+        
+        permanent_filter = "FTYPE IN ('Lake/Pond', 'Stream/River', 'Reservoir', 'Canal/Ditch')"
+
+        for env_idx, env in enumerate(envelopes):
+            offset = 0
+            limit = 1000 
+            
+            while True:
+                payload = {
+                    "f": "geojson",
+                    "where": permanent_filter,
+                    "geometry": json.dumps(env),
+                    "geometryType": "esriGeometryEnvelope",
+                    "inSR": "4326",
+                    "spatialRel": "esriSpatialRelIntersects",
+                    "outFields": "NAME,FTYPE,FCODE,SQKM",
+                    "returnGeometry": "true",
+                    "outSR": "4326",
+                    "resultOffset": offset,
+                    "resultRecordCount": limit
+                }
+
+                try:
+                    response = requests.post(query_url, data=payload, timeout=60)
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    features = data.get("features", [])
+                    if not features:
+                        break 
+                        
+                    batch_gdf = gpd.GeoDataFrame.from_features(features, crs="EPSG:4326")
+                    all_features.append(batch_gdf)
+                    
+                    if verbose and offset > 0:
+                        print(f"  Grid {env_idx}: Paginated to offset {offset}...")
+
+                    if len(features) < limit:
+                        break
+                    
+                    offset += limit
+                    
+                except Exception as e:
+                    print(f"Error at grid {env_idx}, offset {offset}: {e}")
+                    break
+
+        if not all_features:
+            print("No water bodies found.")
+            return gpd.GeoDataFrame()
+
+        # Combine and Deduplicate
+        full_gdf = pd.concat(all_features, ignore_index=True)
+        full_gdf = gpd.GeoDataFrame(full_gdf, crs="EPSG:4326")
+        full_gdf = full_gdf.loc[full_gdf.geometry.to_wkt().drop_duplicates().index]
+        
+        # Clip to exact AOI
+        final_gdf = gpd.clip(full_gdf, self.boundary_gdf)
+
+        # Conditional Saving
+        if save:
+            self.output_dir.mkdir(parents=True, exist_ok=True)
+            output_path = self.output_dir / output_filename
+            final_gdf.to_file(output_path, driver="GPKG")
+            if verbose: print(f"Saved {len(final_gdf)} features to {output_path}")
+        else:
+            if verbose: print(f"PWB Extraction complete.")
+
+        return final_gdf