fimeval 0.1.56__py3-none-any.whl → 0.1.58__py3-none-any.whl

fimeval/BenchFIMQuery/utilis.py

@@ -0,0 +1,334 @@
+"""
+This utility function contains how to retrieve all the necessary metadata of benchmark FIM
+from the s3 bucket during benchmark FIM querying.
+
+Authors: Supath Dhital, sdhital@crimson.ua.edu
+Updated date: 25 Nov, 2025
+"""
+
+from __future__ import annotations
+import os, re, json, datetime as dt
+from typing import List, Dict, Any, Optional
+
+import urllib.parse
+import boto3
+from botocore import UNSIGNED
+from botocore.config import Config
+
+# constants
+BUCKET = "sdmlab"
+CATALOG_KEY = (
+    "FIM_Database/FIM_Viz/catalog_core.json"  # Path of the json file in the s3 bucket
+)
+
+# s3 client
+_S3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
+
+
+# helpers for direct S3 file links
+def s3_http_url(bucket: str, key: str) -> str:
+    """Build a public-style S3 HTTPS URL."""
+    return f"https://{bucket}.s3.amazonaws.com/{urllib.parse.quote(key, safe='/')}"
+
+
+# utils
+_YMD_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+_YMD_COMPACT_RE = re.compile(r"^\d{8}$")
+_YMDH_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}$")
+_YMDHMS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(:\d{2})?$")
+
+
+#Support functions to ensute the geopackage file is local
+def _s3_bucket_key_from_http_url(url: str) -> Optional[tuple[str, str]]:
+    try:
+        u = urllib.parse.urlparse(url)
+        host = (u.netloc or "").lower()
+        path = (u.path or "").lstrip("/")
+        if not host or not path:
+            return None
+
+        if ".s3." in host or host.endswith(".s3.amazonaws.com"):
+            bucket = host.split(".s3", 1)[0]
+            key = path
+            return bucket, key
+
+        return None
+    except Exception:
+        return None
+
+
+def _ensure_local_gpkg(uri: str) -> str:
+    """
+    Ensure we can read a GPKG even when uri is an https S3 URL by caching locally.
+    Returns a path usable by geopandas.read_file().
+    """
+    if not isinstance(uri, str) or not uri.strip():
+        return uri
+
+    u = uri.strip()
+    if os.path.exists(u):
+        return u
+
+    # Cache https S3 gpkg locally
+    if u.lower().startswith("http") and ".amazonaws.com/" in u.lower() and u.lower().endswith(".gpkg"):
+        parsed = _s3_bucket_key_from_http_url(u.split("?", 1)[0])
+        if not parsed:
+            return u
+
+        bucket, key = parsed
+        cache_dir = os.path.join(os.path.expanduser("~"), ".fimeval_cache", "aoi_gpkg")
+        os.makedirs(cache_dir, exist_ok=True)
+
+        local = os.path.join(cache_dir, os.path.basename(key))
+        if not os.path.exists(local):
+            _download(bucket, key, local)
+        return local
+    return u
+
+def _normalize_user_dt(s: str) -> str:
+    s = s.strip()
+    s = s.replace("/", "-")
+    s = re.sub(r"\s+", " ", s)
+    return s
+
+def _display_raster_name(rec: Dict[str, Any]) -> str:
+    tif_url = rec.get("tif_url")
+    if isinstance(tif_url, str) and tif_url.strip():
+        # drop querystring if any
+        tif_url = tif_url.split("?", 1)[0]
+        return os.path.basename(tif_url)
+
+    # fallback: last path token of id
+    rid = rec.get("id")
+    if isinstance(rid, str) and rid.strip():
+        return rid.strip().split("/")[-1] + ".tif"
+
+    return "NA"
+
+def _to_date(s: str) -> dt.date:
+    s = _normalize_user_dt(s)
+    if _YMD_COMPACT_RE.match(s):
+        return dt.datetime.strptime(s, "%Y%m%d").date()
+    if _YMD_RE.match(s):
+        return dt.date.fromisoformat(s)
+    try:
+        return dt.datetime.fromisoformat(s).date()
+    except Exception:
+        m = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2})$", s)
+        if m:
+            return dt.datetime.fromisoformat(f"{m.group(1)} {m.group(2)}:00:00").date()
+        raise ValueError(f"Bad date format: {s}")
+
+
+def _to_hour_or_none(s: str) -> Optional[int]:
+    s = _normalize_user_dt(s)
+    if _YMD_RE.match(s) or _YMD_COMPACT_RE.match(s):
+        return None
+    m = re.match(r"^\d{4}-\d{2}-\d{2}[ T](\d{2})$", s)
+    if m:
+        return int(m.group(1))
+    try:
+        dt_obj = dt.datetime.fromisoformat(s)
+        return dt_obj.hour
+    except Exception:
+        m2 = re.match(r"^\d{4}-\d{2}-\d{2}T(\d{2})$", s)
+        if m2:
+            return int(m2.group(1))
+        return None
+
+
+def _record_day(rec: Dict[str, Any]) -> Optional[dt.date]:
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str):
+        try:
+            return dt.date.fromisoformat(ymd)
+        except Exception:
+            pass
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and len(raw) >= 8:
+        try:
+            return dt.datetime.strptime(raw[:8], "%Y%m%d").date()
+        except Exception:
+            return None
+    return None
+
+
+def _record_hour_or_none(rec: Dict[str, Any]) -> Optional[int]:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        try:
+            return int(raw.split("T", 1)[1][:2])
+        except Exception:
+            return None
+    return None
+
+
+# Printing helpers
+def _pretty_date_for_print(rec: Dict[str, Any]) -> str:
+    raw = rec.get("date_of_flood")
+    if isinstance(raw, str) and "T" in raw and len(raw) >= 11:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}T{raw.split('T',1)[1][:2]}"
+    ymd = rec.get("date_ymd")
+    if isinstance(ymd, str) and _YMD_RE.match(ymd):
+        return ymd
+    if isinstance(raw, str) and len(raw) >= 8:
+        return f"{raw[:4]}-{raw[4:6]}-{raw[6:8]}"
+    return "unknown"
+
+
+def _context_str(
+    huc8: Optional[str] = None,
+    date_input: Optional[str] = None,
+    file_name: Optional[str] = None,
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+) -> str:
+    """
+    Builds a readable context summary for printing headers.
+    Example outputs:
+      - "HUC 12090301"
+      - "HUC 12090301, date '2017-08-30'"
+      - "HUC 12090301, range 2017-08-30 to 2017-09-01"
+      - "HUC 12090301, file 'PSS_3_0m_20170830T162251_BM.tif'"
+    """
+    parts = []
+    if huc8:
+        parts.append(f"HUC {huc8}")
+    if date_input:
+        parts.append(f"date '{date_input}'")
+    if start_date or end_date:
+        if start_date and end_date:
+            parts.append(f"range {start_date} to {end_date}")
+        elif start_date:
+            parts.append(f"from {start_date}")
+        elif end_date:
+            parts.append(f"until {end_date}")
+    if file_name:
+        parts.append(f"file '{file_name}'")
+
+    return ", ".join(parts) if parts else "your filters"
+
+
+def format_records_for_print(
+    records: List[Dict[str, Any]], context: Optional[str] = None
+) -> str:
+    if not records:
+        ctx = context or "your filters"
+        return f"Benchmark FIMs were not matched for {ctx}."
+
+    header = (
+        f"Following are the available benchmark data for {context}:\n"
+        if context
+        else ""
+    )
+
+    def _is_synthetic_tier_local(r: Dict[str, Any]) -> bool:
+        t = str(r.get("tier") or r.get("quality") or "").lower()
+        return "tier_4" in t or t.strip() == "4"
+
+    def _return_period_text_local(r: Dict[str, Any]) -> str:
+        rp = (
+            r.get("return_period")
+            or r.get("return_period_yr")
+            or r.get("rp")
+            or r.get("rp_years")
+        )
+        if rp is None:
+            return "synthetic flow (return period unknown)"
+        try:
+            rp_int = int(float(str(rp).strip().replace("yr", "").replace("-year", "")))
+            return f"{rp_int}-year synthetic flow"
+        except Exception:
+            return f"{rp} synthetic flow"
+
+    blocks: List[str] = []
+    for r in records:
+        tier = r.get("tier") or r.get("quality") or "Unknown"
+        res = r.get("resolution_m")
+        res_txt = f"{res}m" if res is not None else "NA"
+        fname = _display_raster_name(r)
+
+        # Build lines with Tier-aware event text
+        lines = [f"Data Tier: {tier}"]
+        if _is_synthetic_tier_local(r):
+            lines.append(f"Return Period: {_return_period_text_local(r)}")
+        else:
+            date_str = _pretty_date_for_print(r)
+            lines.append(f"Benchmark FIM date: {date_str}")
+
+        lines.extend(
+            [
+                f"Spatial Resolution: {res_txt}",
+                f"Benchmark FIM raster name in DB: {fname}",
+            ]
+        )
+        blocks.append("\n".join(lines))
+
+    return (header + "\n\n".join(blocks)).strip()
+
+
+# S3 and json catalog
+def load_catalog_core() -> Dict[str, Any]:
+    obj = _S3.get_object(Bucket=BUCKET, Key=CATALOG_KEY)
+    return json.loads(obj["Body"].read().decode("utf-8", "replace"))
+
+
+def _list_prefix(prefix: str) -> List[str]:
+    keys: List[str] = []
+    paginator = _S3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=BUCKET, Prefix=prefix):
+        for obj in page.get("Contents", []) or []:
+            keys.append(obj["Key"])
+    return keys
+
+
+def _download(bucket: str, key: str, dest_path: str) -> str:
+    os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+    _S3.download_file(bucket, key, dest_path)
+    return dest_path
+
+
+# Get the files from s3 bucket
+def _folder_from_record(rec: Dict[str, Any]) -> str:
+    s3_key = rec.get("s3_key")
+    if not s3_key or "/" not in s3_key:
+        raise ValueError("Record lacks s3_key to derive folder")
+    return s3_key.rsplit("/", 1)[0] + "/"
+
+
+def _tif_key_from_record(rec: Dict[str, Any]) -> Optional[str]:
+    tif_url = rec.get("tif_url")
+    if isinstance(tif_url, str) and ".amazonaws.com/" in tif_url:
+        return tif_url.split(".amazonaws.com/", 1)[1]
+    fname = rec.get("file_name")
+    if not fname:
+        return None
+    return _folder_from_record(rec) + fname
+
+
+# Download that tif and the boundary file --> need to add building footprint automation as well.
+def download_fim_assets(record: Dict[str, Any], dest_dir: str) -> Dict[str, Any]:
+    """
+    Download the .tif (if present) and any .gpkg from the record's folder to dest_dir.
+    """
+    os.makedirs(dest_dir, exist_ok=True)
+    out = {"tif": None, "gpkg_files": []}
+
+    # TIF
+    tif_key = _tif_key_from_record(record)
+    if tif_key:
+        local = os.path.join(dest_dir, os.path.basename(tif_key))
+        if not os.path.exists(local):
+            _download(BUCKET, tif_key, local)
+        out["tif"] = local
+
+    # GPKGs (list folder)
+    folder = _folder_from_record(record)
+    for key in _list_prefix(folder):
+        if key.lower().endswith(".gpkg"):
+            local = os.path.join(dest_dir, os.path.basename(key))
+            if not os.path.exists(local):
+                _download(BUCKET, key, local)
+            out["gpkg_files"].append(local)
+
+    return out

fimeval/BuildingFootprint/__init__.py

@@ -1,3 +1,4 @@
 from .evaluationwithBF import EvaluationWithBuildingFootprint
+from .arcgis_API import getBuildingFootprint
 
-__all__ = ["EvaluationWithBuildingFootprint"]
+__all__ = ["EvaluationWithBuildingFootprint", "getBuildingFootprint"]

fimeval/BuildingFootprint/arcgis_API.py

@@ -0,0 +1,195 @@
+"""
+Author: Supath Dhital
+Date CreatedL January 2026
+
+Description: This will extract Microsoft Building Footprints using ArcGIS REST API for a given boundary.
+"""
+
+import geopandas as gpd
+import requests
+import pandas as pd
+from pathlib import Path
+from typing import Union, Optional
+
+
+# Main class
+class getBuildingFootprint:
+    """Extract Microsoft Building Footprints within a boundary using spatial queries."""
+
+    MSBFP_URL = "https://services.arcgis.com/P3ePLMYs2RVChkJx/arcgis/rest/services/MSBFP2/FeatureServer/0"
+
+    def __init__(
+        self,
+        boundary: Union[str, Path, gpd.GeoDataFrame],
+        layer: Optional[str] = None,
+        output_dir: Optional[Union[str, Path]] = None,
+        service_url: Optional[str] = None,
+    ):
+        """
+        Parameters
+        ----------
+        boundary : str, Path, or GeoDataFrame
+            Boundary as file path or GeoDataFrame
+        layer : str, optional
+            Layer name if boundary is a geopackage with multiple layers
+        service_url : str, optional
+            Custom ArcGIS feature service URL
+        """
+        self.boundary = self._load_boundary(boundary, layer)
+        self.service_url = service_url or self.MSBFP_URL
+
+        # Setup output directory
+        if output_dir is None:
+            output_dir = Path.cwd() / "BFOutputs"
+        else:
+            output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Run the code
+        self.extract(output_dir=output_dir)
+
+    def _load_boundary(
+        self, boundary: Union[str, Path, gpd.GeoDataFrame], layer: Optional[str]
+    ) -> gpd.GeoDataFrame:
+        """Load and validate boundary."""
+        if isinstance(boundary, gpd.GeoDataFrame):
+            gdf = boundary.copy()
+        else:
+            kwargs = {"layer": layer} if layer else {}
+            gdf = gpd.read_file(boundary, **kwargs)
+
+        # Ensure WGS84
+        if gdf.crs != "EPSG:4326":
+            gdf = gdf.to_crs("EPSG:4326")
+
+        return gdf
+
+    def extract(
+        self,
+        output_dir: Optional[Union[str, Path]] = None,
+        output_filename: str = "building_footprints.gpkg",
+        batch_size: int = 2000,
+        timeout: int = 60,
+        verbose: bool = True,
+    ) -> gpd.GeoDataFrame:
+        """
+        Extract building footprints within the boundary.
+
+        Parameters
+        ----------
+        output_dir : str or Path, optional
+            Output directory (defaults to cwd/building_footprints)
+        output_filename : str, default="building_footprints.gpkg"
+            Output filename
+        batch_size : int, default=2000
+            Number of features to retrieve per request
+        timeout : int, default=60
+            Request timeout in seconds
+        verbose : bool, default=True
+            Print progress messages
+
+        Returns
+        -------
+        GeoDataFrame
+            Extracted building footprints
+        """
+
+        # Get bounding box
+        xmin, ymin, xmax, ymax = self.boundary.total_bounds
+
+        if verbose:
+            print(f"Querying {self.service_url}...")
+
+        # Query the service
+        all_features = []
+        offset = 0
+        query_url = f"{self.service_url}/query"
+
+        while True:
+            params = {
+                "f": "geojson",
+                "where": "1=1",
+                "geometry": f"{xmin},{ymin},{xmax},{ymax}",
+                "geometryType": "esriGeometryEnvelope",
+                "inSR": "4326",
+                "spatialRel": "esriSpatialRelIntersects",
+                "outFields": "*",
+                "returnGeometry": "true",
+                "outSR": "4326",
+                "resultOffset": offset,
+                "resultRecordCount": batch_size,
+            }
+
+            try:
+                response = requests.get(query_url, params=params, timeout=timeout)
+
+                if response.status_code != 200:
+                    if verbose:
+                        print(f"Error {response.status_code}")
+                    break
+
+                data = response.json()
+
+                if "error" in data:
+                    if verbose:
+                        print(f"Server error: {data['error'].get('message')}")
+                    break
+
+                if "features" in data and data["features"]:
+                    batch_gdf = gpd.GeoDataFrame.from_features(
+                        data["features"], crs="EPSG:4326"
+                    )
+                    all_features.append(batch_gdf)
+
+                    if verbose:
+                        total = sum(len(gdf) for gdf in all_features)
+
+                    if len(data["features"]) < batch_size:
+                        break
+                    offset += batch_size
+                else:
+                    break
+
+            except requests.exceptions.Timeout:
+                if verbose:
+                    print("  Request timed out, retrying...")
+                continue
+            except Exception as e:
+                if verbose:
+                    print(f"  Error: {e}")
+                break
+
+        if not all_features:
+            if verbose:
+                print("No features found.")
+            return gpd.GeoDataFrame()
+
+        # Combine and process
+        gdf = pd.concat(all_features, ignore_index=True)
+        gdf = gpd.GeoDataFrame(gdf, crs="EPSG:4326")
+
+        # Remove duplicates
+        for id_field in ["OBJECTID", "FID", "ID"]:
+            if id_field in gdf.columns:
+                initial = len(gdf)
+                gdf = gdf.drop_duplicates(subset=[id_field])
+                if verbose and (initial - len(gdf)) > 0:
+                    print(f"Removed {initial - len(gdf)} duplicates")
+                break
+
+        # Clip to exact boundary
+        if verbose:
+            print(f"Clipping {len(gdf)} features to boundary...")
+        gdf = gpd.clip(gdf, self.boundary)
+
+        # Save
+        output_path = output_dir / output_filename
+        gdf.to_file(output_path, driver="GPKG")
+
+        if verbose:
+            print(f"\n{'='*60}")
+            print(f"SUCCESS: Saved {len(gdf)} buildings to:")
+            print(f"  {output_path}")
+            print(f"{'='*60}")
+
+        return gdf

fimeval/BuildingFootprint/evaluationwithBF.py

@@ -10,6 +10,9 @@ import seaborn as sns
 import matplotlib.pyplot as plt
 import matplotlib.gridspec as gridspec
 
+# Import building footprint module
+from .arcgis_API import getBuildingFootprint
+
 
 def Changeintogpkg(input_path, output_dir, layer_name):
     input_path = str(input_path)
@@ -355,38 +358,12 @@ def detect_shapefile(folder):
     return None
 
 
-def ensure_pyspark(version: str | None = "3.5.4") -> None:
-    """Install pyspark at runtime via `uv pip` into this env (no-op if present)."""
-    import importlib, shutil, subprocess, sys, re
-
-    try:
-        import importlib.util
-
-        if importlib.util.find_spec("pyspark"):
-            return
-    except Exception:
-        pass
-    uv = shutil.which("uv")
-    if not uv:
-        raise RuntimeError(
-            "`uv` not found on PATH. Please install uv or add it to PATH."
-        )
-    if version is None:
-        spec = "pyspark"
-    else:
-        v = version.strip()
-        spec = f"pyspark{v}" if re.match(r"^[<>=!~]", v) else f"pyspark=={v}"
-    subprocess.check_call([uv, "pip", "install", "--python", sys.executable, spec])
-
-
 def EvaluationWithBuildingFootprint(
     main_dir,
     method_name,
     output_dir,
-    country=None,
     building_footprint=None,
     shapefile_dir=None,
-    geeprojectID=None,
 ):
     tif_files_main = glob.glob(os.path.join(main_dir, "*.tif"))
     if tif_files_main:
@@ -410,31 +387,23 @@ def EvaluationWithBuildingFootprint(
 
                 building_footprintMS = building_footprint
 
+                # If no building footprint provided, extract using ArcGIS API
                 if building_footprintMS is None:
-                    ensure_pyspark()
-                    from .microsoftBF import BuildingFootprintwithISO
-
                     out_dir = os.path.join(method_path, "BuildingFootprint")
                     if not os.path.exists(out_dir):
                         os.makedirs(out_dir)
                     EX_building_footprint = find_existing_footprint(out_dir)
-                    if not EX_building_footprint:
+                    if EX_building_footprint:
+                        building_footprintMS = EX_building_footprint
+                    else:
                         boundary_dir = shapefile_dir if shapefile_dir else boundary
-
-                        if geeprojectID:
-                            BuildingFootprintwithISO(
-                                country,
-                                boundary_dir,
-                                out_dir,
-                                geeprojectID=geeprojectID,
-                            )
-                        else:
-                            BuildingFootprintwithISO(country, boundary_dir, out_dir)
-                        building_footprintMS = os.path.join(
-                            out_dir, f"building_footprint.gpkg"
+                        getBuildingFootprint(
+                            boundary=boundary_dir,
+                            output_dir=out_dir,
                         )
-                    else:
-                        building_footprintMS = EX_building_footprint
+                        # After downloading, find the newly created footprint
+                        building_footprintMS = find_existing_footprint(out_dir)
+
                 process_TIFF(
                     tif_files,
                     contingency_files,
@@ -471,33 +440,22 @@ def EvaluationWithBuildingFootprint(
                         building_footprintMS = building_footprint
 
                         if building_footprintMS is None:
-                            ensure_pyspark()
-                            from .microsoftBF import BuildingFootprintwithISO
-
                             out_dir = os.path.join(method_path, "BuildingFootprint")
                             if not os.path.exists(out_dir):
                                 os.makedirs(out_dir)
                             EX_building_footprint = find_existing_footprint(out_dir)
-                            if not EX_building_footprint:
+                            if EX_building_footprint:
+                                building_footprintMS = EX_building_footprint
+                            else:
                                 boundary_dir = (
                                     shapefile_dir if shapefile_dir else boundary
                                 )
-                                if geeprojectID:
-                                    BuildingFootprintwithISO(
-                                        country,
-                                        boundary_dir,
-                                        out_dir,
-                                        geeprojectID=geeprojectID,
-                                    )
-                                else:
-                                    BuildingFootprintwithISO(
-                                        country, boundary_dir, out_dir
-                                    )
-                                building_footprintMS = os.path.join(
-                                    out_dir, f"building_footprint.gpkg"
+                                getBuildingFootprint(
+                                    boundary=boundary_dir,
+                                    output_dir=out_dir,
                                 )
-                            else:
-                                building_footprintMS = EX_building_footprint
+                                # After downloading, find the newly created footprint
+                                building_footprintMS = find_existing_footprint(out_dir)
 
                         process_TIFF(
                             tif_files,

fimeval/ContingencyMap/__init__.py

@@ -1,6 +1,6 @@
 from .evaluationFIM import EvaluateFIM
 from .printcontingency import PrintContingencyMap
 from .plotevaluationmetrics import PlotEvaluationMetrics
-from .PWBs3 import get_PWB
+from .water_bodies import get_PWB, ExtractPWB
 
-__all__ = ["EvaluateFIM", "PrintContingencyMap", "PlotEvaluationMetrics", "get_PWB"]
+__all__ = ["EvaluateFIM", "PrintContingencyMap", "PlotEvaluationMetrics", "get_PWB", "ExtractPWB"]