fimeval 0.1.52__py3-none-any.whl → 0.1.54__py3-none-any.whl

fimeval/BuildingFootprint/evaluationwithBF.py

@@ -1,5 +1,4 @@
 import os
-import ee
 import glob
 import geopandas as gpd
 import rasterio
@@ -31,7 +30,6 @@ def GetFloodedBuildingCountInfo(
     basename,
 ):
     output_dir = os.path.dirname(building_fp_path)
-
     building_fp_gpkg = Changeintogpkg(
         building_fp_path, output_dir, "building_footprint"
     )
@@ -42,7 +40,6 @@ def GetFloodedBuildingCountInfo(
     with rasterio.open(raster1_path) as src:
         target_crs = str(src.crs)
 
-    # Reproject all GeoDataFrames to the target CRS
     if building_gdf.crs != target_crs:
         building_gdf = building_gdf.to_crs(target_crs)
         print("reproject building_gdf")
@@ -55,43 +52,32 @@ def GetFloodedBuildingCountInfo(
     clipped_buildings["centroid"] = clipped_buildings.geometry.centroid
 
     centroid_counts = {
-        "Benchmark": 0,
-        "Candidate": 0,
         "False Positive": 0,
         "False Negative": 0,
         "True Positive": 0,
     }
 
-    def count_centroids_in_raster(raster_path, label):
+    # Count centroids in the contingency map
+    def count_centroids_in_contingency(raster_path):
         with rasterio.open(raster_path) as src:
             raster_data = src.read(1)
-            transform = src.transform
-
             for centroid in clipped_buildings["centroid"]:
                 row, col = src.index(centroid.x, centroid.y)
                 if 0 <= row < raster_data.shape[0] and 0 <= col < raster_data.shape[1]:
                     pixel_value = raster_data[row, col]
-                    if label in ["Benchmark", "Candidate"]:
-                        if pixel_value == 2:  # False Positive
-                            centroid_counts[label] += 1
-                    else:
-                        if pixel_value == 2:
-                            centroid_counts["False Positive"] += 1
-                        elif pixel_value == 3:
-                            centroid_counts["False Negative"] += 1
-                        elif pixel_value == 4:
-                            centroid_counts["True Positive"] += 1
-
-    if "bm" in str(raster1_path).lower():
-        count_centroids_in_raster(raster1_path, "Benchmark")
-        count_centroids_in_raster(raster2_path, "Candidate")
-        
-    elif "candidate" in str(raster2_path).lower():
-        count_centroids_in_raster(raster1_path, "Candidate")
-        count_centroids_in_raster(raster2_path, "Benchmark")
+                    if pixel_value == 2:
+                        centroid_counts["False Positive"] += 1
+                    elif pixel_value == 3:
+                        centroid_counts["False Negative"] += 1
+                    elif pixel_value == 4:
+                        centroid_counts["True Positive"] += 1
+
+    count_centroids_in_contingency(contingency_map)
+
+    # Calculate Candidate and Benchmark counts from the contingency map counts
+    centroid_counts["Candidate"] = centroid_counts["True Positive"] + centroid_counts["False Positive"]
+    centroid_counts["Benchmark"] = centroid_counts["True Positive"] + centroid_counts["False Negative"]
 
-    if "contingency" in str(contingency_map).lower():
-        count_centroids_in_raster(contingency_map, "Contingency")
 
     total_buildings = len(clipped_buildings)
     percentages = {
@@ -106,10 +92,15 @@ def GetFloodedBuildingCountInfo(
     CSI = TP / (TP + FP + FN) if (TP + FP + FN) > 0 else 0
     FAR = FP / (TP + FP) if (TP + FP) > 0 else 0
     POD = TP / (TP + FN) if (TP + FN) > 0 else 0
-    
-    
-    BDR = (centroid_counts["Candidate"]- centroid_counts["Benchmark"])/centroid_counts["Benchmark"]
 
+    if centroid_counts["Benchmark"] > 0:
+        BDR = (
+            (centroid_counts["Candidate"] - centroid_counts["Benchmark"])
+            / centroid_counts["Benchmark"]
+        )
+    else:
+            BDR = 0
+        
     counts_data = {
         "Category": [
             "Candidate",
@@ -223,6 +214,7 @@ def GetFloodedBuildingCountInfo(
     print(f"Performance metrics chart is saved as PNG at {output_path}")
     fig.show()
 
+
 def process_TIFF(
     tif_files, contingency_files, building_footprint, boundary, method_path
 ):
@@ -266,22 +258,49 @@ def process_TIFF(
         print("Warning: No benchmark file found.")
     elif not candidate_paths:
         print("Warning: No candidate files found.")
-        
+
+
 def find_existing_footprint(out_dir):
     gpkg_files = list(Path(out_dir).glob("*.gpkg"))
     return gpkg_files[0] if gpkg_files else None
 
-#Incase user defined individual shapefile for each case study
+
+# Incase user defined individual shapefile for each case study
 def detect_shapefile(folder):
-        shapefile = None
-        for ext in (".shp", ".gpkg", ".geojson", ".kml"):
-            for file in os.listdir(folder):
-                if file.lower().endswith(ext):
-                    shapefile = os.path.join(folder, file)
-                    print(f"Auto-detected shapefile: {shapefile}")
-                    return shapefile
-        return None
-    
+    shapefile = None
+    for ext in (".shp", ".gpkg", ".geojson", ".kml"):
+        for file in os.listdir(folder):
+            if file.lower().endswith(ext):
+                shapefile = os.path.join(folder, file)
+                print(f"Auto-detected shapefile: {shapefile}")
+                return shapefile
+    return None
+
+
+def ensure_pyspark(version: str | None = "3.5.4") -> None:
+    """Install pyspark at runtime via `uv pip` into this env (no-op if present)."""
+    import importlib, shutil, subprocess, sys, re
+
+    try:
+        import importlib.util
+
+        if importlib.util.find_spec("pyspark"):
+            return
+    except Exception:
+        pass
+    uv = shutil.which("uv")
+    if not uv:
+        raise RuntimeError(
+            "`uv` not found on PATH. Please install uv or add it to PATH."
+        )
+    if version is None:
+        spec = "pyspark"
+    else:
+        v = version.strip()
+        spec = f"pyspark{v}" if re.match(r"^[<>=!~]", v) else f"pyspark=={v}"
+    subprocess.check_call([uv, "pip", "install", "--python", sys.executable, spec])
+
+
 def EvaluationWithBuildingFootprint(
     main_dir,
     method_name,
@@ -289,6 +308,7 @@ def EvaluationWithBuildingFootprint(
     country=None,
     building_footprint=None,
     shapefile_dir=None,
+    geeprojectID=None,
 ):
     tif_files_main = glob.glob(os.path.join(main_dir, "*.tif"))
     if tif_files_main:
@@ -303,9 +323,7 @@ def EvaluationWithBuildingFootprint(
 
                 if shapefile_dir:
                     boundary = shapefile_dir
-                elif os.path.exists(
-                    os.path.join(method_path, "BoundaryforEvaluation")
-                ):
+                elif os.path.exists(os.path.join(method_path, "BoundaryforEvaluation")):
                     boundary = os.path.join(
                         method_path, "BoundaryforEvaluation", "FIMEvaluatedExtent.shp"
                     )
@@ -313,8 +331,10 @@ def EvaluationWithBuildingFootprint(
                     boundary = detect_shapefile(main_dir)
 
                 building_footprintMS = building_footprint
+
                 if building_footprintMS is None:
-                    import msfootprint as msf
+                    ensure_pyspark()
+                    from .microsoftBF import BuildingFootprintwithISO
 
                     out_dir = os.path.join(method_path, "BuildingFootprint")
                     if not os.path.exists(out_dir):
@@ -323,7 +343,15 @@ def EvaluationWithBuildingFootprint(
                     if not EX_building_footprint:
                         boundary_dir = shapefile_dir if shapefile_dir else boundary
 
-                        msf.BuildingFootprintwithISO(country, boundary_dir, out_dir)
+                        if geeprojectID:
+                            BuildingFootprintwithISO(
+                                country,
+                                boundary_dir,
+                                out_dir,
+                                geeprojectID=geeprojectID,
+                            )
+                        else:
+                            BuildingFootprintwithISO(country, boundary_dir, out_dir)
                         building_footprintMS = os.path.join(
                             out_dir, f"building_footprint.gpkg"
                         )
@@ -355,14 +383,18 @@ def EvaluationWithBuildingFootprint(
                             os.path.join(method_path, "BoundaryforEvaluation")
                         ):
                             boundary = os.path.join(
-                                method_path, "BoundaryforEvaluation", "FIMEvaluatedExtent.shp"
+                                method_path,
+                                "BoundaryforEvaluation",
+                                "FIMEvaluatedExtent.shp",
                             )
                         else:
                             boundary = detect_shapefile(os.path.join(main_dir, folder))
 
                         building_footprintMS = building_footprint
+
                         if building_footprintMS is None:
-                            import msfootprint as msf
+                            ensure_pyspark()
+                            from .microsoftBF import BuildingFootprintwithISO
 
                             out_dir = os.path.join(method_path, "BuildingFootprint")
                             if not os.path.exists(out_dir):
@@ -372,9 +404,17 @@ def EvaluationWithBuildingFootprint(
                                 boundary_dir = (
                                     shapefile_dir if shapefile_dir else boundary
                                 )
-                                msf.BuildingFootprintwithISO(
-                                    country, boundary_dir, out_dir
-                                )
+                                if geeprojectID:
+                                    BuildingFootprintwithISO(
+                                        country,
+                                        boundary_dir,
+                                        out_dir,
+                                        geeprojectID=geeprojectID,
+                                    )
+                                else:
+                                    BuildingFootprintwithISO(
+                                        country, boundary_dir, out_dir
+                                    )
                                 building_footprintMS = os.path.join(
                                     out_dir, f"building_footprint.gpkg"
                                 )

fimeval/BuildingFootprint/microsoftBF.py

@@ -0,0 +1,132 @@
+# Importing necessary libraries
+import geemap
+import ee
+import os
+from shapely.geometry import box
+import pandas as pd
+from pathlib import Path
+import geopandas as gpd
+from pyspark.sql import SparkSession
+from shapely.wkt import loads
+import shutil
+
+# Suppress the warnings
+import warnings
+
+warnings.filterwarnings("ignore")
+
+# Authenticate and initialize Earth Engine
+ee.Authenticate()
+
+
+# %%
+def split_into_tiles(boundary, tile_size=0.1):
+    bounds = boundary.total_bounds
+    x_min, y_min, x_max, y_max = bounds
+    tiles = []
+    x = x_min
+    while x < x_max:
+        y = y_min
+        while y < y_max:
+            tile = box(x, y, x + tile_size, y + tile_size)
+            if tile.intersects(boundary.unary_union):
+                tiles.append(tile)
+            y += tile_size
+        x += tile_size
+    return gpd.GeoDataFrame(geometry=tiles, crs=boundary.crs)
+
+
+# Merge the final geojson files
+def mergeGeoJSONfiles(output_dir, merged_file):
+    output_dir = Path(output_dir)
+    files = list(output_dir.glob("*.geojson"))
+    gdfs = [gpd.read_file(file) for file in files]
+    merged_gdf = gpd.GeoDataFrame(pd.concat(gdfs, ignore_index=True), crs="EPSG:4326")
+    merged_gdf.to_file(merged_file, driver="GPKG")
+
+
+# Process each batch with number of tiles
+def process_batch(partition, collection_name, output_dir, boundary_wkt, projectID=None):
+    try:
+        if projectID:
+            ee.Initialize(project=projectID)
+        else:
+            ee.Initialize()
+
+    except Exception:
+        print("To initialize, please provide the earth engine project ID")
+
+    # Convert WKT boundary to geometry
+    boundary = loads(boundary_wkt)
+    results = []
+
+    for tile_wkt in partition:
+        try:
+            tile = loads(tile_wkt)
+            aoi = ee.Geometry(tile.__geo_interface__)
+            collection = ee.FeatureCollection(collection_name).filterBounds(aoi)
+
+            # Download features and filter by boundary
+            gdf = geemap.ee_to_gdf(collection)
+            gdf = gdf[gdf.geometry.intersects(boundary)]
+
+            # Save each tile as a GeoJSON file
+            tile_id = f"tile_{hash(tile)}"
+            output_file = Path(output_dir) / f"{tile_id}.geojson"
+            gdf.to_file(output_file, driver="GeoJSON")
+            results.append(f"Saved: {output_file}")
+        except Exception as e:
+            results.append(f"Error processing tile: {e}")
+
+    return results
+
+
+def getBuildingFootprintSpark(
+    countryISO, boundary_file, out_dir, tile_size, projectID=None
+):
+    spark = SparkSession.builder.appName("BuildingFootprints").getOrCreate()
+
+    # Make temporary directory
+    temp_dir = out_dir / "temp"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+
+    # Load and process boundary
+    boundary = gpd.read_file(boundary_file).to_crs("EPSG:4326")
+    tiles = split_into_tiles(boundary, tile_size)
+    boundary_wkt = boundary.unary_union.wkt
+
+    collection_names = [f"projects/sat-io/open-datasets/VIDA_COMBINED/{countryISO}"]
+
+    # Distribute processing
+    for collection_name in collection_names:
+        tiles_rdd = spark.sparkContext.parallelize(
+            tiles.geometry.apply(lambda x: x.wkt).tolist(), numSlices=10
+        )
+        results = tiles_rdd.mapPartitions(
+            lambda partition: process_batch(
+                partition, collection_name, str(temp_dir), boundary_wkt, projectID
+            )
+        ).collect()
+
+    # Merge GeoJSON files
+    mergeGeoJSONfiles(temp_dir, out_dir / "building_footprint.gpkg")
+
+    # Clean up the temp directory
+    shutil.rmtree(temp_dir, ignore_errors=True)
+
+    print(f"Building footprint data saved to {out_dir / 'building_footprint.gpkg'}")
+
+
+# %%
+# Export the building footprint
+def BuildingFootprintwithISO(countryISO, ROI, out_dir, geeprojectID=None):
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    filename = out_dir / "building_footprint.gpkg"
+
+    if filename.exists():
+        os.remove(filename)
+
+    getBuildingFootprintSpark(
+        countryISO, ROI, out_dir, tile_size=0.05, projectID=geeprojectID
+    )

fimeval/ContingencyMap/PWBs3.py

@@ -1,4 +1,4 @@
-#import Libraries
+# import Libraries
 import geopandas as gpd
 import boto3
 import botocore
@@ -7,27 +7,27 @@ import tempfile
 
 # Initialize an anonymous S3 client
 s3 = boto3.client(
-    's3',
-    config=botocore.config.Config(signature_version=botocore.UNSIGNED)
+    "s3", config=botocore.config.Config(signature_version=botocore.UNSIGNED)
 )
 
-bucket_name = 'sdmlab'
-pwb_folder = "PWB/"  
+bucket_name = "sdmlab"
+pwb_folder = "PWB/"
+
 
 def PWB_inS3(s3_client, bucket, prefix):
     """Download all components of a shapefile from S3 into a temporary directory."""
     tmp_dir = tempfile.mkdtemp()
     response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
-    if 'Contents' not in response:
+    if "Contents" not in response:
         raise ValueError("No files found in the specified S3 folder.")
-    
-    for obj in response['Contents']:
-        file_key = obj['Key']
-        file_name = os.path.basename(file_key) 
-        if file_name.endswith(('.shp', '.shx', '.dbf', '.prj', '.cpg')):
+
+    for obj in response["Contents"]:
+        file_key = obj["Key"]
+        file_name = os.path.basename(file_key)
+        if file_name.endswith((".shp", ".shx", ".dbf", ".prj", ".cpg")):
             local_path = os.path.join(tmp_dir, file_name)
             s3_client.download_file(bucket, file_key, local_path)
-            
+
     shp_files = [f for f in os.listdir(tmp_dir) if f.endswith(".shp")]
     if not shp_files:
         raise ValueError("No .shp file found after download.")
@@ -35,7 +35,8 @@ def PWB_inS3(s3_client, bucket, prefix):
     shp_path = os.path.join(tmp_dir, shp_files[0])
     return shp_path
 
+
 def get_PWB():
     shp_path = PWB_inS3(s3, bucket_name, pwb_folder)
     pwb = gpd.read_file(shp_path)
-    return pwb
+    return pwb

fimeval/ContingencyMap/evaluationFIM.py

@@ -21,28 +21,45 @@ from .metrics import evaluationmetrics
 from .PWBs3 import get_PWB
 from ..utilis import MakeFIMsUniform
 
-#giving the permission to the folder
+
+# giving the permission to the folder
 def is_writable(path):
     """Check if the directory and its contents are writable."""
     path = Path(path)
     return os.access(path, os.W_OK)
 
+
 def fix_permissions(path):
     path = Path(path).resolve()
-    script_path = Path(__file__).parent / "fix_permissions.sh"
-
-    if not script_path.exists():
-        raise FileNotFoundError(f"Shell script not found: {script_path}")
 
     if is_writable(path):
-        return 
+        return
+
+    uname = platform.system()
 
     try:
-        result = subprocess.run(["bash", str(script_path), str(path)],
-                                check=True, capture_output=True, text=True)
-        print(result.stdout)
+        if uname in ["Darwin", "Linux"]:
+            subprocess.run(
+                ["chmod", "-R", "u+rwX", str(path)],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            print(f"Permissions granted for user (u+rwX): {path}")
+
+        elif "MINGW" in uname or "MSYS" in uname or "CYGWIN" in uname:
+            subprocess.run(
+                ["icacls", str(path), "/grant", "Everyone:F", "/T"],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            print(f"Permissions granted for working folder: {path}")
+
+        else:
+            print(f"Unsupported OS: {uname}")
     except subprocess.CalledProcessError as e:
-        print(f"Shell script failed:\n{e.stderr}")
+        print(f"Failed to fix permissions for {path}:\n{e.stderr}")
 
 
 # Function for the evalution of the model
@@ -79,7 +96,7 @@ def evaluateFIM(
     # Get the smallest matched raster extent and make a boundary shapefile
     smallest_raster_path = get_smallest_raster_path(benchmark_path, *candidate_paths)
 
-    #If method is AOI, and direct shapefile directory is not provided, then it will search for the shapefile in the folder
+    # If method is AOI, and direct shapefile directory is not provided, then it will search for the shapefile in the folder
     if method.__name__ == "AOI":
         # If shapefile is not provided, search in the folder
         if shapefile is None:
@@ -358,7 +375,8 @@ def evaluateFIM(
     print(f"Evaluation metrics saved to {csv_file}")
     return results
 
-#Safely deleting the folder
+
+# Safely deleting the folder
 def safe_delete_folder(folder_path):
     fix_permissions(folder_path)
     try:
@@ -370,19 +388,28 @@ def safe_delete_folder(folder_path):
     except Exception as e:
         print(f"Error deleting {folder_path}: {e}")
 
-def EvaluateFIM(main_dir, method_name, output_dir, PWB_dir=None, shapefile_dir=None, target_crs=None, target_resolution=None):
+
+def EvaluateFIM(
+    main_dir,
+    method_name,
+    output_dir,
+    PWB_dir=None,
+    shapefile_dir=None,
+    target_crs=None,
+    target_resolution=None,
+):
     main_dir = Path(main_dir)
     # Read the permanent water bodies
     if PWB_dir is None:
         gdf = get_PWB()
     else:
         gdf = gpd.read_file(PWB_dir)
-    
-    #Grant the permission to the main directory
+
+    # Grant the permission to the main directory
     print(f"Fixing permissions for {main_dir}...")
     fix_permissions(main_dir)
 
-    #runt the process
+    # runt the process
     def process_TIFF(tif_files, folder_dir):
         benchmark_path = None
         candidate_path = []
@@ -422,7 +449,9 @@ def EvaluateFIM(main_dir, method_name, output_dir, PWB_dir=None, shapefile_dir=N
     # Check if main_dir directly contains tif files
     TIFFfiles_main_dir = list(main_dir.glob("*.tif"))
     if TIFFfiles_main_dir:
-        MakeFIMsUniform(main_dir, target_crs=target_crs, target_resolution=target_resolution)
+        MakeFIMsUniform(
+            main_dir, target_crs=target_crs, target_resolution=target_resolution
+        )
 
         # processing folder
         processing_folder = main_dir / "processing"
@@ -434,15 +463,20 @@ def EvaluateFIM(main_dir, method_name, output_dir, PWB_dir=None, shapefile_dir=N
         for folder in main_dir.iterdir():
             if folder.is_dir():
                 tif_files = list(folder.glob("*.tif"))
-                
+
                 if tif_files:
-                    MakeFIMsUniform(folder, target_crs=target_crs, target_resolution=target_resolution)
-                    
+                    MakeFIMsUniform(
+                        folder,
+                        target_crs=target_crs,
+                        target_resolution=target_resolution,
+                    )
+
                     processing_folder = folder / "processing"
                     TIFFfiles = list(processing_folder.glob("*.tif"))
 
                     process_TIFF(TIFFfiles, folder)
                     safe_delete_folder(processing_folder)
                 else:
-                    print(f"Skipping {folder.name} as it doesn't contain any tif files.")
-
+                    print(
+                        f"Skipping {folder.name} as it doesn't contain any tif files."
+                    )

fimeval/ContingencyMap/metrics.py

@@ -7,21 +7,21 @@ def evaluationmetrics(out_image1, out_image2):
     unique_values, counts = np.unique(merged, return_counts=True)
     class_pixel_counts = dict(zip(unique_values, counts))
     class_pixel_counts
-    TN = class_pixel_counts.get(1,0)
-    FP = class_pixel_counts.get(2,0)
-    FN = class_pixel_counts.get(3,0)
-    TP = class_pixel_counts.get(4,0)
+    TN = class_pixel_counts.get(1, 0)
+    FP = class_pixel_counts.get(2, 0)
+    FN = class_pixel_counts.get(3, 0)
+    TP = class_pixel_counts.get(4, 0)
     epsilon = 1e-8
-    TPR = TP / (TP + FN+epsilon)
-    FNR = FN / (TP + FN+epsilon)
-    Acc = (TP + TN) / (TP + TN + FP + FN+epsilon)
-    Prec = TP / (TP + FP+epsilon)
-    sen = TP / (TP + FN+epsilon)
-    F1_score = 2 * (Prec * sen) / (Prec + sen+epsilon)
-    CSI = TP / (TP + FN + FP+epsilon)
-    POD = TP / (TP + FN+epsilon)
-    FPR = FP / (FP + TN+epsilon)
-    FAR = FP / (TP + FP+epsilon)
+    TPR = TP / (TP + FN + epsilon)
+    FNR = FN / (TP + FN + epsilon)
+    Acc = (TP + TN) / (TP + TN + FP + FN + epsilon)
+    Prec = TP / (TP + FP + epsilon)
+    sen = TP / (TP + FN + epsilon)
+    F1_score = 2 * (Prec * sen) / (Prec + sen + epsilon)
+    CSI = TP / (TP + FN + FP + epsilon)
+    POD = TP / (TP + FN + epsilon)
+    FPR = FP / (FP + TN + epsilon)
+    FAR = FP / (TP + FP + epsilon)
 
     return (
         unique_values,

fimeval/ContingencyMap/printcontingency.py

@@ -65,7 +65,7 @@ def getContingencyMap(raster_path, method_path):
         2: "False positive",
         3: "False negative",
         4: "True positive",
-        5: "Permanent water bodies"
+        5: "Permanent water bodies",
     }
     legend_patches = [
         Patch(

fimeval/__init__.py

@@ -1,11 +1,20 @@
-#Evaluation modules
+# Evaluation modules
 from .ContingencyMap.evaluationFIM import EvaluateFIM
 from .ContingencyMap.printcontingency import PrintContingencyMap
 from .ContingencyMap.plotevaluationmetrics import PlotEvaluationMetrics
 from .ContingencyMap.PWBs3 import get_PWB
 
-#Utility modules
+# Utility modules
 from .utilis import compress_tif_lzw
 
 # Evaluation with Building foorprint module
 from .BuildingFootprint.evaluationwithBF import EvaluationWithBuildingFootprint
+
+__all__ = [
+    "EvaluateFIM",
+    "PrintContingencyMap",
+    "PlotEvaluationMetrics",
+    "get_PWB",
+    "EvaluationWithBuildingFootprint",
+    "compress_tif_lzw",
+]