giga-spatial 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

gigaspatial/processing/algorithms.py

@@ -0,0 +1,188 @@
+import sys, os
+
+import numpy as np
+from typing import Literal, List, Tuple, Union, Optional
+import geopandas as gpd
+import pandas as pd
+from scipy.spatial import cKDTree
+import networkx as nx
+
+from gigaspatial.processing.geo import (
+    convert_to_geodataframe,
+)
+from gigaspatial.config import config
+
+LOGGER = config.get_logger("GigaSpatialProcessing")
+
+
+def build_distance_graph(
+    left_df: Union[pd.DataFrame, gpd.GeoDataFrame],
+    right_df: Union[pd.DataFrame, gpd.GeoDataFrame],
+    distance_threshold: float,
+    max_k: int = 100,
+    return_dataframe: bool = False,
+    verbose: bool = True,
+    exclude_same_index: Optional[bool] = None,
+) -> Union[nx.Graph, Tuple[nx.Graph, pd.DataFrame]]:
+    """
+    Build a graph of spatial matches between two dataframes using KD-tree.
+
+    Args:
+        left_df: Left dataframe to match from
+        right_df: Right dataframe to match to
+        distance_threshold: Maximum distance for matching (in meters)
+        max_k: Maximum number of neighbors to consider per point (default: 100)
+        return_dataframe: If True, also return the matches DataFrame
+        verbose: If True, print statistics about the graph
+        exclude_same_index: If True, exclude self-matches. If None, auto-detect based on df equality
+
+    Returns:
+        NetworkX Graph, or tuple of (Graph, DataFrame) if return_dataframe=True
+
+    Raises:
+        ValueError: If distance_threshold is negative or max_k is not positive
+    """
+
+    # Input validation
+    if distance_threshold < 0:
+        raise ValueError("distance_threshold must be non-negative")
+
+    if max_k <= 0:
+        raise ValueError("max_k must be positive")
+
+    if left_df.empty or right_df.empty:
+        if verbose:
+            LOGGER.warning("Warning: One or both dataframes are empty")
+        G = nx.Graph()
+        return (G, pd.DataFrame()) if return_dataframe else G
+
+    def get_utm_coordinates(df: Union[pd.DataFrame, gpd.GeoDataFrame]) -> np.ndarray:
+        """Extract coordinates as numpy array in UTM projection."""
+        if isinstance(df, pd.DataFrame):
+            gdf = convert_to_geodataframe(df)
+        else:
+            gdf = df.copy()
+
+        # More robust UTM CRS estimation
+        try:
+            gdf_utm = gdf.to_crs(gdf.estimate_utm_crs())
+        except Exception as e:
+            if verbose:
+                LOGGER.warning(
+                    f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+                )
+            gdf_utm = gdf.to_crs("EPSG:3857")  # Fallback to Web Mercator
+
+        return gdf_utm.get_coordinates().to_numpy()
+
+    # Auto-detect same dataframe case
+    if exclude_same_index is None:
+        exclude_same_index = left_df.equals(right_df)
+        if verbose and exclude_same_index:
+            LOGGER.info("Auto-detected same dataframe - excluding self-matches")
+
+    # Get coordinates
+    left_coords = get_utm_coordinates(left_df)
+    right_coords = (
+        get_utm_coordinates(right_df) if not exclude_same_index else left_coords
+    )
+
+    # Build KD-tree and query
+    kdtree = cKDTree(right_coords)
+
+    # Use the provided max_k parameter, but don't exceed available points
+    k_to_use = min(max_k, len(right_coords))
+
+    if verbose and k_to_use < max_k:
+        LOGGER.info(
+            f"Note: max_k ({max_k}) reduced to {k_to_use} (number of available points)"
+        )
+
+    # Note: Distance calculations here are based on Euclidean distance in UTM projection.
+    # This can introduce errors up to ~50 cm for a 50 meter threshold, especially near the poles where distortion increases.
+    distances, indices = kdtree.query(
+        left_coords, k=k_to_use, distance_upper_bound=distance_threshold
+    )
+
+    # Handle single k case (when k_to_use = 1, results are 1D)
+    if distances.ndim == 1:
+        distances = distances.reshape(-1, 1)
+        indices = indices.reshape(-1, 1)
+
+    # Extract valid pairs using vectorized operations
+    left_indices = np.arange(len(distances))[:, np.newaxis]
+    left_indices = np.broadcast_to(left_indices, distances.shape)
+    valid_mask = np.isfinite(distances)
+
+    if exclude_same_index:
+        same_index_mask = left_indices == indices
+        valid_mask = valid_mask & ~same_index_mask
+
+    valid_left = left_indices[valid_mask]
+    valid_right = indices[valid_mask]
+    valid_distances = distances[valid_mask]
+
+    # Map back to original indices
+    valid_left_indices = left_df.index.values[valid_left]
+    valid_right_indices = right_df.index.values[valid_right]
+
+    # Create matches DataFrame
+    matches_df = pd.DataFrame(
+        {
+            "left_idx": valid_left_indices,
+            "right_idx": valid_right_indices,
+            "distance": valid_distances,
+        }
+    )
+
+    # Build graph more efficiently
+    G = nx.from_pandas_edgelist(
+        matches_df,
+        source="left_idx",
+        target="right_idx",
+        edge_attr="distance",
+        create_using=nx.Graph(),
+    )
+
+    # Add isolated nodes (nodes without any matches within threshold)
+    # This ensures all original indices are represented in the graph
+    all_left_nodes = set(left_df.index.values)
+    all_right_nodes = set(right_df.index.values)
+
+    if not exclude_same_index:
+        all_nodes = all_left_nodes | all_right_nodes
+    else:
+        all_nodes = all_left_nodes  # Same dataframe, so same node set
+
+    # Add nodes that don't have edges
+    existing_nodes = set(G.nodes())
+    isolated_nodes = all_nodes - existing_nodes
+    G.add_nodes_from(isolated_nodes)
+
+    # Print statistics
+    if verbose:
+        print(
+            f"Total potential matches: {len(left_df)} × {len(right_df)} = {len(left_df) * len(right_df):,}"
+        )
+        print(f"Matches found within {distance_threshold}m: {len(matches_df):,}")
+        print(f"Graph nodes: {G.number_of_nodes():,}")
+        print(f"Graph edges: {G.number_of_edges():,}")
+
+        components = list(nx.connected_components(G))
+        print(f"Connected components: {len(components):,}")
+
+        if len(components) > 1:
+            component_sizes = [len(c) for c in components]
+            print(f"Largest component size: {max(component_sizes):,}")
+            print(
+                f"Isolated nodes: {sum(1 for size in component_sizes if size == 1):,}"
+            )
+
+        if len(matches_df) > 0:
+            print(
+                f"Distance stats - min: {matches_df['distance'].min():.1f}m, "
+                f"max: {matches_df['distance'].max():.1f}m, "
+                f"mean: {matches_df['distance'].mean():.1f}m"
+            )
+
+    return (G, matches_df) if return_dataframe else G

gigaspatial/processing/geo.py

@@ -272,8 +272,13 @@ def buffer_geodataframe(
     input_crs = gdf_work.crs
 
     try:
-        # Create a custom UTM CRS based on the calculated UTM zone
-        utm_crs = gdf_work.estimate_utm_crs()
+        try:
+            utm_crs = gdf_work.estimate_utm_crs()
+        except Exception as e:
+            LOGGER.warning(
+                f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+            )
+            utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
         # Transform to UTM, create buffer, and transform back
         gdf_work = gdf_work.to_crs(utm_crs)
@@ -452,7 +457,13 @@ def add_area_in_meters(
     gdf_with_area = gdf.copy()
 
     # Calculate the UTM CRS for accurate area calculation
-    utm_crs = gdf_with_area.estimate_utm_crs()
+    try:
+        utm_crs = gdf_with_area.estimate_utm_crs()
+    except Exception as e:
+        LOGGER.warning(
+            f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+        )
+        utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
     # Transform to UTM CRS and calculate the area in square meters
     gdf_with_area[area_column_name] = gdf_with_area.to_crs(utm_crs).geometry.area
@@ -858,57 +869,111 @@ def aggregate_polygons_to_zones(
     zones: gpd.GeoDataFrame,
     value_columns: Union[str, List[str]],
     aggregation: Union[str, Dict[str, str]] = "sum",
-    area_weighted: bool = True,
+    predicate: Literal["intersects", "within", "fractional"] = "intersects",
     zone_id_column: str = "zone_id",
     output_suffix: str = "",
     drop_geometry: bool = False,
 ) -> gpd.GeoDataFrame:
     """
-    Aggregate polygon data to zones with area-weighted values.
+    Aggregates polygon data to zones based on a specified spatial relationship.
 
-    This function maps polygon data to zones, weighting values by the
-    fractional area of overlap between polygons and zones.
+    This function performs a spatial join between polygons and zones and then
+    aggregates values from the polygons to their corresponding zones. The aggregation
+    method depends on the `predicate` parameter, which determines the nature of the
+    spatial relationship.
 
     Args:
-        polygons (Union[pd.DataFrame, gpd.GeoDataFrame]): Polygon data to aggregate
-        zones (gpd.GeoDataFrame): Zones to aggregate polygons to
-        value_columns (Union[str, List[str]]): Column(s) containing values to aggregate
-        aggregation (Union[str, Dict[str, str]]): Aggregation method(s) to use:
-            - Single string: Use same method for all columns ("sum", "mean", "max", etc.)
-            - Dict: Map column names to aggregation methods
-        area_weighted (bool): Whether to weight values by fractional area overlap
-            If False, values are not weighted before aggregation
-        zone_id_column (str): Column in zones containing zone identifiers
-        output_suffix (str): Suffix to add to output column names
-        drop_geometry (bool): Whether to drop the geometry column from output
+        polygons (Union[pd.DataFrame, gpd.GeoDataFrame]):
+            Polygon data to aggregate. Must be a GeoDataFrame or convertible to one.
+        zones (gpd.GeoDataFrame):
+            The target zones to which the polygon data will be aggregated.
+        value_columns (Union[str, List[str]]):
+            The column(s) in `polygons` containing the numeric values to aggregate.
+        aggregation (Union[str, Dict[str, str]], optional):
+            The aggregation method(s) to use. Can be a single string (e.g., "sum",
+            "mean", "max") to apply the same method to all columns, or a dictionary
+            mapping column names to aggregation methods (e.g., `{'population': 'sum'}`).
+            Defaults to "sum".
+        predicate (Literal["intersects", "within", "fractional"], optional):
+            The spatial relationship to use for aggregation:
+            - "intersects": Aggregates values for any polygon that intersects a zone.
+            - "within": Aggregates values for polygons entirely contained within a zone.
+            - "fractional": Performs area-weighted aggregation. The value of a polygon
+              is distributed proportionally to the area of its overlap with each zone.
+              This requires calculating a UTM CRS for accurate area measurements.
+            Defaults to "intersects".
+        zone_id_column (str, optional):
+            The name of the column in `zones` that contains the unique zone identifiers.
+            Defaults to "zone_id".
+        output_suffix (str, optional):
+            A suffix to add to the names of the new aggregated columns in the output
+            GeoDataFrame. Defaults to "".
+        drop_geometry (bool, optional):
+            If True, the geometry column will be dropped from the output GeoDataFrame.
+            Defaults to False.
 
     Returns:
-        gpd.GeoDataFrame: Zones with aggregated polygon values
+        gpd.GeoDataFrame:
+            The `zones` GeoDataFrame with new columns containing the aggregated values.
+            Zones with no intersecting or contained polygons will have `0` values.
+
+    Raises:
+        TypeError: If `zones` is not a GeoDataFrame or `polygons` cannot be converted.
+        ValueError: If `zone_id_column` or any `value_columns` are not found, or
+                    if the geometry types in `polygons` are not polygons.
+        RuntimeError: If an error occurs during the area-weighted aggregation process.
 
     Example:
-        >>> landuse_stats = aggregate_polygons_to_zones(
+        >>> import geopandas as gpd
+        >>> # Assuming 'landuse_polygons' and 'grid_zones' are GeoDataFrames
+        >>> # Aggregate total population within each grid zone using area-weighting
+        >>> pop_by_zone = aggregate_polygons_to_zones(
         ...     landuse_polygons,
         ...     grid_zones,
-        ...     value_columns=["area", "population"],
-        ...     aggregation="sum"
+        ...     value_columns="population",
+        ...     predicate="fractional",
+        ...     aggregation="sum",
+        ...     output_suffix="_pop"
+        ... )
+        >>> # Aggregate the count of landuse parcels intersecting each zone
+        >>> count_by_zone = aggregate_polygons_to_zones(
+        ...     landuse_polygons,
+        ...     grid_zones,
+        ...     value_columns="parcel_id",
+        ...     predicate="intersects",
+        ...     aggregation="count"
         ... )
     """
     # Input validation
     if not isinstance(zones, gpd.GeoDataFrame):
         raise TypeError("zones must be a GeoDataFrame")
 
+    if zones.empty:
+        raise ValueError("zones GeoDataFrame is empty")
+
     if zone_id_column not in zones.columns:
         raise ValueError(f"Zone ID column '{zone_id_column}' not found in zones")
 
+    if predicate not in ["intersects", "within", "fractional"]:
+        raise ValueError(
+            f"Unsupported predicate: {predicate}. Predicate can be one of `intersects`, `within`, `fractional`"
+        )
+
     # Convert polygons to GeoDataFrame if necessary
     if not isinstance(polygons, gpd.GeoDataFrame):
         try:
             polygons_gdf = convert_to_geodataframe(polygons)
-        except:
-            raise TypeError("polygons must be a GeoDataFrame or convertible to one")
+        except Exception as e:
+            raise TypeError(
+                f"polygons must be a GeoDataFrame or convertible to one: {e}"
+            )
     else:
         polygons_gdf = polygons.copy()
 
+    if polygons_gdf.empty:
+        LOGGER.warning("Empty polygons GeoDataFrame provided")
+        return zones
+
     # Validate geometry types
     non_polygon_geoms = [
         geom_type
@@ -935,8 +1000,53 @@ def aggregate_polygons_to_zones(
         polygons_gdf = polygons_gdf.to_crs(zones.crs)
 
     # Handle aggregation method
+    agg_funcs = _process_aggregation_methods(aggregation, value_columns)
+
+    # Prepare minimal zones for spatial operations (only zone_id_column and geometry)
+    minimal_zones = zones[[zone_id_column, "geometry"]].copy()
+
+    if predicate == "fractional":
+        aggregated_data = _fractional_aggregation(
+            polygons_gdf, minimal_zones, value_columns, agg_funcs, zone_id_column
+        )
+    else:
+        aggregated_data = _simple_aggregation(
+            polygons_gdf,
+            minimal_zones,
+            value_columns,
+            agg_funcs,
+            zone_id_column,
+            predicate,
+        )
+
+    # Merge aggregated results back to complete zones data
+    result = zones.merge(
+        aggregated_data[[col for col in aggregated_data.columns if col != "geometry"]],
+        on=zone_id_column,
+        how="left",
+    )
+
+    # Fill NaN values with zeros for the newly aggregated columns only
+    aggregated_cols = [col for col in result.columns if col not in zones.columns]
+    for col in aggregated_cols:
+        if pd.api.types.is_numeric_dtype(result[col]):
+            result[col] = result[col].fillna(0)
+
+    # Apply output suffix consistently to result columns only
+    if output_suffix:
+        rename_dict = {col: f"{col}{output_suffix}" for col in aggregated_cols}
+        result = result.rename(columns=rename_dict)
+
+    if drop_geometry:
+        result = result.drop(columns=["geometry"])
+
+    return result
+
+
+def _process_aggregation_methods(aggregation, value_columns):
+    """Process and validate aggregation methods"""
     if isinstance(aggregation, str):
-        agg_funcs = {col: aggregation for col in value_columns}
+        return {col: aggregation for col in value_columns}
     elif isinstance(aggregation, dict):
         # Validate dictionary keys
         missing_aggs = [col for col in value_columns if col not in aggregation]
@@ -949,106 +1059,98 @@ def aggregate_polygons_to_zones(
                 f"Aggregation methods specified for non-existent columns: {extra_aggs}"
             )
 
-        agg_funcs = aggregation
+        return aggregation
     else:
         raise TypeError("aggregation must be a string or dictionary")
 
-    # Create a copy of the zones
-    result = zones.copy()
 
-    if area_weighted:
-        # Use area-weighted aggregation with polygon overlay
+def _fractional_aggregation(
+    polygons_gdf, zones, value_columns, agg_funcs, zone_id_column
+):
+    """Perform area-weighted (fractional) aggregation"""
+    try:
+        # Compute UTM CRS for accurate area calculations
         try:
-            # Compute UTM CRS for accurate area calculations
             overlay_utm_crs = polygons_gdf.estimate_utm_crs()
+        except Exception as e:
+            LOGGER.warning(f"UTM CRS estimation failed, using Web Mercator. Error: {e}")
+            overlay_utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
-            # Prepare polygons for overlay
-            polygons_utm = polygons_gdf.to_crs(overlay_utm_crs)
-            polygons_utm["orig_area"] = polygons_utm.area
+        # Prepare polygons for overlay - only necessary columns
+        polygons_utm = polygons_gdf.to_crs(overlay_utm_crs)
+        polygons_utm["orig_area"] = polygons_utm.area
 
-            # Keep only necessary columns
-            overlay_cols = value_columns + ["geometry", "orig_area"]
-            overlay_gdf = polygons_utm[overlay_cols].copy()
+        # Keep only necessary columns
+        overlay_cols = value_columns + ["geometry", "orig_area"]
+        overlay_gdf = polygons_utm[overlay_cols].copy()
 
-            # Prepare zones for overlay
-            zones_utm = zones.to_crs(overlay_utm_crs)
+        # Prepare zones for overlay
+        zones_utm = zones.to_crs(overlay_utm_crs)
 
-            # Perform the spatial overlay
-            gdf_overlayed = gpd.overlay(
-                overlay_gdf, zones_utm[[zone_id_column, "geometry"]], how="intersection"
-            )
+        # Perform the spatial overlay
+        gdf_overlayed = gpd.overlay(overlay_gdf, zones_utm, how="intersection")
 
-            # Calculate fractional areas
-            gdf_overlayed["intersection_area"] = gdf_overlayed.area
-            gdf_overlayed["area_fraction"] = (
-                gdf_overlayed["intersection_area"] / gdf_overlayed["orig_area"]
-            )
+        if gdf_overlayed.empty:
+            LOGGER.warning("No intersections found during fractional aggregation")
+            return zones
 
-            # Apply area weighting to value columns
-            for col in value_columns:
-                gdf_overlayed[col] = gdf_overlayed[col] * gdf_overlayed["area_fraction"]
+        # Calculate fractional areas
+        gdf_overlayed["intersection_area"] = gdf_overlayed.area
+        gdf_overlayed["area_fraction"] = (
+            gdf_overlayed["intersection_area"] / gdf_overlayed["orig_area"]
+        )
 
-            # Aggregate by zone ID
-            aggregated = gdf_overlayed.groupby(zone_id_column)[value_columns].agg(
-                agg_funcs
-            )
+        # Apply area weighting to value columns
+        for col in value_columns:
+            gdf_overlayed[col] = gdf_overlayed[col] * gdf_overlayed["area_fraction"]
 
-            # Handle column naming for multi-level index
-            if isinstance(aggregated.columns, pd.MultiIndex):
-                aggregated.columns = [
-                    f"{col[0]}_{col[1]}{output_suffix}" for col in aggregated.columns
-                ]
+        # Aggregate by zone ID
+        aggregated = gdf_overlayed.groupby(zone_id_column)[value_columns].agg(agg_funcs)
 
-            # Reset index
-            aggregated = aggregated.reset_index()
+        # Handle column naming for multi-level index
+        aggregated = _handle_multiindex_columns(aggregated)
 
-            # Merge aggregated values back to the zones
-            result = result.merge(aggregated, on=zone_id_column, how="left")
+        # Reset index and merge back to zones
+        aggregated = aggregated.reset_index()
 
-            # Fill NaN values with zeros
-            for col in result.columns:
-                if (
-                    col != zone_id_column
-                    and col != "geometry"
-                    and pd.api.types.is_numeric_dtype(result[col])
-                ):
-                    result[col] = result[col].fillna(0)
+        # Return only the aggregated data (will be merged with full zones later)
+        return aggregated
 
-        except Exception as e:
-            raise RuntimeError(f"Error during area-weighted aggregation: {e}")
+    except Exception as e:
+        raise RuntimeError(f"Error during area-weighted aggregation: {e}")
 
-    else:
-        # Non-weighted aggregation - simpler approach
-        # Perform spatial join
-        joined = gpd.sjoin(polygons_gdf, zones, how="inner", predicate="intersects")
 
-        # Remove geometry column for aggregation
-        if "geometry" in joined.columns:
-            joined = joined.drop(columns=["geometry"])
+def _simple_aggregation(
+    polygons_gdf, zones, value_columns, agg_funcs, zone_id_column, predicate
+):
+    """Perform simple (non-weighted) aggregation"""
+    # Perform spatial join
+    joined = gpd.sjoin(polygons_gdf, zones, how="inner", predicate=predicate)
 
-        # Group by zone ID and aggregate
-        aggregated = joined.groupby(zone_id_column)[value_columns].agg(agg_funcs)
+    if joined.empty:
+        LOGGER.warning(f"No {predicate} relationships found during spatial join")
+        return zones
 
-        # Handle column naming for multi-level index
-        if isinstance(aggregated.columns, pd.MultiIndex):
-            aggregated.columns = [
-                f"{col[0]}_{col[1]}{output_suffix}" for col in aggregated.columns
-            ]
+    # Remove geometry column for aggregation (keep only necessary columns)
+    agg_cols = value_columns + [zone_id_column]
+    joined_subset = joined[agg_cols].copy()
 
-        # Reset index and merge back to zones
-        aggregated = aggregated.reset_index()
-        result = result.merge(aggregated, on=zone_id_column, how="left")
+    # Group by zone ID and aggregate
+    aggregated = joined_subset.groupby(zone_id_column)[value_columns].agg(agg_funcs)
 
-        # Fill NaN values with zeros
-        for col in result.columns:
-            if (
-                col != zone_id_column
-                and col != "geometry"
-                and pd.api.types.is_numeric_dtype(result[col])
-            ):
-                result[col] = result[col].fillna(0)
+    # Handle column naming for multi-level index
+    aggregated = _handle_multiindex_columns(aggregated)
 
-    if drop_geometry:
-        result = result.drop(columns=["geometry"])
+    # Reset index and merge back to zones
+    aggregated = aggregated.reset_index()
 
-    return result
+    # Return only the aggregated data (will be merged with full zones later)
+    return aggregated
+
+
+def _handle_multiindex_columns(aggregated):
+    """Handle multi-level column index from groupby aggregation"""
+    if isinstance(aggregated.columns, pd.MultiIndex):
+        # Flatten multi-level columns: combine column name with aggregation method
+        aggregated.columns = [f"{col[0]}_{col[1]}" for col in aggregated.columns]
+    return aggregated