giga-spatial 0.6.4__py3-none-any.whl → 0.6.5__py3-none-any.whl

gigaspatial/processing/algorithms.py

@@ -0,0 +1,188 @@
+import sys, os
+
+import numpy as np
+from typing import Literal, List, Tuple, Union, Optional
+import geopandas as gpd
+import pandas as pd
+from scipy.spatial import cKDTree
+import networkx as nx
+
+from gigaspatial.processing.geo import (
+    convert_to_geodataframe,
+)
+from gigaspatial.config import config
+
+LOGGER = config.get_logger("GigaSpatialProcessing")
+
+
+def build_distance_graph(
+    left_df: Union[pd.DataFrame, gpd.GeoDataFrame],
+    right_df: Union[pd.DataFrame, gpd.GeoDataFrame],
+    distance_threshold: float,
+    max_k: int = 100,
+    return_dataframe: bool = False,
+    verbose: bool = True,
+    exclude_same_index: Optional[bool] = None,
+) -> Union[nx.Graph, Tuple[nx.Graph, pd.DataFrame]]:
+    """
+    Build a graph of spatial matches between two dataframes using KD-tree.
+
+    Args:
+        left_df: Left dataframe to match from
+        right_df: Right dataframe to match to
+        distance_threshold: Maximum distance for matching (in meters)
+        max_k: Maximum number of neighbors to consider per point (default: 100)
+        return_dataframe: If True, also return the matches DataFrame
+        verbose: If True, print statistics about the graph
+        exclude_same_index: If True, exclude self-matches. If None, auto-detect based on df equality
+
+    Returns:
+        NetworkX Graph, or tuple of (Graph, DataFrame) if return_dataframe=True
+
+    Raises:
+        ValueError: If distance_threshold is negative or max_k is not positive
+    """
+
+    # Input validation
+    if distance_threshold < 0:
+        raise ValueError("distance_threshold must be non-negative")
+
+    if max_k <= 0:
+        raise ValueError("max_k must be positive")
+
+    if left_df.empty or right_df.empty:
+        if verbose:
+            LOGGER.warning("Warning: One or both dataframes are empty")
+        G = nx.Graph()
+        return (G, pd.DataFrame()) if return_dataframe else G
+
+    def get_utm_coordinates(df: Union[pd.DataFrame, gpd.GeoDataFrame]) -> np.ndarray:
+        """Extract coordinates as numpy array in UTM projection."""
+        if isinstance(df, pd.DataFrame):
+            gdf = convert_to_geodataframe(df)
+        else:
+            gdf = df.copy()
+
+        # More robust UTM CRS estimation
+        try:
+            gdf_utm = gdf.to_crs(gdf.estimate_utm_crs())
+        except Exception as e:
+            if verbose:
+                LOGGER.warning(
+                    f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+                )
+            gdf_utm = gdf.to_crs("EPSG:3857")  # Fallback to Web Mercator
+
+        return gdf_utm.get_coordinates().to_numpy()
+
+    # Auto-detect same dataframe case
+    if exclude_same_index is None:
+        exclude_same_index = left_df.equals(right_df)
+        if verbose and exclude_same_index:
+            LOGGER.info("Auto-detected same dataframe - excluding self-matches")
+
+    # Get coordinates
+    left_coords = get_utm_coordinates(left_df)
+    right_coords = (
+        get_utm_coordinates(right_df) if not exclude_same_index else left_coords
+    )
+
+    # Build KD-tree and query
+    kdtree = cKDTree(right_coords)
+
+    # Use the provided max_k parameter, but don't exceed available points
+    k_to_use = min(max_k, len(right_coords))
+
+    if verbose and k_to_use < max_k:
+        LOGGER.info(
+            f"Note: max_k ({max_k}) reduced to {k_to_use} (number of available points)"
+        )
+
+    # Note: Distance calculations here are based on Euclidean distance in UTM projection.
+    # This can introduce errors up to ~50 cm for a 50 meter threshold, especially near the poles where distortion increases.
+    distances, indices = kdtree.query(
+        left_coords, k=k_to_use, distance_upper_bound=distance_threshold
+    )
+
+    # Handle single k case (when k_to_use = 1, results are 1D)
+    if distances.ndim == 1:
+        distances = distances.reshape(-1, 1)
+        indices = indices.reshape(-1, 1)
+
+    # Extract valid pairs using vectorized operations
+    left_indices = np.arange(len(distances))[:, np.newaxis]
+    left_indices = np.broadcast_to(left_indices, distances.shape)
+    valid_mask = np.isfinite(distances)
+
+    if exclude_same_index:
+        same_index_mask = left_indices == indices
+        valid_mask = valid_mask & ~same_index_mask
+
+    valid_left = left_indices[valid_mask]
+    valid_right = indices[valid_mask]
+    valid_distances = distances[valid_mask]
+
+    # Map back to original indices
+    valid_left_indices = left_df.index.values[valid_left]
+    valid_right_indices = right_df.index.values[valid_right]
+
+    # Create matches DataFrame
+    matches_df = pd.DataFrame(
+        {
+            "left_idx": valid_left_indices,
+            "right_idx": valid_right_indices,
+            "distance": valid_distances,
+        }
+    )
+
+    # Build graph more efficiently
+    G = nx.from_pandas_edgelist(
+        matches_df,
+        source="left_idx",
+        target="right_idx",
+        edge_attr="distance",
+        create_using=nx.Graph(),
+    )
+
+    # Add isolated nodes (nodes without any matches within threshold)
+    # This ensures all original indices are represented in the graph
+    all_left_nodes = set(left_df.index.values)
+    all_right_nodes = set(right_df.index.values)
+
+    if not exclude_same_index:
+        all_nodes = all_left_nodes | all_right_nodes
+    else:
+        all_nodes = all_left_nodes  # Same dataframe, so same node set
+
+    # Add nodes that don't have edges
+    existing_nodes = set(G.nodes())
+    isolated_nodes = all_nodes - existing_nodes
+    G.add_nodes_from(isolated_nodes)
+
+    # Print statistics
+    if verbose:
+        print(
+            f"Total potential matches: {len(left_df)} × {len(right_df)} = {len(left_df) * len(right_df):,}"
+        )
+        print(f"Matches found within {distance_threshold}m: {len(matches_df):,}")
+        print(f"Graph nodes: {G.number_of_nodes():,}")
+        print(f"Graph edges: {G.number_of_edges():,}")
+
+        components = list(nx.connected_components(G))
+        print(f"Connected components: {len(components):,}")
+
+        if len(components) > 1:
+            component_sizes = [len(c) for c in components]
+            print(f"Largest component size: {max(component_sizes):,}")
+            print(
+                f"Isolated nodes: {sum(1 for size in component_sizes if size == 1):,}"
+            )
+
+        if len(matches_df) > 0:
+            print(
+                f"Distance stats - min: {matches_df['distance'].min():.1f}m, "
+                f"max: {matches_df['distance'].max():.1f}m, "
+                f"mean: {matches_df['distance'].mean():.1f}m"
+            )
+
+    return (G, matches_df) if return_dataframe else G

gigaspatial/processing/geo.py

@@ -272,8 +272,13 @@ def buffer_geodataframe(
     input_crs = gdf_work.crs
 
     try:
-        # Create a custom UTM CRS based on the calculated UTM zone
-        utm_crs = gdf_work.estimate_utm_crs()
+        try:
+            utm_crs = gdf_work.estimate_utm_crs()
+        except Exception as e:
+            LOGGER.warning(
+                f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+            )
+            utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
         # Transform to UTM, create buffer, and transform back
         gdf_work = gdf_work.to_crs(utm_crs)
@@ -452,7 +457,13 @@ def add_area_in_meters(
     gdf_with_area = gdf.copy()
 
     # Calculate the UTM CRS for accurate area calculation
-    utm_crs = gdf_with_area.estimate_utm_crs()
+    try:
+        utm_crs = gdf_with_area.estimate_utm_crs()
+    except Exception as e:
+        LOGGER.warning(
+            f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+        )
+        utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
     # Transform to UTM CRS and calculate the area in square meters
     gdf_with_area[area_column_name] = gdf_with_area.to_crs(utm_crs).geometry.area
@@ -858,39 +869,79 @@ def aggregate_polygons_to_zones(
     zones: gpd.GeoDataFrame,
     value_columns: Union[str, List[str]],
     aggregation: Union[str, Dict[str, str]] = "sum",
-    area_weighted: bool = True,
+    predicate: Literal["intersects", "within", "fractional"] = "intersects",
     zone_id_column: str = "zone_id",
     output_suffix: str = "",
     drop_geometry: bool = False,
 ) -> gpd.GeoDataFrame:
     """
-    Aggregate polygon data to zones with area-weighted values.
+    Aggregates polygon data to zones based on a specified spatial relationship.
 
-    This function maps polygon data to zones, weighting values by the
-    fractional area of overlap between polygons and zones.
+    This function performs a spatial join between polygons and zones and then
+    aggregates values from the polygons to their corresponding zones. The aggregation
+    method depends on the `predicate` parameter, which determines the nature of the
+    spatial relationship.
 
     Args:
-        polygons (Union[pd.DataFrame, gpd.GeoDataFrame]): Polygon data to aggregate
-        zones (gpd.GeoDataFrame): Zones to aggregate polygons to
-        value_columns (Union[str, List[str]]): Column(s) containing values to aggregate
-        aggregation (Union[str, Dict[str, str]]): Aggregation method(s) to use:
-            - Single string: Use same method for all columns ("sum", "mean", "max", etc.)
-            - Dict: Map column names to aggregation methods
-        area_weighted (bool): Whether to weight values by fractional area overlap
-            If False, values are not weighted before aggregation
-        zone_id_column (str): Column in zones containing zone identifiers
-        output_suffix (str): Suffix to add to output column names
-        drop_geometry (bool): Whether to drop the geometry column from output
+        polygons (Union[pd.DataFrame, gpd.GeoDataFrame]):
+            Polygon data to aggregate. Must be a GeoDataFrame or convertible to one.
+        zones (gpd.GeoDataFrame):
+            The target zones to which the polygon data will be aggregated.
+        value_columns (Union[str, List[str]]):
+            The column(s) in `polygons` containing the numeric values to aggregate.
+        aggregation (Union[str, Dict[str, str]], optional):
+            The aggregation method(s) to use. Can be a single string (e.g., "sum",
+            "mean", "max") to apply the same method to all columns, or a dictionary
+            mapping column names to aggregation methods (e.g., `{'population': 'sum'}`).
+            Defaults to "sum".
+        predicate (Literal["intersects", "within", "fractional"], optional):
+            The spatial relationship to use for aggregation:
+            - "intersects": Aggregates values for any polygon that intersects a zone.
+            - "within": Aggregates values for polygons entirely contained within a zone.
+            - "fractional": Performs area-weighted aggregation. The value of a polygon
+              is distributed proportionally to the area of its overlap with each zone.
+              This requires calculating a UTM CRS for accurate area measurements.
+            Defaults to "intersects".
+        zone_id_column (str, optional):
+            The name of the column in `zones` that contains the unique zone identifiers.
+            Defaults to "zone_id".
+        output_suffix (str, optional):
+            A suffix to add to the names of the new aggregated columns in the output
+            GeoDataFrame. Defaults to "".
+        drop_geometry (bool, optional):
+            If True, the geometry column will be dropped from the output GeoDataFrame.
+            Defaults to False.
 
     Returns:
-        gpd.GeoDataFrame: Zones with aggregated polygon values
+        gpd.GeoDataFrame:
+            The `zones` GeoDataFrame with new columns containing the aggregated values.
+            Zones with no intersecting or contained polygons will have `0` values.
+
+    Raises:
+        TypeError: If `zones` is not a GeoDataFrame or `polygons` cannot be converted.
+        ValueError: If `zone_id_column` or any `value_columns` are not found, or
+                    if the geometry types in `polygons` are not polygons.
+        RuntimeError: If an error occurs during the area-weighted aggregation process.
 
     Example:
-        >>> landuse_stats = aggregate_polygons_to_zones(
+        >>> import geopandas as gpd
+        >>> # Assuming 'landuse_polygons' and 'grid_zones' are GeoDataFrames
+        >>> # Aggregate total population within each grid zone using area-weighting
+        >>> pop_by_zone = aggregate_polygons_to_zones(
+        ...     landuse_polygons,
+        ...     grid_zones,
+        ...     value_columns="population",
+        ...     predicate="fractional",
+        ...     aggregation="sum",
+        ...     output_suffix="_pop"
+        ... )
+        >>> # Aggregate the count of landuse parcels intersecting each zone
+        >>> count_by_zone = aggregate_polygons_to_zones(
         ...     landuse_polygons,
         ...     grid_zones,
-        ...     value_columns=["area", "population"],
-        ...     aggregation="sum"
+        ...     value_columns="parcel_id",
+        ...     predicate="intersects",
+        ...     aggregation="count"
         ... )
     """
     # Input validation
@@ -900,6 +951,11 @@ def aggregate_polygons_to_zones(
     if zone_id_column not in zones.columns:
         raise ValueError(f"Zone ID column '{zone_id_column}' not found in zones")
 
+    if predicate not in ["intersects", "within", "fractional"]:
+        raise ValueError(
+            f"Unsupported predicate: {predicate}. Predicate can be one of `intersects`, `within`, `fractional`"
+        )
+
     # Convert polygons to GeoDataFrame if necessary
     if not isinstance(polygons, gpd.GeoDataFrame):
         try:
@@ -956,11 +1012,17 @@ def aggregate_polygons_to_zones(
     # Create a copy of the zones
     result = zones.copy()
 
-    if area_weighted:
+    if predicate == "fractional":
         # Use area-weighted aggregation with polygon overlay
         try:
             # Compute UTM CRS for accurate area calculations
-            overlay_utm_crs = polygons_gdf.estimate_utm_crs()
+            try:
+                overlay_utm_crs = polygons_gdf.estimate_utm_crs()
+            except Exception as e:
+                LOGGER.warning(
+                    f"Warning: UTM CRS estimation failed, using Web Mercator. Error: {e}"
+                )
+                overlay_utm_crs = "EPSG:3857"  # Fallback to Web Mercator
 
             # Prepare polygons for overlay
             polygons_utm = polygons_gdf.to_crs(overlay_utm_crs)
@@ -1020,7 +1082,7 @@ def aggregate_polygons_to_zones(
     else:
         # Non-weighted aggregation - simpler approach
         # Perform spatial join
-        joined = gpd.sjoin(polygons_gdf, zones, how="inner", predicate="intersects")
+        joined = gpd.sjoin(polygons_gdf, zones, how="inner", predicate=predicate)
 
         # Remove geometry column for aggregation
         if "geometry" in joined.columns: