ssb-sgis 1.1.16__py3-none-any.whl → 1.2.0__py3-none-any.whl

sgis/__init__.py

@@ -76,6 +76,10 @@ from .geopandas_tools.polygon_operations import get_holes
 from .geopandas_tools.polygon_operations import get_polygon_clusters
 from .geopandas_tools.polygon_operations import split_polygons_by_lines
 from .geopandas_tools.polygons_as_rings import PolygonsAsRings
+from .geopandas_tools.runners import GridSizeOverlayRunner
+from .geopandas_tools.runners import OverlayRunner
+from .geopandas_tools.runners import RTreeQueryRunner
+from .geopandas_tools.runners import UnionRunner
 from .geopandas_tools.sfilter import sfilter
 from .geopandas_tools.sfilter import sfilter_inverse
 from .geopandas_tools.sfilter import sfilter_split

sgis/conf.py

@@ -1,3 +1,6 @@
+from collections.abc import Iterable
+from typing import Any
+
 try:
     from gcsfs import GCSFileSystem
 
@@ -66,7 +69,56 @@ except ImportError:
 
     file_system = LocalFileSystem
 
-config = {
-    "n_jobs": 1,
-    "file_system": file_system,
-}
+from .geopandas_tools.runners import OverlayRunner
+from .geopandas_tools.runners import RTreeQueryRunner
+from .geopandas_tools.runners import UnionRunner
+
+
+class Config:
+    """Dictlike config with a 'get_instance' method."""
+
+    def __init__(self, data: dict) -> None:
+        """Initialise with dict."""
+        self.data = data
+
+    def get_instance(self, key: str, *args, **kwargs) -> Any:
+        """Get the dict value and call it if callable."""
+        x = self.data[key]
+        if callable(x):
+            return x(*args, **kwargs)
+        return x
+
+    def __getattr__(self, attr: str) -> Any:
+        """Get dict attribute."""
+        return getattr(self.data, attr)
+
+    def __getitem__(self, key: str) -> Any:
+        """Get dict value."""
+        return self.data[key]
+
+    def __setitem__(self, key: str, value) -> None:
+        """Set dict value."""
+        self.data[key] = value
+
+    def __iter__(self) -> Iterable[str]:
+        """Iterate over dict keys."""
+        return iter(self.data)
+
+    def __len__(self) -> int:
+        """Length of dict."""
+        return len(self.data)
+
+    def __str__(self) -> str:
+        """String representation of dict."""
+        return str(self.data)
+
+
+config = Config(
+    {
+        "n_jobs": 1,
+        "file_system": file_system,
+        "rtree_runner": RTreeQueryRunner,
+        "overlay_runner": OverlayRunner,
+        "union_runner": UnionRunner,
+    }
+)

sgis/geopandas_tools/buffer_dissolve_explode.py

@@ -23,12 +23,13 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import get_num_geometries
 
+from ..conf import config
 from ..parallel.parallel import Parallel
-from .general import _parallel_unary_union
-from .general import _unary_union_for_notna
 from .geometry_types import make_all_singlepart
 from .polygon_operations import get_cluster_mapper
 from .polygon_operations import get_grouped_centroids
+from .runners import UnionRunner
+from .utils import _unary_union_for_notna
 
 
 def _decide_ignore_index(kwargs: dict) -> tuple[dict, bool]:
@@ -53,8 +54,8 @@ def buffdissexp(
     index_parts: bool = False,
     copy: bool = True,
     grid_size: float | int | None = None,
-    n_jobs: int = 1,
     join_style: int | str = "round",
+    n_jobs: int = 1,
     **dissolve_kwargs,
 ) -> GeoDataFrame:
     """Buffers and dissolves overlapping geometries.
@@ -187,27 +188,27 @@ def _dissolve(
     gdf: GeoDataFrame,
     aggfunc: str = "first",
     grid_size: None | float = None,
-    n_jobs: int = 1,
     as_index: bool = True,
+    n_jobs: int = 1,
+    union_runner: UnionRunner | None = None,
     **dissolve_kwargs,
 ) -> GeoDataFrame:
-
     if not len(gdf):
         return gdf
 
-    geom_col = gdf._geometry_column_name
+    if union_runner is None:
+        union_runner = config.get_instance("union_runner", n_jobs)
 
-    gdf[geom_col] = gdf[geom_col].make_valid()
+    geom_col = gdf.geometry.name
+    by = dissolve_kwargs.pop("by", None)
+    by_was_none = not bool(by)
 
+    # make sure geometries are dissolved rowwise to make dissolving simpler later
     more_than_one = get_num_geometries(gdf.geometry.values) > 1
     gdf.loc[more_than_one, geom_col] = gdf.loc[more_than_one, geom_col].apply(
         _unary_union_for_notna
     )
 
-    by = dissolve_kwargs.pop("by", None)
-
-    by_was_none = not bool(by)
-
     if by is None and dissolve_kwargs.get("level") is None:
         by = np.zeros(len(gdf), dtype="int64")
         other_cols = list(gdf.columns.difference({geom_col}))
@@ -215,30 +216,23 @@ def _dissolve(
         if isinstance(by, str):
             by = [by]
         other_cols = list(gdf.columns.difference({geom_col} | set(by or {})))
-
     try:
         is_one_hit = (
             gdf.groupby(by, as_index=True, **dissolve_kwargs).transform("size") == 1
         )
     except IndexError:
-        # if no rows when dropna=True
+        # if no rows after dropping na if dropna=True
         original_by = [x for x in by]
         query = gdf[by.pop(0)].notna()
         for col in gdf[by]:
             query &= gdf[col].notna()
         gdf = gdf.loc[query]
         assert not len(gdf), gdf
-        if not by_was_none and as_index:
-            try:
-                gdf = gdf.set_index(original_by)
-            except Exception as e:
-                print(gdf)
-                print(original_by)
-                raise e
-
+        if as_index and not by_was_none:
+            gdf = gdf.set_index(original_by)
         return gdf
 
-    if not by_was_none and as_index:
+    if as_index and not by_was_none:
         one_hit = gdf[is_one_hit].set_index(by)
     else:
         one_hit = gdf[is_one_hit]
@@ -250,38 +244,21 @@ def _dissolve(
     dissolved = many_hits.groupby(by, as_index=True, **dissolve_kwargs)[other_cols].agg(
         aggfunc
     )
-
-    if n_jobs > 1:
-        try:
-            agged = _parallel_unary_union(
-                many_hits,
-                n_jobs=n_jobs,
-                by=by,
-                grid_size=grid_size,
-                as_index=True,
-                **dissolve_kwargs,
-            )
-            dissolved[geom_col] = agged
-            return GeoDataFrame(dissolved, geometry=geom_col, crs=gdf.crs)
-        except Exception as e:
-            print(e, dissolved, agged, many_hits)
-            raise e
-
-    geoms_agged = many_hits.groupby(by, **dissolve_kwargs)[geom_col].agg(
-        lambda x: _unary_union_for_notna(x, grid_size=grid_size)
+    dissolved[geom_col] = union_runner.run(
+        many_hits,
+        by=by,
+        grid_size=grid_size,
+        as_index=True,
+        **dissolve_kwargs,
     )
-
-    dissolved[geom_col] = geoms_agged
-
     if not as_index:
         dissolved = dissolved.reset_index()
-
     try:
         return GeoDataFrame(
             pd.concat([dissolved, one_hit]).sort_index(), geometry=geom_col, crs=gdf.crs
         )
     except TypeError as e:
-        raise e.__class__(e, dissolved.index, one_hit.index) from e
+        raise e.__class__(f"{e}. {dissolved.index}. {one_hit.index}") from e
 
 
 def diss(
@@ -582,7 +559,7 @@ def buff(
     if copy:
         gdf = gdf.copy()
 
-    gdf[gdf._geometry_column_name] = gdf.buffer(
+    gdf[gdf.geometry.name] = gdf.buffer(
         distance, resolution=resolution, join_style=join_style, **buffer_kwargs
     ).make_valid()
 

sgis/geopandas_tools/conversion.py

@@ -146,31 +146,24 @@ def to_bbox(
     except Exception:
         pass
 
-    try:
-        minx = float(np.min(obj["minx"]))  # type: ignore [index]
-        miny = float(np.min(obj["miny"]))  # type: ignore [index]
-        maxx = float(np.max(obj["maxx"]))  # type: ignore [index]
-        maxy = float(np.max(obj["maxy"]))  # type: ignore [index]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
-    try:
-        minx = float(np.min(obj.minx))  # type: ignore [union-attr]
-        miny = float(np.min(obj.miny))  # type: ignore [union-attr]
-        maxx = float(np.max(obj.maxx))  # type: ignore [union-attr]
-        maxy = float(np.max(obj.maxy))  # type: ignore [union-attr]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
-
-    try:
-        minx = float(np.min(obj["west_longitude"]))  # type: ignore [index]
-        miny = float(np.min(obj["south_latitude"]))  # type: ignore [index]
-        maxx = float(np.max(obj["east_longitude"]))  # type: ignore [index]
-        maxy = float(np.max(obj["north_latitude"]))  # type: ignore [index]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
+    def to_int_if_possible(x):
+        if isinstance(x, int) or float(x).is_integer():
+            return int(x)
+        return float(x)
+
+    for attrs in [
+        ("minx", "miny", "maxx", "maxy"),
+        ("xmin", "ymin", "xmax", "xmax"),
+        ("west_longitude", "south_latitude", "east_longitude", "north_latitude"),
+    ]:
+        try:
+            return tuple(to_int_if_possible(obj[attr]) for attr in attrs)
+        except Exception:
+            pass
+        try:
+            return tuple(to_int_if_possible(getattr(obj, attr)) for attr in attrs)
+        except Exception:
+            pass
 
     if hasattr(obj, "geometry"):
         try:

sgis/geopandas_tools/duplicates.py

@@ -6,21 +6,19 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import STRtree
 from shapely import difference
-from shapely import make_valid
 from shapely import simplify
-from shapely.errors import GEOSException
 
+from ..conf import config
 from .general import _determine_geom_type_args
-from .general import _grouped_unary_union
-from .general import _parallel_unary_union_geoseries
 from .general import _push_geom_col
 from .general import clean_geoms
 from .geometry_types import get_geom_type
 from .geometry_types import make_all_singlepart
 from .geometry_types import to_single_geom_type
-from .overlay import _run_overlay_dask
 from .overlay import clean_overlay
-from .overlay import make_valid_and_keep_geom_type
+from .runners import OverlayRunner
+from .runners import RTreeQueryRunner
+from .runners import UnionRunner
 from .sfilter import sfilter_inverse
 
 PRECISION = 1e-3
@@ -31,8 +29,11 @@ def update_geometries(
     geom_type: str | None = None,
     keep_geom_type: bool | None = None,
     grid_size: int | None = None,
-    n_jobs: int = 1,
     predicate: str | None = "intersects",
+    n_jobs: int = 1,
+    union_runner: UnionRunner | None = None,
+    rtree_runner: RTreeQueryRunner | None = None,
+    overlay_runner: OverlayRunner | None = None,
 ) -> GeoDataFrame:
     """Puts geometries on top of each other rowwise.
 
@@ -50,8 +51,14 @@ def update_geometries(
             "line" or "point".
         grid_size: Precision grid size to round the geometries. Will use the highest
             precision of the inputs by default.
-        n_jobs: Number of threads.
         predicate: Spatial predicate for the spatial tree.
+        n_jobs: Number of workers.
+        union_runner: Optionally debug/manipulate the spatial union operations.
+            See the 'runners' module for example implementations.
+        rtree_runner: Optionally debug/manipulate the spatial indexing operations.
+            See the 'runners' module for example implementations.
+        overlay_runner: Optionally debug/manipulate the spatial overlay operations.
+            See the 'runners' module for example implementations.
 
     Example:
     --------
@@ -98,6 +105,16 @@ def update_geometries(
     if len(gdf) <= 1:
         return gdf
 
+    if rtree_runner is None:
+        rtree_runner = config.get_instance("rtree_runner", n_jobs)
+    if union_runner is None:
+        union_runner = config.get_instance("union_runner", n_jobs)
+    if overlay_runner is None:
+        overlay_runner = config.get_instance("overlay_runner", n_jobs)
+
+    if geom_type == "polygon" or get_geom_type(gdf) == "polygon":
+        gdf.geometry = gdf.buffer(0)
+
     copied = make_all_singlepart(clean_geoms(gdf))
 
     copied, geom_type, keep_geom_type = _determine_geom_type_args(
@@ -108,66 +125,35 @@ def update_geometries(
     index_mapper = {i: idx for i, idx in enumerate(copied.index)}
     copied = copied.reset_index(drop=True)
 
-    tree = STRtree(copied.geometry.values)
-    left, right = tree.query(copied.geometry.values, predicate=predicate)
+    left, right = rtree_runner.run(
+        copied.geometry.values, copied.geometry.values, predicate=predicate
+    )
     indices = pd.Series(right, index=left).loc[lambda x: x.index > x.values]
 
     # select geometries from 'right', index from 'left', dissolve by 'left'
     erasers = pd.Series(copied.geometry.loc[indices.values].values, index=indices.index)
-    if n_jobs > 1:
-        erasers = _parallel_unary_union_geoseries(
-            erasers,
-            level=0,
-            n_jobs=n_jobs,
-            grid_size=grid_size,
-        )
-        erasers = pd.Series(erasers, index=indices.index.unique())
-    else:
-        only_one = erasers.groupby(level=0).transform("size") == 1
-        one_hit = erasers[only_one]
-        many_hits = _grouped_unary_union(
-            erasers[~only_one], level=0, grid_size=grid_size
-        )
-        erasers = pd.concat([one_hit, many_hits]).sort_index()
+    only_one = erasers.groupby(level=0).transform("size") == 1
+    one_hit = erasers[only_one]
+    many_hits = union_runner.run(erasers[~only_one], level=0, grid_size=grid_size)
+    erasers = pd.concat([one_hit, many_hits]).sort_index()
 
     # match up the aggregated erasers by index
-    if n_jobs > 1:
-        arr1 = copied.geometry.loc[erasers.index].to_numpy()
-        arr2 = erasers.to_numpy()
-        try:
-            erased = _run_overlay_dask(
-                arr1, arr2, func=difference, n_jobs=n_jobs, grid_size=grid_size
-            )
-        except GEOSException:
-            arr1 = make_valid_and_keep_geom_type(
-                arr1, geom_type=geom_type, n_jobs=n_jobs
-            )
-            arr2 = make_valid_and_keep_geom_type(
-                arr2, geom_type=geom_type, n_jobs=n_jobs
-            )
-            erased = _run_overlay_dask(
-                arr1, arr2, func=difference, n_jobs=n_jobs, grid_size=grid_size
-            )
-        erased = GeoSeries(erased, index=erasers.index)
-    else:
-        erased = make_valid(
-            difference(
-                copied.geometry.loc[erasers.index],
-                erasers,
-                grid_size=grid_size,
-            )
-        )
+    arr1 = copied.geometry.loc[erasers.index].to_numpy()
+    arr2 = erasers.to_numpy()
+    erased = overlay_runner.run(
+        difference, arr1, arr2, grid_size=grid_size, geom_type=geom_type
+    )
 
+    erased = GeoSeries(erased, index=erasers.index)
     copied.loc[erased.index, geom_col] = erased
-
     copied = copied.loc[~copied.is_empty]
-
     copied.index = copied.index.map(index_mapper)
-
     copied = make_all_singlepart(copied)
 
     # TODO check why polygons dissappear in rare cases. For now, just add back the missing
-    dissapeared = sfilter_inverse(gdf, copied.buffer(-PRECISION))
+    dissapeared = sfilter_inverse(
+        gdf, copied.buffer(-PRECISION), rtree_runner=rtree_runner
+    )
     copied = pd.concat([copied, dissapeared])
 
     # TODO fix dupliates again with dissolve?
@@ -188,7 +174,7 @@ def get_intersections(
     keep_geom_type: bool | None = None,
     predicate: str | None = "intersects",
     grid_size: float | None = None,
-    n_jobs: int = 1,
+    **kwargs,
 ) -> GeoDataFrame:
     """Find geometries that intersect in a GeoDataFrame.
 
@@ -211,6 +197,7 @@ def get_intersections(
             precision of the inputs by default.
         n_jobs: Number of threads.
         predicate: Spatial predicate for the spatial tree.
+        **kwargs: Keyword arguments passed to clean_overlay.
 
     Returns:
         A GeoDataFrame of the overlapping polygons.
@@ -283,9 +270,9 @@ def get_intersections(
         gdf,
         geom_type,
         keep_geom_type,
-        n_jobs=n_jobs,
         grid_size=grid_size,
         predicate=predicate,
+        **kwargs,
     ).pipe(clean_geoms)
 
     duplicated_geoms.index = duplicated_geoms["orig_idx"].values
@@ -301,9 +288,9 @@ def _get_intersecting_geometries(
     gdf: GeoDataFrame,
     geom_type: str | None,
     keep_geom_type: bool,
-    n_jobs: int,
     grid_size: float | None = None,
     predicate: str | None = None,
+    **kwargs,
 ) -> GeoDataFrame:
     right = gdf[[gdf._geometry_column_name]]
     right["idx_right"] = right.index
@@ -327,7 +314,7 @@ def _get_intersecting_geometries(
             grid_size=grid_size,
             geom_type=geom_type,
             keep_geom_type=keep_geom_type,
-            n_jobs=n_jobs,
+            **kwargs,
         ).loc[are_not_identical]
     else:
         if keep_geom_type:
@@ -347,7 +334,7 @@ def _get_intersecting_geometries(
                     grid_size=grid_size,
                     predicate=predicate,
                     geom_type=geom_type,
-                    n_jobs=n_jobs,
+                    **kwargs,
                 )
             ]
         intersected = pd.concat(intersected, ignore_index=True).loc[are_not_identical]

sgis/geopandas_tools/general.py

@@ -6,7 +6,6 @@ from collections.abc import Hashable
 from collections.abc import Iterable
 from typing import Any
 
-import joblib
 import numpy as np
 import pandas as pd
 import pyproj
@@ -23,7 +22,6 @@ from shapely import get_parts
 from shapely import linestrings
 from shapely import make_valid
 from shapely import points as shapely_points
-from shapely import union_all
 from shapely.geometry import LineString
 from shapely.geometry import MultiPoint
 from shapely.geometry import Point
@@ -333,7 +331,7 @@ def sort_large_first(gdf: GeoDataFrame | GeoSeries) -> GeoDataFrame | GeoSeries:
     # using enumerate, then iloc on the sorted dict keys.
     # to avoid creating a temporary area column (which doesn't work for GeoSeries).
     area_mapper = dict(enumerate(gdf.area.values))
-    sorted_areas = dict(reversed(sorted(area_mapper.items(), key=lambda item: item[1])))
+    sorted_areas = dict(reversed(sorted(area_mapper.items(), key=_get_dict_value)))
     return gdf.iloc[list(sorted_areas)]
 
 
@@ -349,9 +347,7 @@ def sort_long_first(gdf: GeoDataFrame | GeoSeries) -> GeoDataFrame | GeoSeries:
     # using enumerate, then iloc on the sorted dict keys.
     # to avoid creating a temporary area column (which doesn't work for GeoSeries).
     length_mapper = dict(enumerate(gdf.length.values))
-    sorted_lengths = dict(
-        reversed(sorted(length_mapper.items(), key=lambda item: item[1]))
-    )
+    sorted_lengths = dict(reversed(sorted(length_mapper.items(), key=_get_dict_value)))
     return gdf.iloc[list(sorted_lengths)]
 
 
@@ -367,7 +363,7 @@ def sort_short_first(gdf: GeoDataFrame | GeoSeries) -> GeoDataFrame | GeoSeries:
     # using enumerate, then iloc on the sorted dict keys.
     # to avoid creating a temporary area column (which doesn't work for GeoSeries).
     length_mapper = dict(enumerate(gdf.length.values))
-    sorted_lengths = dict(sorted(length_mapper.items(), key=lambda item: item[1]))
+    sorted_lengths = dict(sorted(length_mapper.items(), key=_get_dict_value))
     return gdf.iloc[list(sorted_lengths)]
 
 
@@ -384,10 +380,14 @@ def sort_small_first(gdf: GeoDataFrame | GeoSeries) -> GeoDataFrame | GeoSeries:
     # using enumerate, then iloc on the sorted dict keys.
     # to avoid creating a temporary area column (which doesn't work for GeoSeries).
     area_mapper = dict(enumerate(gdf.area.values))
-    sorted_areas = dict(sorted(area_mapper.items(), key=lambda item: item[1]))
+    sorted_areas = dict(sorted(area_mapper.items(), key=_get_dict_value))
     return gdf.iloc[list(sorted_areas)]
 
 
+def _get_dict_value(item: tuple[Hashable, Any]) -> Any:
+    return item[1]
+
+
 def make_lines_between_points(
     *arrs: NDArray[Point] | GeometryArray | GeoSeries,
 ) -> NDArray[LineString]:
@@ -1121,79 +1121,3 @@ def _determine_geom_type_args(
         if geom_type == "mixed":
             raise ValueError("Cannot set keep_geom_type=True with mixed geometries")
     return gdf, geom_type, keep_geom_type
-
-
-def _unary_union_for_notna(geoms, **kwargs):
-    try:
-        return make_valid(union_all(geoms, **kwargs))
-    except TypeError:
-        return union_all([geom for geom in geoms.dropna().values], **kwargs)
-
-
-def _grouped_unary_union(
-    df: GeoDataFrame | GeoSeries | pd.DataFrame | pd.Series,
-    by: str | list[str] | None = None,
-    level: int | None = None,
-    as_index: bool = True,
-    grid_size: float | int | None = None,
-    dropna: bool = False,
-    **kwargs,
-) -> GeoSeries | GeoDataFrame:
-    """Vectorized unary_union for groups.
-
-    Experimental. Messy code.
-    """
-    try:
-        geom_col = df._geometry_column_name
-    except AttributeError:
-        try:
-            geom_col = df.name
-            if geom_col is None:
-                geom_col = "geometry"
-        except AttributeError:
-            geom_col = "geometry"
-
-    if isinstance(df, pd.Series):
-        return GeoSeries(
-            df.groupby(level=level, as_index=as_index, **kwargs).agg(
-                lambda x: _unary_union_for_notna(x, grid_size=grid_size)
-            )
-        )
-
-    return GeoSeries(
-        df.groupby(by, level=level, as_index=as_index, **kwargs)[geom_col].agg(
-            lambda x: _unary_union_for_notna(x, grid_size=grid_size)
-        )
-    )
-
-
-def _parallel_unary_union(
-    gdf: GeoDataFrame, n_jobs: int = 1, by=None, grid_size=None, **kwargs
-) -> list[Geometry]:
-    try:
-        geom_col = gdf._geometry_column_name
-    except AttributeError:
-        geom_col = "geometry"
-
-    with joblib.Parallel(n_jobs=n_jobs, backend="threading") as parallel:
-        delayed_operations = []
-        for _, geoms in gdf.groupby(by, **kwargs)[geom_col]:
-            delayed_operations.append(
-                joblib.delayed(_unary_union_for_notna)(geoms, grid_size=grid_size)
-            )
-
-        return parallel(delayed_operations)
-
-
-def _parallel_unary_union_geoseries(
-    ser: GeoSeries, n_jobs: int = 1, grid_size=None, **kwargs
-) -> list[Geometry]:
-
-    with joblib.Parallel(n_jobs=n_jobs, backend="threading") as parallel:
-        delayed_operations = []
-        for _, geoms in ser.groupby(**kwargs):
-            delayed_operations.append(
-                joblib.delayed(_unary_union_for_notna)(geoms, grid_size=grid_size)
-            )
-
-        return parallel(delayed_operations)