ssb-sgis 1.1.17__tar.gz → 1.2.1__tar.gz

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 1.1.17
+Version: 1.2.1
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-sgis"
-version = "1.1.17"
+version = "1.2.1"
 description = "GIS functions used at Statistics Norway."
 authors = ["Morten Letnes <morten.letnes@ssb.no>"]
 license = "MIT"

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/src/sgis/__init__.py

@@ -42,6 +42,7 @@ from .geopandas_tools.general import make_lines_between_points
 from .geopandas_tools.general import points_in_bounds
 from .geopandas_tools.general import random_points
 from .geopandas_tools.general import random_points_in_polygons
+from .geopandas_tools.general import random_points_norway
 from .geopandas_tools.general import sort_large_first
 from .geopandas_tools.general import sort_long_first
 from .geopandas_tools.general import sort_short_first
@@ -76,6 +77,10 @@ from .geopandas_tools.polygon_operations import get_holes
 from .geopandas_tools.polygon_operations import get_polygon_clusters
 from .geopandas_tools.polygon_operations import split_polygons_by_lines
 from .geopandas_tools.polygons_as_rings import PolygonsAsRings
+from .geopandas_tools.runners import GridSizeOverlayRunner
+from .geopandas_tools.runners import OverlayRunner
+from .geopandas_tools.runners import RTreeQueryRunner
+from .geopandas_tools.runners import UnionRunner
 from .geopandas_tools.sfilter import sfilter
 from .geopandas_tools.sfilter import sfilter_inverse
 from .geopandas_tools.sfilter import sfilter_split

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/src/sgis/conf.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 try:
     from gcsfs import GCSFileSystem
 
@@ -66,7 +68,23 @@ except ImportError:
 
     file_system = LocalFileSystem
 
+from .geopandas_tools.runners import OverlayRunner
+from .geopandas_tools.runners import RTreeQueryRunner
+from .geopandas_tools.runners import UnionRunner
+
+
+def _get_instance(data: dict, key: str, **kwargs) -> Any:
+    """Get the dict value and call it if callable."""
+    x = data[key]
+    if callable(x):
+        return x(**kwargs)
+    return x
+
+
 config = {
     "n_jobs": 1,
     "file_system": file_system,
+    "rtree_runner": RTreeQueryRunner,
+    "overlay_runner": OverlayRunner,
+    "union_runner": UnionRunner,
 }

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/src/sgis/geopandas_tools/buffer_dissolve_explode.py

@@ -23,12 +23,14 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import get_num_geometries
 
+from ..conf import _get_instance
+from ..conf import config
 from ..parallel.parallel import Parallel
-from .general import _parallel_unary_union
-from .general import _unary_union_for_notna
 from .geometry_types import make_all_singlepart
 from .polygon_operations import get_cluster_mapper
 from .polygon_operations import get_grouped_centroids
+from .runners import UnionRunner
+from .utils import _unary_union_for_notna
 
 
 def _decide_ignore_index(kwargs: dict) -> tuple[dict, bool]:
@@ -53,8 +55,8 @@ def buffdissexp(
     index_parts: bool = False,
     copy: bool = True,
     grid_size: float | int | None = None,
-    n_jobs: int = 1,
     join_style: int | str = "round",
+    n_jobs: int = 1,
     **dissolve_kwargs,
 ) -> GeoDataFrame:
     """Buffers and dissolves overlapping geometries.
@@ -187,27 +189,27 @@ def _dissolve(
     gdf: GeoDataFrame,
     aggfunc: str = "first",
     grid_size: None | float = None,
-    n_jobs: int = 1,
     as_index: bool = True,
+    n_jobs: int = 1,
+    union_runner: UnionRunner | None = None,
     **dissolve_kwargs,
 ) -> GeoDataFrame:
-
     if not len(gdf):
         return gdf
 
-    geom_col = gdf._geometry_column_name
+    if union_runner is None:
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
 
-    gdf[geom_col] = gdf[geom_col].make_valid()
+    geom_col = gdf.geometry.name
+    by = dissolve_kwargs.pop("by", None)
+    by_was_none = not bool(by)
 
+    # make sure geometries are dissolved rowwise to make dissolving simpler later
     more_than_one = get_num_geometries(gdf.geometry.values) > 1
     gdf.loc[more_than_one, geom_col] = gdf.loc[more_than_one, geom_col].apply(
         _unary_union_for_notna
     )
 
-    by = dissolve_kwargs.pop("by", None)
-
-    by_was_none = not bool(by)
-
     if by is None and dissolve_kwargs.get("level") is None:
         by = np.zeros(len(gdf), dtype="int64")
         other_cols = list(gdf.columns.difference({geom_col}))
@@ -215,30 +217,23 @@ def _dissolve(
         if isinstance(by, str):
             by = [by]
         other_cols = list(gdf.columns.difference({geom_col} | set(by or {})))
-
     try:
         is_one_hit = (
             gdf.groupby(by, as_index=True, **dissolve_kwargs).transform("size") == 1
         )
     except IndexError:
-        # if no rows when dropna=True
+        # if no rows after dropping na if dropna=True
         original_by = [x for x in by]
         query = gdf[by.pop(0)].notna()
         for col in gdf[by]:
             query &= gdf[col].notna()
         gdf = gdf.loc[query]
         assert not len(gdf), gdf
-        if not by_was_none and as_index:
-            try:
-                gdf = gdf.set_index(original_by)
-            except Exception as e:
-                print(gdf)
-                print(original_by)
-                raise e
-
+        if as_index and not by_was_none:
+            gdf = gdf.set_index(original_by)
         return gdf
 
-    if not by_was_none and as_index:
+    if as_index and not by_was_none:
         one_hit = gdf[is_one_hit].set_index(by)
     else:
         one_hit = gdf[is_one_hit]
@@ -250,38 +245,21 @@ def _dissolve(
     dissolved = many_hits.groupby(by, as_index=True, **dissolve_kwargs)[other_cols].agg(
         aggfunc
     )
-
-    if n_jobs > 1:
-        try:
-            agged = _parallel_unary_union(
-                many_hits,
-                n_jobs=n_jobs,
-                by=by,
-                grid_size=grid_size,
-                as_index=True,
-                **dissolve_kwargs,
-            )
-            dissolved[geom_col] = agged
-            return GeoDataFrame(dissolved, geometry=geom_col, crs=gdf.crs)
-        except Exception as e:
-            print(e, dissolved, agged, many_hits)
-            raise e
-
-    geoms_agged = many_hits.groupby(by, **dissolve_kwargs)[geom_col].agg(
-        lambda x: _unary_union_for_notna(x, grid_size=grid_size)
+    dissolved[geom_col] = union_runner.run(
+        many_hits,
+        by=by,
+        grid_size=grid_size,
+        as_index=True,
+        **dissolve_kwargs,
     )
-
-    dissolved[geom_col] = geoms_agged
-
     if not as_index:
         dissolved = dissolved.reset_index()
-
     try:
         return GeoDataFrame(
             pd.concat([dissolved, one_hit]).sort_index(), geometry=geom_col, crs=gdf.crs
         )
     except TypeError as e:
-        raise e.__class__(e, dissolved.index, one_hit.index) from e
+        raise e.__class__(f"{e}. {dissolved.index}. {one_hit.index}") from e
 
 
 def diss(
@@ -582,7 +560,7 @@ def buff(
     if copy:
         gdf = gdf.copy()
 
-    gdf[gdf._geometry_column_name] = gdf.buffer(
+    gdf[gdf.geometry.name] = gdf.buffer(
         distance, resolution=resolution, join_style=join_style, **buffer_kwargs
     ).make_valid()
 

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/src/sgis/geopandas_tools/conversion.py

@@ -146,31 +146,24 @@ def to_bbox(
     except Exception:
         pass
 
-    try:
-        minx = float(np.min(obj["minx"]))  # type: ignore [index]
-        miny = float(np.min(obj["miny"]))  # type: ignore [index]
-        maxx = float(np.max(obj["maxx"]))  # type: ignore [index]
-        maxy = float(np.max(obj["maxy"]))  # type: ignore [index]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
-    try:
-        minx = float(np.min(obj.minx))  # type: ignore [union-attr]
-        miny = float(np.min(obj.miny))  # type: ignore [union-attr]
-        maxx = float(np.max(obj.maxx))  # type: ignore [union-attr]
-        maxy = float(np.max(obj.maxy))  # type: ignore [union-attr]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
-
-    try:
-        minx = float(np.min(obj["west_longitude"]))  # type: ignore [index]
-        miny = float(np.min(obj["south_latitude"]))  # type: ignore [index]
-        maxx = float(np.max(obj["east_longitude"]))  # type: ignore [index]
-        maxy = float(np.max(obj["north_latitude"]))  # type: ignore [index]
-        return minx, miny, maxx, maxy
-    except Exception:
-        pass
+    def to_int_if_possible(x):
+        if isinstance(x, int) or float(x).is_integer():
+            return int(x)
+        return float(x)
+
+    for attrs in [
+        ("minx", "miny", "maxx", "maxy"),
+        ("xmin", "ymin", "xmax", "xmax"),
+        ("west_longitude", "south_latitude", "east_longitude", "north_latitude"),
+    ]:
+        try:
+            return tuple(to_int_if_possible(obj[attr]) for attr in attrs)
+        except Exception:
+            pass
+        try:
+            return tuple(to_int_if_possible(getattr(obj, attr)) for attr in attrs)
+        except Exception:
+            pass
 
     if hasattr(obj, "geometry"):
         try:

{ssb_sgis-1.1.17 → ssb_sgis-1.2.1}/src/sgis/geopandas_tools/duplicates.py

@@ -6,21 +6,20 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import STRtree
 from shapely import difference
-from shapely import make_valid
 from shapely import simplify
-from shapely.errors import GEOSException
 
+from ..conf import _get_instance
+from ..conf import config
 from .general import _determine_geom_type_args
-from .general import _grouped_unary_union
-from .general import _parallel_unary_union_geoseries
 from .general import _push_geom_col
 from .general import clean_geoms
 from .geometry_types import get_geom_type
 from .geometry_types import make_all_singlepart
 from .geometry_types import to_single_geom_type
-from .overlay import _run_overlay_dask
 from .overlay import clean_overlay
-from .overlay import make_valid_and_keep_geom_type
+from .runners import OverlayRunner
+from .runners import RTreeQueryRunner
+from .runners import UnionRunner
 from .sfilter import sfilter_inverse
 
 PRECISION = 1e-3
@@ -31,8 +30,11 @@ def update_geometries(
     geom_type: str | None = None,
     keep_geom_type: bool | None = None,
     grid_size: int | None = None,
-    n_jobs: int = 1,
     predicate: str | None = "intersects",
+    n_jobs: int = 1,
+    union_runner: UnionRunner | None = None,
+    rtree_runner: RTreeQueryRunner | None = None,
+    overlay_runner: OverlayRunner | None = None,
 ) -> GeoDataFrame:
     """Puts geometries on top of each other rowwise.
 
@@ -50,8 +52,14 @@ def update_geometries(
             "line" or "point".
         grid_size: Precision grid size to round the geometries. Will use the highest
             precision of the inputs by default.
-        n_jobs: Number of threads.
         predicate: Spatial predicate for the spatial tree.
+        n_jobs: Number of workers.
+        union_runner: Optionally debug/manipulate the spatial union operations.
+            See the 'runners' module for example implementations.
+        rtree_runner: Optionally debug/manipulate the spatial indexing operations.
+            See the 'runners' module for example implementations.
+        overlay_runner: Optionally debug/manipulate the spatial overlay operations.
+            See the 'runners' module for example implementations.
 
     Example:
     --------
@@ -98,6 +106,13 @@ def update_geometries(
     if len(gdf) <= 1:
         return gdf
 
+    if rtree_runner is None:
+        rtree_runner = _get_instance(config, "rtree_runner", n_jobs=n_jobs)
+    if union_runner is None:
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
+    if overlay_runner is None:
+        overlay_runner = _get_instance(config, "overlay_runner", n_jobs=n_jobs)
+
     if geom_type == "polygon" or get_geom_type(gdf) == "polygon":
         gdf.geometry = gdf.buffer(0)
 
@@ -111,66 +126,35 @@ def update_geometries(
     index_mapper = {i: idx for i, idx in enumerate(copied.index)}
     copied = copied.reset_index(drop=True)
 
-    tree = STRtree(copied.geometry.values)
-    left, right = tree.query(copied.geometry.values, predicate=predicate)
+    left, right = rtree_runner.run(
+        copied.geometry.values, copied.geometry.values, predicate=predicate
+    )
     indices = pd.Series(right, index=left).loc[lambda x: x.index > x.values]
 
     # select geometries from 'right', index from 'left', dissolve by 'left'
     erasers = pd.Series(copied.geometry.loc[indices.values].values, index=indices.index)
-    if n_jobs > 1:
-        erasers = _parallel_unary_union_geoseries(
-            erasers,
-            level=0,
-            n_jobs=n_jobs,
-            grid_size=grid_size,
-        )
-        erasers = pd.Series(erasers, index=indices.index.unique())
-    else:
-        only_one = erasers.groupby(level=0).transform("size") == 1
-        one_hit = erasers[only_one]
-        many_hits = _grouped_unary_union(
-            erasers[~only_one], level=0, grid_size=grid_size
-        )
-        erasers = pd.concat([one_hit, many_hits]).sort_index()
+    only_one = erasers.groupby(level=0).transform("size") == 1
+    one_hit = erasers[only_one]
+    many_hits = union_runner.run(erasers[~only_one], level=0, grid_size=grid_size)
+    erasers = pd.concat([one_hit, many_hits]).sort_index()
 
     # match up the aggregated erasers by index
-    if n_jobs > 1:
-        arr1 = copied.geometry.loc[erasers.index].to_numpy()
-        arr2 = erasers.to_numpy()
-        try:
-            erased = _run_overlay_dask(
-                arr1, arr2, func=difference, n_jobs=n_jobs, grid_size=grid_size
-            )
-        except GEOSException:
-            arr1 = make_valid_and_keep_geom_type(
-                arr1, geom_type=geom_type, n_jobs=n_jobs
-            )
-            arr2 = make_valid_and_keep_geom_type(
-                arr2, geom_type=geom_type, n_jobs=n_jobs
-            )
-            erased = _run_overlay_dask(
-                arr1, arr2, func=difference, n_jobs=n_jobs, grid_size=grid_size
-            )
-        erased = GeoSeries(erased, index=erasers.index)
-    else:
-        erased = make_valid(
-            difference(
-                copied.geometry.loc[erasers.index],
-                erasers,
-                grid_size=grid_size,
-            )
-        )
+    arr1 = copied.geometry.loc[erasers.index].to_numpy()
+    arr2 = erasers.to_numpy()
+    erased = overlay_runner.run(
+        difference, arr1, arr2, grid_size=grid_size, geom_type=geom_type
+    )
 
+    erased = GeoSeries(erased, index=erasers.index)
     copied.loc[erased.index, geom_col] = erased
-
     copied = copied.loc[~copied.is_empty]
-
     copied.index = copied.index.map(index_mapper)
-
     copied = make_all_singlepart(copied)
 
     # TODO check why polygons dissappear in rare cases. For now, just add back the missing
-    dissapeared = sfilter_inverse(gdf, copied.buffer(-PRECISION))
+    dissapeared = sfilter_inverse(
+        gdf, copied.buffer(-PRECISION), rtree_runner=rtree_runner
+    )
     copied = pd.concat([copied, dissapeared])
 
     # TODO fix dupliates again with dissolve?
@@ -191,7 +175,7 @@ def get_intersections(
     keep_geom_type: bool | None = None,
     predicate: str | None = "intersects",
     grid_size: float | None = None,
-    n_jobs: int = 1,
+    **kwargs,
 ) -> GeoDataFrame:
     """Find geometries that intersect in a GeoDataFrame.
 
@@ -214,6 +198,7 @@ def get_intersections(
             precision of the inputs by default.
         n_jobs: Number of threads.
         predicate: Spatial predicate for the spatial tree.
+        **kwargs: Keyword arguments passed to clean_overlay.
 
     Returns:
         A GeoDataFrame of the overlapping polygons.
@@ -286,9 +271,9 @@ def get_intersections(
         gdf,
         geom_type,
         keep_geom_type,
-        n_jobs=n_jobs,
         grid_size=grid_size,
         predicate=predicate,
+        **kwargs,
     ).pipe(clean_geoms)
 
     duplicated_geoms.index = duplicated_geoms["orig_idx"].values
@@ -304,9 +289,9 @@ def _get_intersecting_geometries(
     gdf: GeoDataFrame,
     geom_type: str | None,
     keep_geom_type: bool,
-    n_jobs: int,
     grid_size: float | None = None,
     predicate: str | None = None,
+    **kwargs,
 ) -> GeoDataFrame:
     right = gdf[[gdf._geometry_column_name]]
     right["idx_right"] = right.index
@@ -330,7 +315,7 @@ def _get_intersecting_geometries(
             grid_size=grid_size,
             geom_type=geom_type,
             keep_geom_type=keep_geom_type,
-            n_jobs=n_jobs,
+            **kwargs,
         ).loc[are_not_identical]
     else:
         if keep_geom_type:
@@ -350,7 +335,7 @@ def _get_intersecting_geometries(
                     grid_size=grid_size,
                     predicate=predicate,
                     geom_type=geom_type,
-                    n_jobs=n_jobs,
+                    **kwargs,
                 )
             ]
         intersected = pd.concat(intersected, ignore_index=True).loc[are_not_identical]