ssb-sgis 0.3.9__tar.gz → 0.3.11__tar.gz

{ssb_sgis-0.3.9 → ssb_sgis-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 0.3.9
+Version: 0.3.11
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT
@@ -24,18 +24,15 @@ Requires-Dist: jenkspy (>=0.3.2)
 Requires-Dist: mapclassify (>=2.5.0)
 Requires-Dist: matplotlib (>=3.7.0)
 Requires-Dist: networkx (>=3.0)
-Requires-Dist: numba (>=0.57.1,<0.58.0)
 Requires-Dist: numpy (>=1.24.2)
 Requires-Dist: pandas (>=1.5.3)
 Requires-Dist: pip (==23.2.1)
 Requires-Dist: pyarrow (>=11.0.0)
 Requires-Dist: rasterio (>=1.3.8,<2.0.0)
 Requires-Dist: requests (>=2.28.2)
-Requires-Dist: rioxarray (>=0.14.1,<0.15.0)
 Requires-Dist: rtree (>=1.0.1,<2.0.0)
 Requires-Dist: scikit-learn (>=1.2.1)
 Requires-Dist: shapely (>=2.0.1)
-Requires-Dist: xarray (==2023.8.0)
 Requires-Dist: xyzservices (>=2023.2.0)
 Project-URL: Changelog, https://github.com/statisticsnorway/ssb-sgis/releases
 Project-URL: Repository, https://github.com/statisticsnorway/ssb-sgis

{ssb_sgis-0.3.9 → ssb_sgis-0.3.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-sgis"
-version = "0.3.9"
+version = "0.3.11"
 description = "GIS functions used at Statistics Norway."
 authors = ["Statistics Norway <ort@ssb.no>"]
 license = "MIT"
@@ -37,9 +37,6 @@ ipython = ">=8.13.2"
 rtree = "^1.0.1"
 geocoder = "^1.38.1"
 rasterio = "^1.3.8"
-xarray = "2023.8.0"
-rioxarray = "^0.14.1"
-numba = "^0.57.1"
 pip = "23.2.1"
 
 [tool.poetry.group.dev.dependencies]
@@ -47,6 +44,9 @@ black = {extras = ["d", "jupyter"], version = ">=23.1.0"}
 coverage = {extras = ["toml"], version = ">=7.2.1"}
 darglint = ">=1.8.1"
 deptry = ">=0.8.0"
+numba = "^0.57.1"
+xarray = "2023.8.0"
+rioxarray = "^0.14.1"
 flake8 = ">=6.0.0"
 flake8-bandit = ">=4.1.1"
 flake8-bugbear = ">=23.2.13"

{ssb_sgis-0.3.9 → ssb_sgis-0.3.11}/src/sgis/__init__.py

@@ -1,4 +1,5 @@
 from .geopandas_tools.bounds import (
+    Gridlooper,
     bounds_to_points,
     bounds_to_polygon,
     get_total_bounds,
@@ -18,11 +19,15 @@ from .geopandas_tools.buffer_dissolve_explode import (
     dissexp_by_cluster,
 )
 from .geopandas_tools.centerlines import get_rough_centerlines
-from .geopandas_tools.cleaning import coverage_clean, remove_spikes
+from .geopandas_tools.cleaning import (
+    coverage_clean,
+    remove_spikes,
+    split_spiky_polygons,
+)
 from .geopandas_tools.conversion import (
     coordinate_array,
-    get_lonlat,
-    get_utm33,
+    from_4326,
+    to_4326,
     to_gdf,
     to_geoseries,
     to_shapely,
@@ -38,9 +43,12 @@ from .geopandas_tools.general import (
     get_common_crs,
     get_grouped_centroids,
     random_points,
+    random_points_in_polygons,
     rename_geometry_if,
     sort_large_first,
     sort_long_first,
+    sort_short_first,
+    sort_small_first,
     to_lines,
 )
 from .geopandas_tools.geocoding import address_to_coords, address_to_gdf
@@ -63,6 +71,7 @@ from .geopandas_tools.polygon_operations import (
     PolygonsAsRings,
     close_all_holes,
     close_small_holes,
+    close_thin_holes,
     eliminate_by_largest,
     eliminate_by_longest,
     eliminate_by_smallest,
@@ -116,7 +125,7 @@ from .raster.sentinel import Sentinel2
 
 
 try:
-    from .io.dapla_functions import check_files, exists, read_geopandas, write_geopandas
+    from .io.dapla_functions import check_files, read_geopandas, write_geopandas
     from .io.write_municipality_data import write_municipality_data
 except ImportError:
     pass

{ssb_sgis-0.3.9 → ssb_sgis-0.3.11}/src/sgis/geopandas_tools/bounds.py

@@ -1,28 +1,178 @@
+import functools
 import numbers
-from typing import Any
 from collections.abc import Callable, Collection, Mapping
+from dataclasses import dataclass
+from typing import Any
 
 import geopandas as gpd
 import numpy as np
+import pandas as pd
 from geopandas import GeoDataFrame, GeoSeries
 from pandas.api.types import is_dict_like
 from shapely import Geometry, box, extract_unique_points
 from shapely.geometry import Polygon
 
+from ..parallel.parallel import Parallel
 from .conversion import to_gdf
 from .general import clean_clip, is_bbox_like
 
 
+@dataclass
+class Gridlooper:
+    """Run functions in a loop cellwise based on a grid.
+
+    Args:
+        gridsize: Size of the grid cells in units of the crs (meters, degrees).
+        mask: Geometry object to create a grid around.
+        gridbuffer: Units to buffer each gridcell by. For edge cases.
+            Defaults to 0.
+        clip: If True (default) geometries are clipped by the grid cells.
+            If False, all geometries that intersect will be selected in each iteration.
+        verbose: Whether to print progress. Defaults to False.
+        keep_geom_type: Whether to keep only the input geometry types after clipping.
+            Defaults to True.
+
+    Examples
+    --------
+
+    Get some points and some polygons.
+
+    >>> import sgis as sg
+    >>> points = sg.read_parquet_url("https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/points_oslo.parquet")
+    >>> points["idx"] = points.index
+    >>> buffered = sg.buff(points, 100)
+    >>> buffered
+        idx                                           geometry
+    0      0  POLYGON ((263222.700 6651184.900, 263222.651 6...
+    1      1  POLYGON ((272556.100 6653369.500, 272556.051 6...
+    2      2  POLYGON ((270182.300 6653032.700, 270182.251 6...
+    3      3  POLYGON ((259904.800 6650339.700, 259904.751 6...
+    4      4  POLYGON ((272976.200 6652889.100, 272976.151 6...
+    ..   ...                                                ...
+    995  995  POLYGON ((266901.700 6647844.500, 266901.651 6...
+    996  996  POLYGON ((261374.000 6653593.400, 261373.951 6...
+    997  997  POLYGON ((263642.900 6645427.000, 263642.851 6...
+    998  998  POLYGON ((269326.700 6650628.000, 269326.651 6...
+    999  999  POLYGON ((264670.300 6644239.500, 264670.251 6...
+
+    [1000 rows x 2 columns]
+
+    Instantiate a gridlooper.
+
+    >>> looper = sg.Gridlooper(gridsize=200, mask=buffered, parallelizer=sg.Parallel(1, backend="multiprocessing"))
+
+    Run the function clean_overlay in a gridloop.
+
+    >>> resultslist = looper.run(
+    ...     sg.clean_overlay,
+    ...     points,
+    ...     buffered,
+    ... )
+    >>> type(resultslist)
+    list
+
+    >>> results = pd.concat(resultslist, ignore_index=True)
+    >>> results
+        idx_1 idx_2                        geometry
+    0      220   220  POINT (254575.200 6661631.500)
+    1      735   735  POINT (256337.400 6649931.700)
+    2      575   575  POINT (256369.200 6650413.300)
+    3       39    39  POINT (256142.300 6650526.300)
+    4      235   235  POINT (256231.300 6650720.200)
+    ...    ...   ...                             ...
+    1481   711   795  POINT (272845.500 6655048.800)
+    1482   711   711  POINT (272845.500 6655048.800)
+    1483   757   757  POINT (273507.600 6652806.600)
+    1484   457   457  POINT (273524.400 6652979.900)
+    1485   284   284  POINT (273650.800 6653000.500)
+
+    [1486 rows x 3 columns]
+
+    """
+
+    gridsize: int
+    mask: GeoDataFrame | GeoSeries | Geometry
+    gridbuffer: int = 0
+    clip: bool = True
+    keep_geom_type: bool = True
+    verbose: bool = False
+    parallelizer: Parallel | None = None
+
+    def __post_init__(self):
+        if not isinstance(self.mask, GeoDataFrame):
+            self.mask = to_gdf(self.mask)
+
+    def run(self, func: Callable, *args, **kwargs):
+        intersects_mask = lambda df: df.index.isin(df.sjoin(self.mask).index)
+        grid: GeoSeries = (
+            make_grid(self.mask, gridsize=self.gridsize).loc[intersects_mask].geometry
+        )
+
+        n = len(grid)
+
+        buffered_grid = grid.buffer(self.gridbuffer, resolution=1, join_style=2)
+
+        if self.parallelizer is not None:
+            func_with_clip = functools.partial(
+                _clip_and_run_func,
+                func=func,
+                args=args,
+                kwargs=kwargs,
+                keep_geom_type=self.keep_geom_type,
+                clip=self.clip,
+            )
+            results = self.parallelizer.map(func_with_clip, buffered_grid)
+            if not self.gridbuffer or not self.clip:
+                return results
+            out = []
+            for cell_res, unbuffered in zip(results, grid, strict=True):
+                out.append(
+                    _clip_back_to_unbuffered_grid(
+                        cell_res, unbuffered, self.keep_geom_type
+                    )
+                )
+            return out
+
+        results = []
+        for i, (unbuffered, buffered) in enumerate(zip(grid, buffered_grid)):
+            cell_kwargs = {
+                key: _clip_if_isinstance(
+                    value, buffered, self.keep_geom_type, self.clip
+                )
+                for key, value in kwargs.items()
+            }
+            cell_args = tuple(
+                _clip_if_isinstance(value, buffered, self.keep_geom_type, self.clip)
+                for value in args
+            )
+
+            cell_res = func(*cell_args, **cell_kwargs)
+
+            # clip back to original
+            if self.gridbuffer and self.clip:
+                cell_res = _clip_back_to_unbuffered_grid(
+                    cell_res, unbuffered, self.keep_geom_type
+                )
+
+            results.append(cell_res)
+
+            if self.verbose:
+                print(f"Done with {i+1} of {n} grid cells", end="\r")
+
+        return results
+
+
 def gridloop(
     func: Callable,
-    mask: GeoDataFrame | GeoSeries | Geometry,
     gridsize: int,
+    mask: GeoDataFrame | GeoSeries | Geometry,
     gridbuffer: int = 0,
     clip: bool = True,
     keep_geom_type: bool = True,
     verbose: bool = False,
     args: tuple | None = None,
     kwargs: dict | None = None,
+    parallelizer: Parallel | None = None,
 ) -> list[Any]:
     """Runs a function in a loop cellwise based on a grid.
 
@@ -108,9 +258,6 @@ def gridloop(
     [1486 rows x 3 columns]
 
     """
-    if not isinstance(mask, GeoDataFrame):
-        mask = to_gdf(mask)
-
     if kwargs is None:
         kwargs = {}
     elif not isinstance(kwargs, dict):
@@ -121,49 +268,52 @@ def gridloop(
     elif not isinstance(args, tuple):
         raise TypeError("args should be a tuple")
 
+    if not isinstance(mask, GeoDataFrame):
+        mask = to_gdf(mask)
+
     intersects_mask = lambda df: df.index.isin(df.sjoin(mask).index)
     grid: GeoSeries = make_grid(mask, gridsize=gridsize).loc[intersects_mask].geometry
 
-    if verbose:
-        n = len(grid)
-
-    def clip_if_isinstance(value, cell, keep_geom_type):
-        if not isinstance(value, (gpd.GeoDataFrame, gpd.GeoSeries, Geometry)):
-            return value
-
-        if isinstance(value, (gpd.GeoDataFrame, gpd.GeoSeries)):
-            if clip:
-                return clean_clip(value, cell, keep_geom_type=keep_geom_type)
-            return value.loc[value.intersects(cell)]
+    n = len(grid)
 
-        return value.intersection(cell).make_valid()
+    buffered_grid = grid.buffer(gridbuffer, resolution=1, join_style=2)
 
-    buffered = grid.buffer(gridbuffer, resolution=1, join_style=2)
+    if parallelizer is not None:
+        func_with_clip = functools.partial(
+            _clip_and_run_func,
+            func=func,
+            args=args,
+            kwargs=kwargs,
+            keep_geom_type=keep_geom_type,
+            clip=clip,
+        )
+        results = parallelizer.map(func_with_clip, buffered_grid)
+        if not gridbuffer or not clip:
+            return results
+        out = []
+        for cell_res, unbuffered in zip(results, grid, strict=True):
+            out.append(
+                _clip_back_to_unbuffered_grid(cell_res, unbuffered, keep_geom_type)
+            )
+        return out
 
     results = []
-    for i, (cell, buffered) in enumerate(zip(grid, buffered)):
-        cell_kwargs = {}
-        for key, value in kwargs.items():
-            value = clip_if_isinstance(value, buffered, keep_geom_type)
-            cell_kwargs[key] = value
-
-        cell_args = ()
-        for arg in args:
-            arg = clip_if_isinstance(arg, buffered, keep_geom_type)
-            cell_args = cell_args + (arg,)
+    for i, (unbuffered, buffered) in enumerate(zip(grid, buffered_grid)):
+        cell_kwargs = {
+            key: _clip_if_isinstance(value, buffered, keep_geom_type, clip)
+            for key, value in kwargs.items()
+        }
+        cell_args = tuple(
+            _clip_if_isinstance(value, buffered, keep_geom_type, clip) for value in args
+        )
 
         cell_res = func(*cell_args, **cell_kwargs)
 
         # clip back to original
         if gridbuffer and clip:
-            if isinstance(cell_res, (gpd.GeoDataFrame, gpd.GeoSeries, Geometry)):
-                cell_res = clip_if_isinstance(cell_res, cell, keep_geom_type)
-            else:
-                try:
-                    for res in cell_res:
-                        res = clip_if_isinstance(res, cell, keep_geom_type)
-                except TypeError:
-                    pass
+            cell_res = _clip_back_to_unbuffered_grid(
+                cell_res, unbuffered, keep_geom_type
+            )
 
         results.append(cell_res)
 
@@ -173,6 +323,56 @@ def gridloop(
     return results
 
 
+def _clip_and_run_func(
+    grid_cell: Polygon,
+    func: Callable,
+    args: tuple,
+    kwargs: dict,
+    keep_geom_type: bool,
+    clip: bool,
+):
+    cell_args = tuple(
+        _clip_if_isinstance(value, grid_cell, keep_geom_type, clip) for value in args
+    )
+    cell_kwargs = {
+        key: _clip_if_isinstance(value, grid_cell, keep_geom_type, clip)
+        for key, value in kwargs.items()
+    }
+
+    return func(*cell_args, **cell_kwargs)
+
+
+def _clip_if_isinstance(value, cell, keep_geom_type, clip: bool):
+    if not isinstance(value, (gpd.GeoDataFrame, gpd.GeoSeries, Geometry)):
+        return value
+
+    if isinstance(value, (gpd.GeoDataFrame, gpd.GeoSeries)):
+        if clip:
+            return clean_clip(value, cell, keep_geom_type=keep_geom_type)
+        return value.loc[value.intersects(cell)]
+
+    return value.intersection(cell).make_valid()
+
+
+def _clip_back_to_unbuffered_grid(results, mask, keep_geom_type):
+    if isinstance(results, (gpd.GeoDataFrame, gpd.GeoSeries, Geometry)):
+        return _clip_if_isinstance(results, mask, keep_geom_type, clip=True)
+    elif isinstance(results, (pd.DataFrame, pd.Series, np.ndarray)):
+        return results
+    try:
+        for key, value in results.items():
+            results[key] = _clip_if_isinstance(value, mask, keep_geom_type, clip=True)
+    except AttributeError:
+        try:
+            return [
+                _clip_if_isinstance(res, mask, keep_geom_type, clip=True)
+                for res in results
+            ]
+        except TypeError:
+            pass
+    return results
+
+
 def make_grid_from_bbox(
     minx: int | float,
     miny: int | float,
@@ -506,7 +706,6 @@ def get_total_bounds(
         minx, miny, maxx, maxy = to_bbox(obj)
         xs += [minx, maxx]
         ys += [miny, maxy]
-
     return min(xs), min(ys), max(xs), max(ys)
 
 

{ssb_sgis-0.3.9 → ssb_sgis-0.3.11}/src/sgis/geopandas_tools/buffer_dissolve_explode.py

@@ -14,7 +14,9 @@ for the following:
 - The buff function returns a GeoDataFrame, the geopandas method returns a GeoSeries.
 """
 
+import numpy as np
 from geopandas import GeoDataFrame, GeoSeries
+from shapely import make_valid, unary_union
 
 from .general import _push_geom_col
 from .geometry_types import make_all_singlepart
@@ -46,6 +48,7 @@ def buffdissexp(
     resolution: int = 50,
     index_parts: bool = False,
     copy: bool = True,
+    grid_size: float | int | None = None,
     **dissolve_kwargs,
 ) -> GeoDataFrame:
     """Buffers and dissolves overlapping geometries.
@@ -75,6 +78,7 @@ def buffdissexp(
         distance,
         resolution=resolution,
         copy=copy,
+        grid_size=grid_size,
         **dissolve_kwargs,
     )
 
@@ -159,11 +163,43 @@ def buffdiss(
     """
     buffered = buff(gdf, distance, resolution=resolution, copy=copy)
 
-    dissolved = buffered.dissolve(**dissolve_kwargs)
+    return _dissolve(buffered, **dissolve_kwargs)
 
-    dissolved[gdf._geometry_column_name] = dissolved.make_valid()
 
-    return dissolved
+def _dissolve(gdf, aggfunc="first", grid_size=None, **dissolve_kwargs):
+    geom_col = gdf._geometry_column_name
+    if grid_size is None:
+        dissolved = gdf.dissolve(aggfunc=aggfunc, **dissolve_kwargs)
+
+        dissolved[geom_col] = dissolved.make_valid()
+        return dissolved
+
+    def merge_geometries(x):
+        return make_valid(unary_union(x, grid_size=grid_size))
+
+    geom_col = gdf._geometry_column_name
+
+    by = dissolve_kwargs.pop("by", None)
+
+    if by is None and dissolve_kwargs.get("level") is None:
+        by = np.zeros(len(gdf), dtype="int64")
+        other_cols = list(gdf.columns.difference({geom_col}))
+    else:
+        if isinstance(by, str):
+            by = [by]
+        other_cols = list(gdf.columns.difference({geom_col} | set(by)))
+
+    dissolved = gdf.groupby(by, **dissolve_kwargs)[other_cols].agg(aggfunc)
+    geoms_agged = gdf.groupby(by, **dissolve_kwargs)[geom_col].agg(merge_geometries)
+
+    if not dissolve_kwargs.get("as_index"):
+        try:
+            geoms_agged = geoms_agged[geom_col]
+        except KeyError:
+            pass
+    dissolved[geom_col] = geoms_agged
+
+    return GeoDataFrame(dissolved, geometry=geom_col, crs=gdf.crs)
 
 
 def dissexp(
@@ -172,6 +208,7 @@ def dissexp(
     aggfunc="first",
     as_index: bool = True,
     index_parts: bool = False,
+    grid_size: float | int | None = None,
     **dissolve_kwargs,
 ):
     """Dissolves overlapping geometries.
@@ -191,19 +228,14 @@ def dissexp(
     Returns:
         A GeoDataFrame where overlapping geometries are dissolved.
     """
-    geom_col = gdf._geometry_column_name
-
     dissolve_kwargs = dissolve_kwargs | {
         "by": by,
-        "aggfunc": aggfunc,
         "as_index": as_index,
     }
 
     dissolve_kwargs, ignore_index = _decide_ignore_index(dissolve_kwargs)
 
-    dissolved = gdf.dissolve(**dissolve_kwargs)
-
-    dissolved[geom_col] = dissolved.make_valid()
+    dissolved = _dissolve(gdf, aggfunc=aggfunc, grid_size=grid_size, **dissolve_kwargs)
 
     return make_all_singlepart(
         dissolved, ignore_index=ignore_index, index_parts=index_parts