ssb-sgis 1.2.0__tar.gz → 1.2.2__tar.gz

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 1.2.0
+Version: 1.2.2
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-sgis"
-version = "1.2.0"
+version = "1.2.2"
 description = "GIS functions used at Statistics Norway."
 authors = ["Morten Letnes <morten.letnes@ssb.no>"]
 license = "MIT"

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/__init__.py

@@ -42,6 +42,7 @@ from .geopandas_tools.general import make_lines_between_points
 from .geopandas_tools.general import points_in_bounds
 from .geopandas_tools.general import random_points
 from .geopandas_tools.general import random_points_in_polygons
+from .geopandas_tools.general import random_points_norway
 from .geopandas_tools.general import sort_large_first
 from .geopandas_tools.general import sort_long_first
 from .geopandas_tools.general import sort_short_first

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/conf.py

@@ -1,4 +1,3 @@
-from collections.abc import Iterable
 from typing import Any
 
 try:
@@ -74,51 +73,18 @@ from .geopandas_tools.runners import RTreeQueryRunner
 from .geopandas_tools.runners import UnionRunner
 
 
-class Config:
-    """Dictlike config with a 'get_instance' method."""
+def _get_instance(data: dict, key: str, **kwargs) -> Any:
+    """Get the dict value and call it if callable."""
+    x = data[key]
+    if callable(x):
+        return x(**kwargs)
+    return x
 
-    def __init__(self, data: dict) -> None:
-        """Initialise with dict."""
-        self.data = data
 
-    def get_instance(self, key: str, *args, **kwargs) -> Any:
-        """Get the dict value and call it if callable."""
-        x = self.data[key]
-        if callable(x):
-            return x(*args, **kwargs)
-        return x
-
-    def __getattr__(self, attr: str) -> Any:
-        """Get dict attribute."""
-        return getattr(self.data, attr)
-
-    def __getitem__(self, key: str) -> Any:
-        """Get dict value."""
-        return self.data[key]
-
-    def __setitem__(self, key: str, value) -> None:
-        """Set dict value."""
-        self.data[key] = value
-
-    def __iter__(self) -> Iterable[str]:
-        """Iterate over dict keys."""
-        return iter(self.data)
-
-    def __len__(self) -> int:
-        """Length of dict."""
-        return len(self.data)
-
-    def __str__(self) -> str:
-        """String representation of dict."""
-        return str(self.data)
-
-
-config = Config(
-    {
-        "n_jobs": 1,
-        "file_system": file_system,
-        "rtree_runner": RTreeQueryRunner,
-        "overlay_runner": OverlayRunner,
-        "union_runner": UnionRunner,
-    }
-)
+config = {
+    "n_jobs": 1,
+    "file_system": file_system,
+    "rtree_runner": RTreeQueryRunner,
+    "overlay_runner": OverlayRunner,
+    "union_runner": UnionRunner,
+}

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/buffer_dissolve_explode.py

@@ -23,6 +23,7 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import get_num_geometries
 
+from ..conf import _get_instance
 from ..conf import config
 from ..parallel.parallel import Parallel
 from .geometry_types import make_all_singlepart
@@ -197,7 +198,7 @@ def _dissolve(
         return gdf
 
     if union_runner is None:
-        union_runner = config.get_instance("union_runner", n_jobs)
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
 
     geom_col = gdf.geometry.name
     by = dissolve_kwargs.pop("by", None)

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/duplicates.py

@@ -8,6 +8,7 @@ from shapely import STRtree
 from shapely import difference
 from shapely import simplify
 
+from ..conf import _get_instance
 from ..conf import config
 from .general import _determine_geom_type_args
 from .general import _push_geom_col
@@ -106,11 +107,11 @@ def update_geometries(
         return gdf
 
     if rtree_runner is None:
-        rtree_runner = config.get_instance("rtree_runner", n_jobs)
+        rtree_runner = _get_instance(config, "rtree_runner", n_jobs=n_jobs)
     if union_runner is None:
-        union_runner = config.get_instance("union_runner", n_jobs)
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
     if overlay_runner is None:
-        overlay_runner = config.get_instance("overlay_runner", n_jobs)
+        overlay_runner = _get_instance(config, "overlay_runner", n_jobs=n_jobs)
 
     if geom_type == "polygon" or get_geom_type(gdf) == "polygon":
         gdf.geometry = gdf.buffer(0)

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/general.py

@@ -21,7 +21,6 @@ from shapely import get_coordinates
 from shapely import get_parts
 from shapely import linestrings
 from shapely import make_valid
-from shapely import points as shapely_points
 from shapely.geometry import LineString
 from shapely.geometry import MultiPoint
 from shapely.geometry import Point
@@ -34,6 +33,7 @@ from .geometry_types import get_geom_type
 from .geometry_types import make_all_singlepart
 from .geometry_types import to_single_geom_type
 from .neighbors import get_k_nearest_neighbors
+from .sfilter import sfilter
 from .sfilter import sfilter_split
 
 
@@ -416,7 +416,7 @@ def random_points(n: int, loc: float | int = 0.5) -> GeoDataFrame:
     """Creates a GeoDataFrame with n random points.
 
     Args:
-        n: Number of points/rows to create.
+        n: Number of points to create.
         loc: Mean ('centre') of the distribution.
 
     Returns:
@@ -459,47 +459,78 @@ def random_points(n: int, loc: float | int = 0.5) -> GeoDataFrame:
     9999  POINT (134.503 168.155)
     [10000 rows x 1 columns]
     """
-    if isinstance(n, (str, float)):
-        n = int(n)
-
     x = np.random.rand(n) * float(loc) * 2
     y = np.random.rand(n) * float(loc) * 2
+    return GeoDataFrame(shapely.points(x, y=y), columns=["geometry"])
 
-    return GeoDataFrame(
-        (Point(x, y) for x, y in zip(x, y, strict=True)), columns=["geometry"]
-    )
 
+def random_points_norway(size: int, *, seed: int | None = None) -> GeoDataFrame:
+    """Creates a GeoDataFrame with crs=25833 n random points aprox. within the borders of mainland Norway.
+
+    Args:
+        size: Number of points to create.
+        seed: Optional random seed.
 
-def random_points_in_polygons(gdf: GeoDataFrame, n: int, seed=None) -> GeoDataFrame:
+    Returns:
+        A GeoDataFrame of points with n rows.
+    """
+    return random_points_in_polygons(
+        [
+            shapely.wkt.loads(x)
+            for x in [
+                "POLYGON ((546192 7586393, 546191 7586393, 526598 7592425, 526597 7592425, 526596 7592425, 526595 7592426, 526594 7592426, 525831 7593004, 525830 7593005, 525327 7593495, 525326 7593496, 525326 7593497, 525325 7593498, 525325 7593499, 525324 7593500, 525192 7594183, 525192 7594184, 524157 7606517, 524157 7606518, 524157 7606519, 524157 7606520, 524157 7606521, 526235 7613535, 526236 7613536, 559423 7676952, 559424 7676953, 559511 7677088, 579978 7708379, 636963 7792940, 636963 7792941, 636964 7792942, 636965 7792943, 641013 7795664, 823514 7912323, 823515 7912323, 823516 7912323, 882519 7931958, 882520 7931959, 882521 7931959, 953896 7939985, 953897 7939985, 973544 7939988, 973545 7939988, 973546 7939988, 975510 7939467, 1051029 7913762, 1051030 7913762, 1055067 7912225, 1055068 7912224, 1056725 7911491, 1098379 7890321, 1098380 7890320, 1098381 7890320, 1099197 7889670, 1099198 7889669, 1099442 7889429, 1099443 7889429, 1099444 7889428, 1099444 7889427, 1099445 7889426, 1099445 7889425, 1099445 7889424, 1099446 7889423, 1114954 7799458, 1115106 7797736, 1115106 7797735, 1115106 7797734, 1115106 7797733, 1115106 7797732, 1115105 7797731, 1115105 7797730, 1114774 7797199, 1112876 7794451, 1057595 7720320, 1057112 7719702, 1057112 7719701, 1057111 7719701, 1057110 7719700, 1057109 7719699, 902599 7637176, 902598 7637176, 902597 7637175, 902596 7637175, 702394 7590633, 702393 7590633, 702392 7590633, 546193 7586393, 546192 7586393))",
+                "POLYGON ((60672 6448410, 60671 6448411, 57185 6448783, 39229 6451077, 39228 6451077, 39227 6451077, 27839 6454916, 27838 6454916, 27808 6454929, 27807 6454929, 8939 6465625, 8938 6465626, 7449 6466699, 7448 6466700, 6876 6467215, 6876 6467216, -31966 6512038, -31968 6512040, -32554 6512779, -32554 6512780, -40259 6524877, -42041 6527698, -42217 6528008, -42546 6528677, -42547 6528678, -77251 6614452, -77252 6614453, -77252 6614454, -77252 6614455, -77252 6614456, -77206 6615751, -77206 6615752, -65669 6811422, -65669 6811423, -65608 6812139, -65608 6812140, -65608 6812141, -50907 6879624, -50907 6879625, -50907 6879626, -50906 6879627, -50889 6879658, -50889 6879659, -16217 6934790, -16217 6934791, -16216 6934792, -2958 6949589, -2957 6949590, 55128 6995098, 144915 7064393, 144915 7064394, 144916 7064395, 144958 7064418, 144959 7064418, 144960 7064418, 144961 7064419, 144962 7064419, 144963 7064419, 150493 7064408, 150494 7064408, 150495 7064408, 150770 7064370, 150771 7064370, 150772 7064370, 188559 7048106, 188560 7048105, 188664 7048054, 188665 7048054, 188666 7048053, 357806 6914084, 357807 6914083, 357808 6914082, 357809 6914081, 357809 6914080, 357810 6914079, 357810 6914078, 359829 6906908, 386160 6804356, 386160 6804355, 386160 6804354, 386160 6804353, 386160 6804352, 386160 6804351, 368140 6699014, 368140 6699013, 363725 6675483, 363725 6675482, 361041 6665071, 361040 6665070, 361040 6665069, 308721 6537573, 308720 6537572, 307187 6534433, 307187 6534432, 307186 6534431, 307185 6534430, 307184 6534429, 307183 6534429, 307182 6534428, 303562 6532881, 300420 6531558, 99437 6459510, 99436 6459510, 67654 6449332, 65417 6448682, 65416 6448682, 65415 6448682, 60673 6448410, 60672 6448410))",
+                "POLYGON ((219870 6914350, 219869 6914350, 219868 6914351, 219867 6914351, 194827 6928565, 194826 6928566, 193100 6929790, 193099 6929790, 193098 6929791, 193098 6929792, 193097 6929793, 157353 7006877, 157353 7006878, 154402 7017846, 154402 7017847, 154392 7017923, 154392 7017924, 154392 7017925, 154392 7017926, 166616 7077346, 166617 7077347, 169164 7087256, 169165 7087257, 170277 7089848, 173146 7096147, 173147 7096148, 174684 7098179, 174685 7098180, 314514 7253805, 314515 7253805, 314515 7253806, 314516 7253806, 314517 7253807, 314518 7253807, 314519 7253808, 314520 7253808, 314521 7253808, 314522 7253808, 314523 7253808, 314524 7253808, 332374.8847495829 7250200.016409928, 327615 7280207, 327615 7280208, 327615 7280209, 327615 7280210, 328471 7285637, 364549 7480637, 364549 7480638, 367030 7488919, 367030 7488920, 367045 7488948, 367045 7488949, 367046 7488950, 419493 7560257, 472291 7626092, 506326 7665544, 506327 7665545, 506328 7665546, 541847 7692387, 541848 7692388, 541849 7692388, 541850 7692389, 541851 7692389, 541852 7692389, 545852 7692619, 546265 7692617, 546266 7692617, 546267 7692617, 546268 7692617, 546269 7692616, 546270 7692616, 546270 7692615, 546271 7692615, 546272 7692614, 623027 7613734, 623028 7613733, 623029 7613732, 627609 7605928, 627610 7605928, 627610 7605927, 627610 7605926, 627611 7605925, 627611 7605924, 630573 7568363, 630573 7568362, 630573 7568361, 630573 7568360, 630573 7568359, 628567 7562381, 621356 7542293, 621356 7542292, 468368 7221876.188770507, 468368 7221876, 459071 7119021, 459071 7119020, 459071 7119019, 459070 7119018, 459070 7119017, 454728 7109371, 451784 7102984, 449525 7098307, 357809 6914071, 357808 6914070, 357808 6914069, 357807 6914068, 357806 6914068, 357806 6914067, 357805 6914067, 357804 6914066, 353158 6912240, 353157 6912239, 353156 6912239, 351669 6911974, 351668 6911974, 351667 6911974, 219871 6914350, 219870 6914350))",
+            ]
+        ],
+        size=size,
+        crs=25833,
+        seed=seed,
+    ).sample(size)
+
+
+def random_points_in_polygons(
+    polygons: Geometry | GeoDataFrame | GeoSeries,
+    size: int,
+    *,
+    seed: int | None = None,
+    crs: Any = 25833,
+) -> GeoDataFrame:
     """Creates a GeoDataFrame with n random points within the geometries of 'gdf'.
 
     Args:
-        gdf: A GeoDataFrame.
-        n: Number of points/rows to create.
-        seed: Optional random seet.
+        polygons: A GeoDataFrame or GeoSeries of polygons. Or a single polygon.
+        size: Number of points to create.
+        seed: Optional random seed.
+        crs: Optional crs of the output GeoDataFrame if input is shapely.Geometry.
 
     Returns:
         A GeoDataFrame of points with n rows.
     """
-    all_points = []
-
+    if crs is None:
+        try:
+            crs = polygons.crs
+        except AttributeError:
+            pass
     rng = np.random.default_rng(seed)
-
-    for i, geom in enumerate(gdf.geometry):
-        minx, miny, maxx, maxy = geom.bounds
-
-        xs = rng.uniform(minx, maxx, size=n * 500)
-        ys = rng.uniform(miny, maxy, size=n * 500)
-
-        points = GeoSeries(shapely_points(xs, y=ys), index=[i] * len(xs))
-        all_points.append(points)
-
-    return (
-        pd.concat(all_points)
-        .loc[lambda x: x.intersects(gdf.geometry)]
-        .groupby(level=0)
-        .head(n)
-    )
+    polygons = to_gdf(polygons, crs=crs).geometry
+    bounds = polygons.bounds
+    minx = np.repeat(bounds["minx"].values, size)
+    maxx = np.repeat(bounds["maxx"].values, size)
+    miny = np.repeat(bounds["miny"].values, size)
+    maxy = np.repeat(bounds["maxy"].values, size)
+    index = np.repeat(np.arange(len(polygons)), size)
+    length = len(index)
+    out = []
+    while sum(len(df) for df in out) < size * len(polygons):
+        xs = rng.uniform(low=minx, high=maxx, size=length)
+        ys = rng.uniform(low=miny, high=maxy, size=length)
+        out.append(
+            GeoDataFrame(
+                shapely.points(xs, y=ys), index=index, columns=["geometry"], crs=crs
+            ).pipe(sfilter, polygons)
+        )
+    return pd.concat(out).groupby(level=0).sample(size, replace=True).sort_index()
 
 
 def polygons_to_lines(

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/neighbors.py

@@ -17,8 +17,11 @@ from pandas import MultiIndex
 from pandas import Series
 from sklearn.neighbors import NearestNeighbors
 
+from ..conf import _get_instance
+from ..conf import config
 from .conversion import coordinate_array
 from .geometry_types import get_geom_type
+from .runners import RTreeQueryRunner
 
 
 def get_neighbor_indices(
@@ -26,6 +29,8 @@ def get_neighbor_indices(
     neighbors: GeoDataFrame | GeoSeries,
     max_distance: int = 0,
     predicate: str = "intersects",
+    rtree_runner: RTreeQueryRunner | None = None,
+    n_jobs: int | None = None,
 ) -> Series:
     """Creates a pandas Series with the index of 'gdf' and values of 'neighbors'.
 
@@ -41,6 +46,9 @@ def get_neighbor_indices(
         predicate: Spatial predicate to use in sjoin. Defaults to "intersects", meaning
             the geometry itself and geometries within will be considered neighbors if
             they are part of the 'neighbors' GeoDataFrame.
+        rtree_runner: Optionally debug/manipulate the spatial indexing operations.
+            See the 'runners' module for example implementations.
+        n_jobs: Number of workers.
 
     Returns:
         A pandas Series with values of the intersecting 'neighbors' indices.
@@ -103,6 +111,9 @@ def get_neighbor_indices(
     if gdf.crs != neighbors.crs:
         raise ValueError(f"'crs' mismatch. Got {gdf.crs} and {neighbors.crs}")
 
+    if rtree_runner is None:
+        rtree_runner = _get_instance(config, "rtree_runner", n_jobs=n_jobs)
+
     if isinstance(neighbors, GeoSeries):
         neighbors = neighbors.to_frame()
     else:
@@ -119,11 +130,21 @@ def get_neighbor_indices(
 
     if predicate == "nearest":
         max_distance = None if max_distance == 0 else max_distance
-        joined = gdf.sjoin_nearest(neighbors, how="inner", max_distance=max_distance)
+        left, right = rtree_runner.run(
+            gdf.geometry.values,
+            neighbors.geometry.values,
+            method="query_nearest",
+            max_distance=max_distance,
+        )
     else:
-        joined = gdf.sjoin(neighbors, how="inner", predicate=predicate)
-
-    return joined.rename(columns={"index_right": "neighbor_index"})["neighbor_index"]
+        left, right = rtree_runner.run(
+            gdf.geometry.values, neighbors.geometry.values, predicate=predicate
+        )
+    index_mapper1 = {i: x for i, x in enumerate(gdf.index)}
+    left = np.array([index_mapper1[i] for i in left])
+    index_mapper2 = {i: x for i, x in enumerate(neighbors.index)}
+    right = np.array([index_mapper2[i] for i in right])
+    return Series(right, index=left, name="neighbor_index")
 
 
 def get_neighbor_dfs(

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/overlay.py

@@ -23,6 +23,7 @@ from shapely import is_empty
 from shapely import make_valid
 from shapely import union_all
 
+from ..conf import _get_instance
 from ..conf import config
 from .general import _determine_geom_type_args
 from .general import clean_geoms
@@ -109,11 +110,11 @@ def clean_overlay(
         raise ValueError(f"'crs' mismatch. Got {df1.crs} and {df2.crs}")
 
     if rtree_runner is None:
-        rtree_runner = config.get_instance("rtree_runner", n_jobs)
+        rtree_runner = _get_instance(config, "rtree_runner", n_jobs=n_jobs)
     if union_runner is None:
-        union_runner = config.get_instance("union_runner", n_jobs)
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
     if overlay_runner is None:
-        overlay_runner = config.get_instance("overlay_runner", n_jobs)
+        overlay_runner = _get_instance(config, "overlay_runner", n_jobs=n_jobs)
 
     crs = df1.crs
 

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/polygon_operations.py

@@ -26,6 +26,7 @@ from shapely.errors import GEOSException
 from shapely.geometry import LinearRing
 from shapely.ops import SplitOp
 
+from ..conf import _get_instance
 from ..conf import config
 from ..debug_config import _DEBUG_CONFIG
 from ..debug_config import _try_debug_print
@@ -776,9 +777,9 @@ def _eliminate(
         return gdf
 
     if union_runner is None:
-        union_runner = config.get_instance("union_runner", n_jobs)
+        union_runner = _get_instance(config, "union_runner", n_jobs=n_jobs)
     if overlay_runner is None:
-        overlay_runner = config.get_instance("overlay_runner", n_jobs)
+        overlay_runner = _get_instance(config, "overlay_runner", n_jobs=n_jobs)
 
     gdf["_range_idx_elim"] = range(len(gdf))
 

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/runners.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 from geopandas import GeoDataFrame
 from geopandas import GeoSeries
+from shapely import Geometry
 from shapely import STRtree
 from shapely import get_parts
 from shapely import make_valid
@@ -38,6 +39,12 @@ class AbstractRunner(ABC):
     def run(self, *args, **kwargs) -> Any:
         """Abstract run method."""
 
+    def __str__(self) -> str:
+        """String representation."""
+        return (
+            f"{self.__class__.__name__}(n_jobs={self.n_jobs}, backend='{self.backend}')"
+        )
+
 
 @dataclass
 class UnionRunner(AbstractRunner):
@@ -68,10 +75,10 @@ class UnionRunner(AbstractRunner):
         """Run groupby on geometries in parallel (if n_jobs > 1)."""
         # assume geometry column is 'geometry' if input is pandas.Series og pandas.DataFrame
         try:
-            geom_col = df.geometry.name
+            geom_col: str = df.geometry.name
         except AttributeError:
             try:
-                geom_col = df.name
+                geom_col: str | None = df.name
                 if geom_col is None:
                     geom_col = "geometry"
             except AttributeError:
@@ -90,10 +97,10 @@ class UnionRunner(AbstractRunner):
             by = np.zeros(len(df), dtype="int64")
 
         try:
-            # DataFrame
+            # (Geo)DataFrame
             groupby_obj = df.groupby(by, **kwargs)[geom_col]
         except KeyError:
-            # Series
+            # (Geo)Series
             groupby_obj = df.groupby(by, **kwargs)
 
         if self.n_jobs is None or self.n_jobs == 1:
@@ -113,9 +120,24 @@ class UnionRunner(AbstractRunner):
         return agged
 
 
-def _strtree_query(arr1, arr2, **kwargs):
+def _strtree_query(
+    arr1: np.ndarray,
+    arr2: np.ndarray,
+    method: str,
+    indices1: np.ndarray | None = None,
+    indices2: np.ndarray | None = None,
+    **kwargs,
+):
     tree = STRtree(arr2)
-    return tree.query(arr1, **kwargs)
+    func = getattr(tree, method)
+    left, right = func(arr1, **kwargs)
+    if indices1 is not None:
+        index_mapper1 = {i: x for i, x in enumerate(indices1)}
+        left = np.array([index_mapper1[i] for i in left])
+    if indices2 is not None:
+        index_mapper2 = {i: x for i, x in enumerate(indices2)}
+        right = np.array([index_mapper2[i] for i in right])
+    return left, right
 
 
 @dataclass
@@ -138,39 +160,52 @@ class RTreeQueryRunner(AbstractRunner):
     backend: str = "loky"
 
     def run(
-        self, arr1: np.ndarray, arr2: np.ndarray, **kwargs
+        self, arr1: np.ndarray, arr2: np.ndarray, method: str = "query", **kwargs
     ) -> tuple[np.ndarray, np.ndarray]:
         """Run a spatial rtree query and return indices of hits from arr1 and arr2 in a tuple of two arrays."""
-        # if (
-        #     self.n_jobs > 1
-        #     and len(arr1) / self.n_jobs > 1000
-        #     # and len(arr1) / len(arr2) > 3
-        # ):
-        #     chunks = np.array_split(np.arange(len(arr1)), self.n_jobs)
-        #     assert sum(len(x) for x in chunks) == len(arr1)
-        #     with joblib.Parallel(self.n_jobs, backend=self.backend) as parallel:
-        #         results = parallel(
-        #             joblib.delayed(_strtree_query)(arr1[chunk], arr2, **kwargs)
-        #             for chunk in chunks
-        #         )
-        #     left = np.concatenate([x[0] for x in results])
-        #     right = np.concatenate([x[1] for x in results])
-        #     return left, right
-        # elif (
-        #     self.n_jobs > 1
-        #     and len(arr2) / self.n_jobs > 1000
-        #     and len(arr2) / len(arr1) > 3
-        # ):
-        #     chunks = np.array_split(np.arange(len(arr2)), self.n_jobs)
-        #     with joblib.Parallel(self.n_jobs, backend=self.backend) as parallel:
-        #         results = parallel(
-        #             joblib.delayed(_strtree_query)(arr1, arr2[chunk], **kwargs)
-        #             for chunk in chunks
-        #         )
-        #     left = np.concatenate([x[0] for x in results])
-        #     right = np.concatenate([x[1] for x in results])
-        #     return left, right
-        return _strtree_query(arr1, arr2, **kwargs)
+        if (
+            (self.n_jobs or 1) > 1
+            and len(arr1) / self.n_jobs > 10_000
+            and len(arr1) / len(arr2)
+        ):
+            chunks = np.array_split(np.arange(len(arr1)), self.n_jobs)
+            assert sum(len(x) for x in chunks) == len(arr1)
+            with joblib.Parallel(self.n_jobs, backend=self.backend) as parallel:
+                results = parallel(
+                    joblib.delayed(_strtree_query)(
+                        arr1[chunk],
+                        arr2,
+                        method=method,
+                        indices1=chunk,
+                        **kwargs,
+                    )
+                    for chunk in chunks
+                )
+            left = np.concatenate([x[0] for x in results])
+            right = np.concatenate([x[1] for x in results])
+            return left, right
+        elif (
+            (self.n_jobs or 1) > 1
+            and len(arr2) / self.n_jobs > 10_000
+            and len(arr2) / len(arr1)
+        ):
+            chunks = np.array_split(np.arange(len(arr2)), self.n_jobs)
+            with joblib.Parallel(self.n_jobs, backend=self.backend) as parallel:
+                results = parallel(
+                    joblib.delayed(_strtree_query)(
+                        arr1,
+                        arr2[chunk],
+                        method=method,
+                        indices2=chunk,
+                        **kwargs,
+                    )
+                    for chunk in chunks
+                )
+            left = np.concatenate([x[0] for x in results])
+            right = np.concatenate([x[1] for x in results])
+            return left, right
+
+        return _strtree_query(arr1, arr2, method=method, **kwargs)
 
 
 @dataclass
@@ -189,8 +224,8 @@ class OverlayRunner(AbstractRunner):
     n_jobs: None = None
     backend: None = None
 
-    @staticmethod
     def run(
+        self,
         func: Callable,
         arr1: np.ndarray,
         arr2: np.ndarray,
@@ -219,7 +254,7 @@ class GridSizeOverlayRunner(OverlayRunner):
 
     n_jobs: int
     backend: str | None
-    grid_sizes: list[float] | None = None
+    grid_sizes: list[float | int] | None = None
 
     def __post_init__(self) -> None:
         """Check that grid_sizes is passed."""
@@ -247,7 +282,9 @@ class GridSizeOverlayRunner(OverlayRunner):
 
         """
         kwargs = dict(
-            grid_size=grid_size, geom_type=geom_type.lower(), grid_sizes=self.grid_sizes
+            grid_size=grid_size,
+            geom_type=geom_type.lower() if geom_type is not None else None,
+            grid_sizes=self.grid_sizes,
         )
         with joblib.Parallel(self.n_jobs, backend="threading") as parallel:
             return parallel(
@@ -256,15 +293,27 @@ class GridSizeOverlayRunner(OverlayRunner):
             )
 
 
-def _run_overlay_rowwise(func, geom1, geom2, grid_size, geom_type, grid_sizes):
+def _fix_gemetry_fast(geom: Geometry, geom_type: str | None) -> Geometry:
+    geom = make_valid(geom)
+    if geom.geom_type == geom_type or geom_type is None:
+        return geom
+    return union_all([g for g in get_parts(geom) if geom_type in g.geom_type])
+
+
+def _run_overlay_rowwise(
+    func: Callable,
+    geom1: Geometry,
+    geom2: Geometry,
+    grid_size: float | int | None,
+    geom_type: str | None,
+    grid_sizes: list[float | int],
+) -> Geometry:
     try:
         return func(geom1, geom2, grid_size=grid_size)
     except GEOSException:
         pass
-    geom1 = get_parts(make_valid(geom1))
-    geom2 = get_parts(make_valid(geom2))
-    geom1 = union_all([g for g in geom1 if pd.notna(g) and geom_type in g.geom_type])
-    geom2 = union_all([g for g in geom2 if pd.notna(g) and geom_type in g.geom_type])
+    geom1 = _fix_gemetry_fast(geom1, geom_type)
+    geom2 = _fix_gemetry_fast(geom2, geom_type)
     try:
         return func(geom1, geom2)
     except GEOSException:

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/geopandas_tools/sfilter.py

@@ -6,6 +6,8 @@ from geopandas import GeoDataFrame
 from geopandas import GeoSeries
 from shapely import Geometry
 
+from ..conf import _get_instance
+from ..conf import config
 from .conversion import to_gdf
 from .runners import RTreeQueryRunner
 
@@ -17,7 +19,7 @@ def sfilter(
     other: GeoDataFrame | GeoSeries | Geometry,
     predicate: str = "intersects",
     distance: int | float | None = None,
-    n_jobs: int = 1,
+    n_jobs: int | None = None,
     rtree_runner: RTreeQueryRunner | None = None,
 ) -> GeoDataFrame:
     """Filter a GeoDataFrame or GeoSeries by spatial predicate.
@@ -296,7 +298,7 @@ def _get_sfilter_indices(
     original_predicate = predicate
 
     if rtree_runner is None:
-        rtree_runner = RTreeQueryRunner(n_jobs)
+        rtree_runner = _get_instance(config, "rtree_runner", n_jobs=n_jobs)
 
     with warnings.catch_warnings():
         # We don't need to show our own warning here

{ssb_sgis-1.2.0 → ssb_sgis-1.2.2}/src/sgis/io/dapla_functions.py

@@ -110,7 +110,8 @@ def read_geopandas(
         try:
             expression = "".join(next(iter(filters))).replace("==", "=")
             glob_func = _get_glob_func(file_system)
-            paths = glob_func(str(Path(gcs_path) / expression))
+            suffix: str = Path(gcs_path).suffix
+            paths = glob_func(str(Path(gcs_path) / expression / f"*{suffix}"))
             if paths:
                 return _read_geopandas_from_iterable(
                     paths,
@@ -128,11 +129,11 @@ def read_geopandas(
         return gpd.GeoDataFrame(
             _read_partitioned_parquet(
                 gcs_path,
-                read_func=pq.read_table,
                 file_system=file_system,
                 mask=mask,
                 filters=filters,
                 child_paths=child_paths,
+                use_threads=use_threads,
                 **kwargs,
             )
         )
@@ -145,7 +146,7 @@ def read_geopandas(
         read_func = gpd.read_file
 
     with file_system.open(gcs_path, mode="rb") as file:
-        return _read_geopandas(
+        return _read_geopandas_single_path(
             file,
             read_func=read_func,
             file_format=file_format,
@@ -163,18 +164,10 @@ def _read_geopandas_from_iterable(
         paths = list(paths.index)
     elif mask is None:
         paths = list(paths)
-    else:
-        if not isinstance(paths, GeoSeries):
-            bounds_series: GeoSeries = get_bounds_series(
-                paths,
-                file_system,
-                use_threads=use_threads,
-                pandas_fallback=pandas_fallback,
-            )
-        else:
-            bounds_series = paths
-        new_bounds_series = sfilter(bounds_series, mask)
-        if not len(new_bounds_series):
+    elif isinstance(paths, GeoSeries):
+        bounds_series = sfilter(paths, mask)
+        if not len(bounds_series):
+            # return GeoDataFrame with correct columns
             if isinstance(kwargs.get("columns"), Iterable):
                 cols = {col: [] for col in kwargs["columns"]}
             else:
@@ -186,29 +179,14 @@ def _read_geopandas_from_iterable(
                         if file_system.isfile(path):
                             raise ArrowInvalid(e, path) from e
             return GeoDataFrame(cols | {"geometry": []})
-        paths = list(new_bounds_series.index)
+        paths = list(bounds_series.index)
 
-    # recursive read with threads
-    threads = (
-        min(len(paths), int(multiprocessing.cpu_count())) or 1 if use_threads else 1
+    results: list[pyarrow.Table] = _read_pyarrow_with_treads(
+        paths, file_system=file_system, mask=mask, use_threads=use_threads, **kwargs
     )
-    with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
-        dfs: list[GeoDataFrame] = parallel(
-            joblib.delayed(read_geopandas)(
-                x,
-                file_system=file_system,
-                pandas_fallback=pandas_fallback,
-                mask=mask,
-                use_threads=use_threads,
-                **kwargs,
-            )
-            for x in paths
-        )
-
-    if dfs:
-        df = pd.concat(dfs, ignore_index=True)
+    if results:
         try:
-            df = GeoDataFrame(df)
+            return _concat_pyarrow_to_geopandas(results, paths, file_system)
         except Exception as e:
             if not pandas_fallback:
                 print(e)
@@ -219,6 +197,49 @@ def _read_geopandas_from_iterable(
     return df
 
 
+def _read_pyarrow_with_treads(
+    paths: list[str | Path | os.PathLike], file_system, use_threads, mask, **kwargs
+) -> list[pyarrow.Table]:
+    read_partial = functools.partial(
+        _read_pyarrow, mask=mask, file_system=file_system, **kwargs
+    )
+    if not use_threads:
+        return [x for x in map(read_partial, paths) if x is not None]
+    with ThreadPoolExecutor() as executor:
+        return [x for x in executor.map(read_partial, paths) if x is not None]
+
+
+def intersects(file, mask, file_system) -> bool:
+    bbox, _ = _get_bounds_parquet_from_open_file(file, file_system)
+    return shapely.box(*bbox).intersects(to_shapely(mask))
+
+
+def _read_pyarrow(path: str, file_system, mask=None, **kwargs) -> pyarrow.Table | None:
+    try:
+        with file_system.open(path, "rb") as file:
+            if mask is not None and not intersects(file, mask, file_system):
+                return
+
+            # 'get' instead of 'pop' because dict is mutable
+            schema = kwargs.get("schema", pq.read_schema(file))
+            new_kwargs = {
+                key: value for key, value in kwargs.items() if key != "schema"
+            }
+
+            return pq.read_table(file, schema=schema, **new_kwargs)
+    except ArrowInvalid as e:
+        glob_func = _get_glob_func(file_system)
+        if not len(
+            {
+                x
+                for x in glob_func(str(Path(path) / "**"))
+                if not paths_are_equal(path, x)
+            }
+        ):
+            raise e
+        # allow not being able to read empty directories that are hard to delete in gcs
+
+
 def _get_bounds_parquet(
     path: str | Path, file_system: GCSFileSystem, pandas_fallback: bool = False
 ) -> tuple[list[float], dict] | tuple[None, None]:
@@ -662,10 +683,10 @@ def expression_match_path(expression: ds.Expression, path: str) -> bool:
     return bool(len(table))
 
 
-def _read_geopandas(
+def _read_geopandas_single_path(
     file,
-    read_func: Callable = gpd.read_parquet,
-    file_format: str = "parquet",
+    read_func: Callable,
+    file_format: str,
     **kwargs,
 ):
     try:
@@ -681,32 +702,29 @@ def _read_geopandas(
         raise e.__class__(f"{e.__class__.__name__}: {e} for {file}.") from e
 
 
-def _read_pandas(gcs_path: str, **kwargs):
+def _read_pandas(gcs_path: str, use_threads: bool = True, **kwargs):
     file_system = _get_file_system(None, kwargs)
 
     if not isinstance(gcs_path, (str | Path | os.PathLike)):
-        # recursive read with threads
-        threads = (
-            min(len(gcs_path), int(multiprocessing.cpu_count())) or 1
-            if kwargs.get("use_threads")
-            else 1
+        results: list[pyarrow.Table] = _read_pyarrow_with_treads(
+            gcs_path,
+            file_system=file_system,
+            mask=None,
+            use_threads=use_threads,
+            **kwargs,
         )
-        with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
-            return pd.concat(
-                parallel(
-                    joblib.delayed(_read_pandas)(x, file_system=file_system, **kwargs)
-                    for x in gcs_path
-                )
-            )
+        results = pyarrow.concat_tables(results, promote_options="permissive")
+        return results.to_pandas()
 
     child_paths = get_child_paths(gcs_path, file_system)
     if child_paths:
         return _read_partitioned_parquet(
             gcs_path,
-            read_func=pd.read_parquet,
             file_system=file_system,
             mask=None,
             child_paths=child_paths,
+            use_threads=use_threads,
+            to_geopandas=False,
             **kwargs,
         )
 
@@ -716,11 +734,12 @@ def _read_pandas(gcs_path: str, **kwargs):
 
 def _read_partitioned_parquet(
     path: str,
-    read_func: Callable,
     filters=None,
     file_system=None,
     mask=None,
     child_paths: list[str] | None = None,
+    use_threads: bool = True,
+    to_geopandas: bool = True,
     **kwargs,
 ):
     file_system = _get_file_system(file_system, kwargs)
@@ -731,60 +750,22 @@ def _read_partitioned_parquet(
 
     filters = _filters_to_expression(filters)
 
-    def intersects(file, mask) -> bool:
-        bbox, _ = _get_bounds_parquet_from_open_file(file, file_system)
-        return shapely.box(*bbox).intersects(to_shapely(mask))
-
-    def read(child_path: str) -> pyarrow.Table | None:
-        try:
-            with file_system.open(child_path, "rb") as file:
-                if mask is not None and not intersects(file, mask):
-                    return
-
-                # 'get' instead of 'pop' because dict is mutable
-                schema = kwargs.get("schema", pq.read_schema(file))
-                new_kwargs = {
-                    key: value for key, value in kwargs.items() if key != "schema"
-                }
-
-                return read_func(file, schema=schema, filters=filters, **new_kwargs)
-        except ArrowInvalid as e:
-            if not len(
-                {
-                    x
-                    for x in glob_func(str(Path(child_path) / "**"))
-                    if not paths_are_equal(child_path, x)
-                }
-            ):
-                raise e
-            # allow not being able to read hard-to-delete empty directories
+    results: list[pyarrow.Table] = _read_pyarrow_with_treads(
+        (
+            path
+            for path in child_paths
+            if filters is None or expression_match_path(filters, path)
+        ),
+        file_system=file_system,
+        mask=mask,
+        use_threads=use_threads,
+        **kwargs,
+    )
 
-    with ThreadPoolExecutor() as executor:
-        results = [
-            df
-            for df in (
-                executor.map(
-                    read,
-                    (
-                        path
-                        for path in child_paths
-                        if filters is None or expression_match_path(filters, path)
-                    ),
-                )
-            )
-            if df is not None
-        ]
-
-    if results:
-        if all(isinstance(x, DataFrame) for x in results):
-            return pd.concat(results)
-        else:
-            results = pyarrow.concat_tables(
-                results,
-                promote_options="permissive",
-            )
-            geo_metadata = _get_geo_metadata(next(iter(child_paths)), file_system)
-            return _arrow_to_geopandas(results, geo_metadata)
+    if results and to_geopandas:
+        return _concat_pyarrow_to_geopandas(results, child_paths, file_system)
+    elif results:
+        return pyarrow.concat_tables(results, promote_options="permissive").to_pandas()
 
     # add columns to empty DataFrame
     first_path = next(iter(child_paths + [path]))
@@ -794,6 +775,17 @@ def _read_partitioned_parquet(
     return df
 
 
+def _concat_pyarrow_to_geopandas(
+    results: list[pyarrow.Table], paths: list[str], file_system: Any
+):
+    results = pyarrow.concat_tables(
+        results,
+        promote_options="permissive",
+    )
+    geo_metadata = _get_geo_metadata(next(iter(paths)), file_system)
+    return _arrow_to_geopandas(results, geo_metadata)
+
+
 def paths_are_equal(path1: Path | str, path2: Path | str) -> bool:
     return Path(path1).parts == Path(path2).parts