ssb-sgis 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

sgis/geopandas_tools/bounds.py

@@ -507,7 +507,7 @@ def make_ssb_grid(
             to make sure all data is covered by the grid.
 
     Returns:
-        GeoDataFrame with grid geometries and a column 'SSBID'.
+        GeoDataFrame with grid geometries and a column 'ssb_rute_id'.
 
     Raises:
         ValueError: If the GeoDataFrame does not have 25833 as crs.
@@ -568,12 +568,12 @@ def make_ssb_grid(
     grid["nordc"] = (
         (np.floor((grid.geometry.centroid.y) / gridsize) * gridsize).apply(int)
     ).apply(str)
-    grid["SSBID"] = grid["ostc"] + grid["nordc"]
-    return grid[["SSBID", "geometry"]]
+    grid["ssb_rute_id"] = grid["ostc"] + grid["nordc"]
+    return grid[["ssb_rute_id", "geometry"]]
 
 
 def add_grid_id(
-    gdf: GeoDataFrame, gridsize: int, out_column: str = "SSBID"
+    gdf: GeoDataFrame, gridsize: int, out_column: str = "ssb_rute_id"
 ) -> GeoDataFrame:
     """Adds an SSB grid ID column to a GeoDataFrame of points.
 

sgis/helpers.py

@@ -28,12 +28,21 @@ def _get_file_system(
     file_system: None | AbstractFileSystem, kwargs: dict
 ) -> AbstractFileSystem:
     if (
-        file_system is not None and "filesystem" in kwargs or "file_system" in kwargs
+        file_system is not None and ("filesystem" in kwargs or "file_system" in kwargs)
     ) or ("filesystem" in kwargs and "file_system" in kwargs):
         raise ValueError("Cannot pass both filesystem and file_system.")
     file_system2 = kwargs.pop("file_system", None)
     file_system3 = kwargs.pop("filesystem", None)
-    return file_system or file_system2 or file_system3 or config["file_system"]()
+    return (
+        file_system
+        or file_system2
+        or file_system3
+        or (
+            config["file_system"]()
+            if callable(config["file_system"])
+            else config["file_system"]
+        )
+    )
 
 
 def get_numpy_func(text: str, error_message: str | None = None) -> Callable:

sgis/io/dapla_functions.py

@@ -14,6 +14,7 @@ from collections.abc import Iterable
 from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO
 from pathlib import Path
+from typing import Any
 
 import geopandas as gpd
 import joblib
@@ -25,6 +26,7 @@ import pyarrow.parquet as pq
 import shapely
 from geopandas import GeoDataFrame
 from geopandas import GeoSeries
+from geopandas.io.arrow import _arrow_to_geopandas
 from geopandas.io.arrow import _geopandas_to_arrow
 from pandas import DataFrame
 from pyarrow import ArrowInvalid
@@ -49,7 +51,7 @@ def read_geopandas(
     pandas_fallback: bool = False,
     file_system: GCSFileSystem | None = None,
     mask: GeoSeries | GeoDataFrame | shapely.Geometry | tuple | None = None,
-    threads: int | None = None,
+    use_threads: bool = True,
     filters: pyarrow.dataset.Expression | None = None,
     **kwargs,
 ) -> GeoDataFrame | DataFrame:
@@ -68,11 +70,11 @@ def read_geopandas(
             not be read with geopandas and the number of rows is more than 0. If True,
             the file will be read with pandas if geopandas fails.
         file_system: Optional file system.
-        mask: Optional geometry mask to keep only intersecting geometries.
-            If 'gcs_path' is an iterable of multiple paths, only the files
-            with a bbox that intersects the mask are read, then filtered by location.
-        threads: Number of threads to use if reading multiple files. Defaults to
-            the number of files to read or the number of available threads (if lower).
+        mask: If gcs_path is a partitioned parquet file or an interable of paths.
+            Only files with a bbox intersecting mask will be read.
+            Note that the data is not filtered on a row level. You should either
+            use clip or sfilter to filter the data after reading.
+        use_threads: Defaults to True.
         filters: To filter out data. Either a pyarrow.dataset.Expression, or a list in the
             structure [[(column, op, val), …],…] where op is [==, =, >, >=, <, <=, !=, in, not in].
             More details here: https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html
@@ -85,72 +87,38 @@ def read_geopandas(
     file_system = _get_file_system(file_system, kwargs)
 
     if not isinstance(gcs_path, (str | Path | os.PathLike)):
-        cols = {}
-        if mask is not None:
-            if not isinstance(gcs_path, GeoSeries):
-                bounds_series: GeoSeries = get_bounds_series(
-                    gcs_path,
-                    file_system,
-                    threads=threads,
-                    pandas_fallback=pandas_fallback,
-                )
-            else:
-                bounds_series = gcs_path
-            new_bounds_series = sfilter(bounds_series, mask)
-            if not len(new_bounds_series):
-                if isinstance(kwargs.get("columns"), Iterable):
-                    cols = {col: [] for col in kwargs["columns"]}
-                else:
-                    cols = {}
-                    for path in bounds_series.index:
-                        try:
-                            cols |= {col: [] for col in _get_columns(path, file_system)}
-                        except ArrowInvalid as e:
-                            if file_system.isfile(path):
-                                raise ArrowInvalid(e, path) from e
-
-                return GeoDataFrame(cols | {"geometry": []})
-            paths = list(new_bounds_series.index)
-        else:
-            if isinstance(gcs_path, GeoSeries):
-                paths = list(gcs_path.index)
-            else:
-                paths = list(gcs_path)
-
-        if threads is None:
-            threads = min(len(paths), int(multiprocessing.cpu_count())) or 1
+        return _read_geopandas_from_iterable(
+            gcs_path,
+            mask=mask,
+            file_system=file_system,
+            use_threads=use_threads,
+            pandas_fallback=pandas_fallback,
+            **kwargs,
+        )
 
-        # recursive read with threads
-        with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
-            dfs: list[GeoDataFrame] = parallel(
-                joblib.delayed(read_geopandas)(
-                    x,
-                    filters=filters,
+    if (
+        isinstance(filters, Iterable)
+        and len(filters) == 1
+        and ("=" in next(iter(filters)) or "==" in next(iter(filters)))
+    ):
+        # try to read only files in the relevant partition, because glob is slow without GCSFileSystem
+        try:
+            expression = "".join(next(iter(filters))).replace("==", "=")
+            glob_func = _get_glob(file_system)
+            paths = glob_func(str(Path(gcs_path) / expression))
+            if paths:
+                return _read_geopandas_from_iterable(
+                    paths,
+                    mask=mask,
                     file_system=file_system,
+                    use_threads=use_threads,
                     pandas_fallback=pandas_fallback,
-                    mask=mask,
-                    threads=threads,
                     **kwargs,
                 )
-                for x in paths
-            )
-
-        if dfs:
-            df = pd.concat(dfs, ignore_index=True)
-            try:
-                df = GeoDataFrame(df)
-            except Exception as e:
-                if not pandas_fallback:
-                    print(e)
-                    raise e
-        else:
-            df = GeoDataFrame(cols | {"geometry": []})
-
-        if mask is not None:
-            return sfilter(df, mask)
-        return df
+        except FileNotFoundError:
+            pass
 
-    child_paths = has_partitions(gcs_path, file_system)
+    child_paths = get_child_paths(gcs_path, file_system)
     if child_paths:
         return gpd.GeoDataFrame(
             _read_partitioned_parquet(
@@ -158,65 +126,93 @@ def read_geopandas(
                 read_func=_read_geopandas,
                 file_system=file_system,
                 mask=mask,
-                pandas_fallback=pandas_fallback,
                 filters=filters,
                 child_paths=child_paths,
                 **kwargs,
             )
         )
 
-    if "parquet" in gcs_path or "prqt" in gcs_path:
-        with file_system.open(gcs_path, mode="rb") as file:
-            try:
-                df = gpd.read_parquet(
-                    file, filters=filters, filesystem=file_system, **kwargs
-                )
-            except ValueError as e:
-                if "Missing geo metadata" not in str(e) and "geometry" not in str(e):
-                    raise e.__class__(
-                        f"{e.__class__.__name__}: {e} for {gcs_path}."
-                    ) from e
-                df = pd.read_parquet(
-                    file, filters=filters, filesystem=file_system, **kwargs
-                )
-                if pandas_fallback or not len(df):
-                    return df
-                else:
-                    more_txt = PANDAS_FALLBACK_INFO if not len(df) else ""
-                    raise e.__class__(
-                        f"{e.__class__.__name__}: {e} for {df}." + more_txt
-                    ) from e
-            except Exception as e:
-                raise e.__class__(f"{e.__class__.__name__}: {e} for {gcs_path}.") from e
+    if gcs_path.endswith(".parquet"):
+        file_format: str = "parquet"
+        read_func = gpd.read_parquet
+    else:
+        file_format: str = Path(gcs_path).suffix.lstrip(".")
+        read_func = gpd.read_file
+
+    with file_system.open(gcs_path, mode="rb") as file:
+        df = _read_geopandas(
+            file,
+            read_func=read_func,
+            file_format=file_format,
+            filters=filters,
+            **kwargs,
+        )
+
+    return df
+
 
+def _read_geopandas_from_iterable(
+    paths, mask, file_system, use_threads, pandas_fallback, **kwargs
+):
+    cols = {}
+    if mask is None and isinstance(paths, GeoSeries):
+        # bounds GeoSeries indexed with file paths
+        paths = list(paths.index)
+    elif mask is None:
+        paths = list(paths)
     else:
-        with file_system.open(gcs_path, mode="rb") as file:
-            try:
-                df = gpd.read_file(
-                    file, filters=filters, filesystem=file_system, **kwargs
-                )
-            except ValueError as e:
-                if "Missing geo metadata" not in str(e) and "geometry" not in str(e):
-                    raise e
-                file_type: str = Path(gcs_path).suffix.strip(".")
-                df = getattr(pd, f"read_{file_type}")(
-                    file, filters=filters, filesystem=file_system, **kwargs
-                )
+        if not isinstance(paths, GeoSeries):
+            bounds_series: GeoSeries = get_bounds_series(
+                paths,
+                file_system,
+                use_threads=use_threads,
+                pandas_fallback=pandas_fallback,
+            )
+        else:
+            bounds_series = paths
+        new_bounds_series = sfilter(bounds_series, mask)
+        if not len(new_bounds_series):
+            if isinstance(kwargs.get("columns"), Iterable):
+                cols = {col: [] for col in kwargs["columns"]}
+            else:
+                cols = {}
+                for path in bounds_series.index:
+                    try:
+                        cols |= {col: [] for col in _get_columns(path, file_system)}
+                    except ArrowInvalid as e:
+                        if file_system.isfile(path):
+                            raise ArrowInvalid(e, path) from e
+            return GeoDataFrame(cols | {"geometry": []})
+        paths = list(new_bounds_series.index)
+
+    # recursive read with threads
+    threads = (
+        min(len(paths), int(multiprocessing.cpu_count())) or 1 if use_threads else 1
+    )
+    with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
+        dfs: list[GeoDataFrame] = parallel(
+            joblib.delayed(read_geopandas)(
+                x,
+                file_system=file_system,
+                pandas_fallback=pandas_fallback,
+                mask=mask,
+                use_threads=use_threads,
+                **kwargs,
+            )
+            for x in paths
+        )
+
+    if dfs:
+        df = pd.concat(dfs, ignore_index=True)
+        try:
+            df = GeoDataFrame(df)
+        except Exception as e:
+            if not pandas_fallback:
+                print(e)
+                raise e
+    else:
+        df = GeoDataFrame(cols | {"geometry": []})
 
-                if pandas_fallback or not len(df):
-                    return df
-                else:
-                    more_txt = PANDAS_FALLBACK_INFO if not len(df) else ""
-                    raise e.__class__(
-                        f"{e.__class__.__name__}: {e} for {df}. " + more_txt
-                    ) from e
-            except Exception as e:
-                raise e.__class__(
-                    f"{e.__class__.__name__}: {e} for {gcs_path}." + more_txt
-                ) from e
-
-    if mask is not None:
-        return sfilter(df, mask)
     return df
 
 
@@ -230,15 +226,25 @@ def _get_bounds_parquet(
 def _get_bounds_parquet_from_open_file(
     file, file_system
 ) -> tuple[list[float], dict] | tuple[None, None]:
-    geo_metadata = _get_geo_metadata(file, file_system)
+    geo_metadata = _get_geo_metadata_primary_column(file, file_system)
+
     if not geo_metadata:
         return None, None
     return geo_metadata["bbox"], geo_metadata["crs"]
 
 
 def _get_geo_metadata(file, file_system) -> dict:
-    meta = pq.read_schema(file).metadata
-    geo_metadata = json.loads(meta[b"geo"])
+    try:
+        meta = pq.read_schema(file).metadata
+    except FileNotFoundError:
+        with file_system.open(file, "rb") as f:
+            meta = pq.read_schema(f).metadata
+
+    return json.loads(meta[b"geo"])
+
+
+def _get_geo_metadata_primary_column(file, file_system) -> dict:
+    geo_metadata = _get_geo_metadata(file, file_system)
     try:
         primary_column = geo_metadata["primary_column"]
     except KeyError as e:
@@ -252,6 +258,7 @@ def _get_geo_metadata(file, file_system) -> dict:
             if not file_system.isfile(file):
                 return {}
             raise ArrowInvalid(e, file) from e
+        # allow for 0 lengthed tables not to have geo metadata
         if not num_rows:
             return {}
     return {}
@@ -272,7 +279,7 @@ def _get_index_cols(schema: pyarrow.Schema) -> list[str]:
 def get_bounds_series(
     paths: list[str | Path] | tuple[str | Path],
     file_system: GCSFileSystem | None = None,
-    threads: int | None = None,
+    use_threads: bool = True,
     pandas_fallback: bool = False,
 ) -> GeoSeries:
     """Get a GeoSeries with file paths as indexes and the file's bounds as values.
@@ -285,8 +292,7 @@ def get_bounds_series(
         file_system: Optional instance of GCSFileSystem.
             If None, an instance is created within the function.
             Note that this is slower in long loops.
-        threads: Number of threads to use if reading multiple files. Defaults to
-            the number of files to read or the number of available threads (if lower).
+        use_threads: Default True.
         pandas_fallback: If False (default), an exception is raised if the file has
             no geo metadata. If True, the geometry value is set to None for this file.
 
@@ -330,8 +336,9 @@ def get_bounds_series(
     """
     file_system = _get_file_system(file_system, {})
 
-    if threads is None:
-        threads = min(len(paths), int(multiprocessing.cpu_count())) or 1
+    threads = (
+        min(len(paths), int(multiprocessing.cpu_count())) or 1 if use_threads else 1
+    )
 
     with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
         bounds: list[tuple[list[float], dict]] = parallel(
@@ -396,9 +403,11 @@ def write_geopandas(
         raise ValueError("File already exists.")
 
     if not isinstance(df, GeoDataFrame):
-        raise ValueError(f"DataFrame must be GeoDataFrame. Got {type(df)}.")
+        raise ValueError(
+            f"DataFrame must be GeoDataFrame. Got {type(df)} for {gcs_path}."
+        )
 
-    if not len(df) and has_partitions(gcs_path, file_system):
+    if not len(df) and get_child_paths(gcs_path, file_system):
         # no need to write empty df
         return
     elif not len(df):
@@ -473,6 +482,23 @@ def _to_geopandas(df, path, **kwargs) -> None:
     pq.write_table(table, path, compression="snappy", **kwargs)
 
 
+def _pyarrow_schema_from_geopandas(df: GeoDataFrame) -> pyarrow.Schema:
+    geom_name = df.geometry.name
+    pandas_columns = [col for col in df if col != geom_name]
+    schema = pyarrow.Schema.from_pandas(df[pandas_columns], preserve_index=True)
+    index_columns = _get_index_cols(schema)
+    return pyarrow.schema(
+        [
+            (
+                (schema.field(col).name, schema.field(col).type)
+                if col != geom_name
+                else (geom_name, pyarrow.binary())
+            )
+            for col in [*df.columns, *index_columns]
+        ]
+    )
+
+
 def _remove_file(path, file_system) -> None:
     try:
         file_system.rm_file(str(path))
@@ -494,38 +520,22 @@ def _write_partitioned_geoparquet(
     file_system=None,
     write_func: Callable = _to_geopandas,
     existing_data_behavior: str = "error",
+    basename_template: str | None = None,
     **kwargs,
 ):
-    if isinstance(partition_cols, str):
-        partition_cols = [partition_cols]
-
     file_system = _get_file_system(file_system, kwargs)
 
-    path = Path(path)
-    unique_id = uuid.uuid4()
+    if basename_template is None:
+        basename_template = uuid.uuid4().hex + "-{i}.parquet"
+
+    if isinstance(partition_cols, str):
+        partition_cols = [partition_cols]
 
     for col in partition_cols:
         if df[col].isna().all() and not kwargs.get("schema"):
             raise ValueError("Must specify 'schema' when all rows are NA.")
 
-    try:
-        glob_func = functools.partial(file_system.glob, detail=False)
-    except AttributeError:
-        glob_func = functools.partial(glob.glob, recursive=True)
-
-    args: list[tuple[Path, DataFrame]] = []
-    dirs: list[Path] = set()
-    for group, rows in df.groupby(partition_cols, dropna=False):
-        name = (
-            "/".join(
-                f"{col}={value if not pd.isna(value) else NULL_VALUE}"
-                for col, value in zip(partition_cols, group, strict=True)
-            )
-            + f"/{unique_id}.parquet"
-        )
-
-        dirs.add((path / name).parent)
-        args.append((path / name, rows))
+    glob_func = _get_glob(file_system)
 
     if file_system.exists(path) and file_system.isfile(path):
         _remove_file(path, file_system)
@@ -533,49 +543,56 @@ def _write_partitioned_geoparquet(
     if kwargs.get("schema"):
         schema = kwargs.pop("schema")
     elif isinstance(df, GeoDataFrame):
-        geom_name = df.geometry.name
-        pandas_columns = [col for col in df if col != geom_name]
-        schema = pyarrow.Schema.from_pandas(df[pandas_columns], preserve_index=True)
-        index_columns = _get_index_cols(schema)
-        schema = pyarrow.schema(
-            [
-                (
-                    (schema.field(col).name, schema.field(col).type)
-                    if col != geom_name
-                    else (geom_name, pyarrow.binary())
-                )
-                for col in [*df.columns, *index_columns]
-                # for col in df.columns
-            ]
-        )
+        schema = _pyarrow_schema_from_geopandas(df)
     else:
         schema = pyarrow.Schema.from_pandas(df, preserve_index=True)
 
-    def get_siblings(path: str, paths: list[str]) -> list[str]:
-        parts = path.parts
-        return {x for x in paths if all(part in parts for part in x.parts)}
-
-    def threaded_write(path_rows):
-        new_path, rows = path_rows
-        # for sibling_path in get_siblings(new_path, child_paths):
-        for sibling_path in glob_func(str(Path(new_path).with_name("**"))):
-            if not paths_are_equal(sibling_path, Path(new_path).parent):
-                if existing_data_behavior == "delete_matching":
-                    _remove_file(sibling_path, file_system)
-                elif existing_data_behavior == "error":
-                    raise pyarrow.ArrowInvalid(
-                        f"Could not write to  {path} as the directory is not empty and existing_data_behavior is to error"
-                    )
+    def as_partition_part(col: str, value: Any) -> str:
+        value = value if not pd.isna(value) else NULL_VALUE
+        return f"{col}={value}"
+
+    paths: list[Path] = []
+    dfs: list[DataFrame] = []
+    for group, rows in df.groupby(partition_cols, dropna=False):
+        partition_parts = "/".join(
+            as_partition_part(col, value)
+            for col, value in zip(partition_cols, group, strict=True)
+        )
+        paths.append(Path(path) / partition_parts)
+        dfs.append(rows)
+
+    def threaded_write(rows: DataFrame, path: str) -> None:
+        this_basename = basename_template.replace("-{i}", "0")
+        for i, sibling_path in enumerate(sorted(glob_func(str(Path(path) / "**")))):
+            if paths_are_equal(sibling_path, path):
+                continue
+            if existing_data_behavior == "delete_matching":
+                _remove_file(sibling_path, file_system)
+            elif existing_data_behavior == "error":
+                raise pyarrow.ArrowInvalid(
+                    f"Could not write to {path} as the directory is not empty and existing_data_behavior is to error"
+                )
+            else:
+                this_basename = basename_template.replace("-{i}", str(i + 1))
+
+        out_path = str(Path(path) / this_basename)
         try:
-            with file_system.open(new_path, mode="wb") as file:
+            with file_system.open(out_path, mode="wb") as file:
                 write_func(rows, file, schema=schema, **kwargs)
         except FileNotFoundError:
-            file_system.makedirs(str(Path(new_path).parent), exist_ok=True)
-            with file_system.open(new_path, mode="wb") as file:
+            file_system.makedirs(str(path), exist_ok=True)
+            with file_system.open(out_path, mode="wb") as file:
                 write_func(rows, file, schema=schema, **kwargs)
 
     with ThreadPoolExecutor() as executor:
-        list(executor.map(threaded_write, args))
+        executor.map(threaded_write, dfs, paths)
+
+
+def _get_glob(file_system) -> functools.partial:
+    try:
+        return functools.partial(file_system.glob)
+    except AttributeError:
+        return functools.partial(glob.glob, recursive=True)
 
 
 def _filters_to_expression(filters) -> list[ds.Expression]:
@@ -612,6 +629,8 @@ def expression_match_path(expression: ds.Expression, path: str) -> bool:
     >>> expression_match_path(path, expression)
     False
     """
+    # keep only the parts in between the two .parquet parts
+    path = str(path).split(".parquet")[1]
     if NULL_VALUE in path:
         return True
     # build a one lengthed pyarrow.Table of the partitioning in the file path
@@ -627,39 +646,60 @@ def expression_match_path(expression: ds.Expression, path: str) -> bool:
     try:
         table = table.filter(expression)
     except pyarrow.ArrowInvalid as e:
-        if "No match for FieldRef" not in str(e):
-            raise e
-        # cannot determine if the expression match without reading the file
-        return True
+        if "No match for FieldRef" in str(e):
+            # if a non-partition col is used in 'filters',
+            # we cannot determine if the expression match without reading the file
+            return True
+        raise e
     return bool(len(table))
 
 
-def _read_geopandas(file, pandas_fallback: bool, **kwargs):
+def _read_geopandas(
+    file,
+    read_func: Callable = gpd.read_parquet,
+    file_format: str = "parquet",
+    **kwargs,
+):
     try:
-        return gpd.read_parquet(file, **kwargs)
-    except Exception as e:
-        if not pandas_fallback:
-            raise e
-        df = pd.read_parquet(file, **kwargs)
-        if len(df):
+        return read_func(file, **kwargs)
+    except ValueError as e:
+        if "Missing geo metadata" not in str(e) and "geometry" not in str(e):
             raise e
-        return df
+        df = getattr(pd, f"read_{file_format}")(file, **kwargs)
+        if not len(df):
+            return GeoDataFrame(df)
+        raise e.__class__(f"{e.__class__.__name__}: {e} for {df}. ") from e
+    except Exception as e:
+        raise e.__class__(f"{e.__class__.__name__}: {e} for {file}.") from e
 
 
 def _read_pandas(gcs_path: str, **kwargs):
     file_system = _get_file_system(None, kwargs)
 
-    child_paths = has_partitions(gcs_path, file_system)
-    if child_paths:
-        return gpd.GeoDataFrame(
-            _read_partitioned_parquet(
-                gcs_path,
-                read_func=pd.read_parquet,
-                file_system=file_system,
-                mask=None,
-                child_paths=child_paths,
-                **kwargs,
+    if not isinstance(gcs_path, (str | Path | os.PathLike)):
+        # recursive read with threads
+        threads = (
+            min(len(gcs_path), int(multiprocessing.cpu_count())) or 1
+            if kwargs.get("use_threads")
+            else 1
+        )
+        with joblib.Parallel(n_jobs=threads, backend="threading") as parallel:
+            return pd.concat(
+                parallel(
+                    joblib.delayed(_read_pandas)(x, file_system=file_system, **kwargs)
+                    for x in gcs_path
+                )
             )
+
+    child_paths = get_child_paths(gcs_path, file_system)
+    if child_paths:
+        return _read_partitioned_parquet(
+            gcs_path,
+            read_func=pd.read_parquet,
+            file_system=file_system,
+            mask=None,
+            child_paths=child_paths,
+            **kwargs,
         )
 
     with file_system.open(gcs_path, "rb") as file:
@@ -678,10 +718,7 @@ def _read_partitioned_parquet(
     file_system = _get_file_system(file_system, kwargs)
 
     if child_paths is None:
-        try:
-            glob_func = functools.partial(file_system.glob)
-        except AttributeError:
-            glob_func = functools.partial(glob.glob, recursive=True)
+        glob_func = _get_glob(file_system)
         child_paths = list(glob_func(str(Path(path) / "**/*.parquet")))
 
     filters = _filters_to_expression(filters)
@@ -690,13 +727,13 @@ def _read_partitioned_parquet(
         bbox, _ = _get_bounds_parquet_from_open_file(file, file_system)
         return shapely.box(*bbox).intersects(to_shapely(mask))
 
-    def read(path) -> GeoDataFrame | None:
+    def read(path: str) -> pyarrow.Table | None:
         with file_system.open(path, "rb") as file:
             if mask is not None and not intersects(file, mask):
                 return
 
+            # 'get' instead of 'pop' because dict is mutable
             schema = kwargs.get("schema", pq.read_schema(file))
-            # copy kwargs because mutable
             new_kwargs = {
                 key: value for key, value in kwargs.items() if key != "schema"
             }
@@ -705,8 +742,8 @@ def _read_partitioned_parquet(
 
     with ThreadPoolExecutor() as executor:
         results = [
-            x
-            for x in (
+            df
+            for df in (
                 executor.map(
                     read,
                     (
@@ -716,30 +753,34 @@ def _read_partitioned_parquet(
                     ),
                 )
             )
-            if x is not None
+            if df is not None
         ]
+
     if results:
-        if mask is not None:
-            return sfilter(pd.concat(results), mask)
-        return pd.concat(results)
+        if all(isinstance(x, DataFrame) for x in results):
+            results = pd.concat(results)
+        else:
+            geo_metadata = _get_geo_metadata(next(iter(child_paths)), file_system)
+            results = _arrow_to_geopandas(
+                pyarrow.concat_tables(
+                    results,
+                    promote_options="permissive",
+                ),
+                geo_metadata,
+            )
+        return results
 
     # add columns to empty DataFrame
     first_path = next(iter(child_paths + [path]))
-    return pd.DataFrame(
-        columns=list(dict.fromkeys(_get_columns(first_path, file_system)))
-    )
+    return pd.DataFrame(columns=_get_columns(first_path, file_system))
 
 
 def paths_are_equal(path1: Path | str, path2: Path | str) -> bool:
     return Path(path1).parts == Path(path2).parts
 
 
-def has_partitions(path, file_system) -> list[str]:
-    try:
-        glob_func = functools.partial(file_system.glob, detail=False)
-    except AttributeError:
-        glob_func = functools.partial(glob.glob, recursive=True)
-
+def get_child_paths(path, file_system) -> list[str]:
+    glob_func = _get_glob(file_system)
     return [
         x
         for x in glob_func(str(Path(path) / "**/*.parquet"))

sgis/maps/wms.py

@@ -15,6 +15,8 @@ from ..geopandas_tools.conversion import to_shapely
 
 JSON_PATH = Path(__file__).parent / "norge_i_bilder.json"
 
+JSON_YEARS = [str(year) for year in range(1999, 2025)]
+
 DEFAULT_YEARS: tuple[str] = tuple(
     str(year)
     for year in range(
@@ -168,7 +170,7 @@ class NorgeIBilderWms(WmsLoader):
 
         self.years = [str(int(year)) for year in self.years]
 
-        if all(year in DEFAULT_YEARS for year in self.years):
+        if all(year in JSON_YEARS for year in self.years):
             try:
                 with open(JSON_PATH, encoding="utf-8") as file:
                     self.tiles = json.load(file)
@@ -185,3 +187,7 @@ class NorgeIBilderWms(WmsLoader):
             ]
         else:
             self.tiles = None
+
+    def __repr__(self) -> str:
+        """Print representation."""
+        return f"{self.__class__.__name__}({len(self.tiles or [])})"

{ssb_sgis-1.1.2.dist-info → ssb_sgis-1.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 1.1.2
+Version: 1.1.4
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT

{ssb_sgis-1.1.2.dist-info → ssb_sgis-1.1.4.dist-info}/RECORD

@@ -3,7 +3,7 @@ sgis/conf.py,sha256=5GYeg00-qXtV_roskBaihZ2s5Ut3oF-cxjICo9UFJU0,2134
 sgis/debug_config.py,sha256=Tfr19kU46hSkkspsIJcrUWvlhaL4U3-f8xEPkujSCAQ,593
 sgis/exceptions.py,sha256=WNaEBPNNx0rmz-YDzlFX4vIE7ocJQruUTqS2RNAu2zU,660
 sgis/geopandas_tools/__init__.py,sha256=bo8lFMcltOz7TtWAi52_ekR2gd3mjfBfKeMDV5zuqFY,28
-sgis/geopandas_tools/bounds.py,sha256=MUtm3w6P_t7RfZ8WJz5iKqG2PyVzFMiAALf6AMLAewk,23798
+sgis/geopandas_tools/bounds.py,sha256=YJyF0gp78hFAjLLZmDquRKCBAtbt7QouG3snTcJeNQs,23822
 sgis/geopandas_tools/buffer_dissolve_explode.py,sha256=t9GJqRMDsHEU74RIlqeMr4QBgbTK0hYlXL4af1RKIks,19955
 sgis/geopandas_tools/centerlines.py,sha256=Q65Sx01SeAlulBEd9oaZkB2maBBNdLcJwAbTILg4SPU,11848
 sgis/geopandas_tools/cleaning.py,sha256=_V3KrJBaL8hZk1Iv6HBfTMTe7GCgcRbWfelkWOxqaIg,24116
@@ -18,10 +18,10 @@ sgis/geopandas_tools/point_operations.py,sha256=JM4hvfIVxZaZdGNlGzcCurrKzkgC_b9h
 sgis/geopandas_tools/polygon_operations.py,sha256=FJ-dXCxLHRsmp0oXsmBOFRprFFwmhrxqOPZkW2WWWQM,50088
 sgis/geopandas_tools/polygons_as_rings.py,sha256=BX_GZS6F9I4NbEpiOlNBd7zywJjdfdJVi_MkeONBuiM,14941
 sgis/geopandas_tools/sfilter.py,sha256=SLcMYprQwnY5DNo0R7TGXk4m6u26H8o4PRn-RPhmeZY,9345
-sgis/helpers.py,sha256=Ah4vFPQwnAFkZRALLBiR5iXwRMrx89kNTx5-5NTHsY8,9475
+sgis/helpers.py,sha256=_h7ke9hJrRNhHW-ZX3gA95fOrX2s1ADKBMxc94p2F4Q,9627
 sgis/io/__init__.py,sha256=uyBr20YDqB2bQttrd5q1JuGOvX32A-MSvS7Wmw5f5qg,177
 sgis/io/_is_dapla.py,sha256=wmfkSe98IrLhUg3dtXZusV6OVC8VlY1kbc5EQDf3P-Q,358
-sgis/io/dapla_functions.py,sha256=jHE67ADF1Fu3WG1-4x696r6f4pSUG3f9JaJu_c2Aqjs,29360
+sgis/io/dapla_functions.py,sha256=nH0aIC7LsyP9SQ_y8iz1ALbqoqq6WK_LqZ9dg3Yvq20,29851
 sgis/io/opener.py,sha256=HWO3G1NB6bpXKM94JadCD513vjat1o1TFjWGWzyVasg,898
 sgis/io/read_parquet.py,sha256=FvZYv1rLkUlrSaUY6QW6E1yntmntTeQuZ9ZRgCDO4IM,3776
 sgis/maps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,7 +34,7 @@ sgis/maps/maps.py,sha256=gxu0rgcVygjudRtM1dVRmsUMilMUIg3vG-UgvASM91E,23072
 sgis/maps/norge_i_bilder.json,sha256=W_mFfte3DxugWbEudZ5fadZ2JeFYb0hyab2Quf4oJME,481311
 sgis/maps/thematicmap.py,sha256=w6q4_gIr8BubQgsPJkc6WXk-tmplDLGcKyjphhFp7ng,21873
 sgis/maps/tilesources.py,sha256=F4mFHxPwkiPJdVKzNkScTX6xbJAMIUtlTq4mQ83oguw,1746
-sgis/maps/wms.py,sha256=XHlCszR0raPbmUc2wYpQ_XRHnSJ6c1ic3w2dNnfMRm4,6252
+sgis/maps/wms.py,sha256=Sely3Pt-cym9kRlrK0JTjtMe21kTxWY6ucD2dAoWHI0,6442
 sgis/networkanalysis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sgis/networkanalysis/_get_route.py,sha256=9I3t9pnccUPr4mozy3TJCOpGCCf3UOIojmsbifubZbA,6368
 sgis/networkanalysis/_od_cost_matrix.py,sha256=zkyPX7ObT996ahaFJ2oI0D0SqQWbWyfy_qLtXwValPg,3434
@@ -58,7 +58,7 @@ sgis/raster/indices.py,sha256=-J1HYmnT240iozvgagvyis6K0_GHZHRuUrPOgyoeIrY,223
 sgis/raster/regex.py,sha256=kYhVpRYzoXutx1dSYmqMoselWXww7MMEsTPmLZwHjbM,3759
 sgis/raster/sentinel_config.py,sha256=nySDqn2R8M6W8jguoBeSAK_zzbAsqmaI59i32446FwY,1268
 sgis/raster/zonal.py,sha256=D4Gyptw-yOLTCO41peIuYbY-DANsJCG19xXDlf1QAz4,2299
-ssb_sgis-1.1.2.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-ssb_sgis-1.1.2.dist-info/METADATA,sha256=SkKkxMGWK9n2CGvI0D4FQ4Hb4UKV9TAba38iTlDiQVI,11740
-ssb_sgis-1.1.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-ssb_sgis-1.1.2.dist-info/RECORD,,
+ssb_sgis-1.1.4.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+ssb_sgis-1.1.4.dist-info/METADATA,sha256=lPqCX4slMY8NR7PPVDLRLYdkFOstiSMvUN6NTWXX_8M,11740
+ssb_sgis-1.1.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+ssb_sgis-1.1.4.dist-info/RECORD,,