ssb-sgis 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

sgis/io/dapla_functions.py

@@ -1,11 +1,10 @@
-"""Functions for reading and writing GeoDataFrames in Statistics Norway's GCS Dapla.
-"""
+"""Functions for reading and writing GeoDataFrames in Statistics Norway's GCS Dapla."""
 
 from pathlib import Path
-from typing import Optional
 
 import dapla as dp
 import geopandas as gpd
+import joblib
 import pandas as pd
 from geopandas import GeoDataFrame
 from geopandas.io.arrow import _geopandas_to_arrow
@@ -14,9 +13,9 @@ from pyarrow import parquet
 
 
 def read_geopandas(
-    gcs_path: str | Path,
+    gcs_path: str | Path | list[str | Path],
     pandas_fallback: bool = False,
-    file_system: Optional[dp.gcs.GCSFileSystem] = None,
+    file_system: dp.gcs.GCSFileSystem | None = None,
     **kwargs,
 ) -> GeoDataFrame | DataFrame:
     """Reads geoparquet or other geodata from a file on GCS.
@@ -28,25 +27,35 @@ def read_geopandas(
         Does not currently read shapefiles or filegeodatabases.
 
     Args:
-        gcs_path: path to a file on Google Cloud Storage.
+        gcs_path: path to one or more files on Google Cloud Storage.
+            Multiple paths are read with threading.
         pandas_fallback: If False (default), an exception is raised if the file can
             not be read with geopandas and the number of rows is more than 0. If True,
-            the file will be read as
+            the file will be read with pandas if geopandas fails.
+        file_system: Optional file system.
         **kwargs: Additional keyword arguments passed to geopandas' read_parquet
             or read_file, depending on the file type.
 
-     Returns:
+    Returns:
          A GeoDataFrame if it has rows. If zero rows, a pandas DataFrame is returned.
     """
+    if file_system is None:
+        file_system = dp.FileClient.get_gcs_file_system()
+
+    if isinstance(gcs_path, (list, tuple)):
+        kwargs |= {"file_system": file_system, "pandas_fallback": pandas_fallback}
+        # recursive read with threads
+        with joblib.Parallel(n_jobs=len(gcs_path), backend="threading") as parallel:
+            dfs: list[GeoDataFrame] = parallel(
+                joblib.delayed(read_geopandas)(x, **kwargs) for x in gcs_path
+            )
+        return pd.concat(dfs)
 
     if not isinstance(gcs_path, str):
         try:
             gcs_path = str(gcs_path)
-        except TypeError:
-            raise TypeError(f"Unexpected type {type(gcs_path)}.")
-
-    if file_system is None:
-        file_system = dp.FileClient.get_gcs_file_system()
+        except TypeError as e:
+            raise TypeError(f"Unexpected type {type(gcs_path)}.") from e
 
     if "parquet" in gcs_path or "prqt" in gcs_path:
         with file_system.open(gcs_path, mode="rb") as file:
@@ -77,11 +86,11 @@ def read_geopandas(
 
 
 def write_geopandas(
-    df: gpd.GeoDataFrame,
+    df: GeoDataFrame,
     gcs_path: str | Path,
     overwrite: bool = True,
     pandas_fallback: bool = False,
-    file_system: Optional[dp.gcs.GCSFileSystem] = None,
+    file_system: dp.gcs.GCSFileSystem | None = None,
     **kwargs,
 ) -> None:
     """Writes a GeoDataFrame to the speficied format.
@@ -93,10 +102,13 @@ def write_geopandas(
         df: The GeoDataFrame to write.
         gcs_path: The path to the file you want to write to.
         overwrite: Whether to overwrite the file if it exists. Defaults to True.
+        pandas_fallback: If False (default), an exception is raised if the file can
+            not be written with geopandas and the number of rows is more than 0. If True,
+            the file will be written without geo-metadata if >0 rows.
+        file_system: Optional file sustem.
         **kwargs: Additional keyword arguments passed to parquet.write_table
             (for parquet) or geopandas' to_file method (if not parquet).
     """
-
     if not isinstance(gcs_path, str):
         try:
             gcs_path = str(gcs_path)
@@ -109,7 +121,8 @@ def write_geopandas(
     if file_system is None:
         file_system = dp.FileClient.get_gcs_file_system()
 
-    pd.io.parquet.BaseImpl.validate_dataframe(df)
+    if not isinstance(df, GeoDataFrame):
+        raise ValueError("DataFrame must be GeoDataFrame.")
 
     if not len(df):
         if pandas_fallback:
@@ -152,7 +165,6 @@ def exists(path: str | Path) -> bool:
     Returns:
         True if the path exists, False if not.
     """
-
     file_system = dp.FileClient.get_gcs_file_system()
     return file_system.exists(path)
 
@@ -185,7 +197,7 @@ def check_files(
     ]
     folderinfo = [x["name"] for x in info if x["storageClass"] == "DIRECTORY"]
 
-    fileinfo += get_files_in_subfolders(folderinfo)
+    fileinfo += _get_files_in_subfolders(folderinfo)
 
     df = pd.DataFrame(fileinfo, columns=["path", "kb", "updated"])
 
@@ -224,12 +236,9 @@ def check_files(
     return df.loc[lambda x: x.index > the_time, ["kb", "mb", "name", "child", "path"]]
 
 
-def get_files_in_subfolders(folderinfo: list[dict]) -> list[dict]:
+def _get_files_in_subfolders(folderinfo: list[dict]) -> list[tuple]:
     file_system = dp.FileClient.get_gcs_file_system()
 
-    if isinstance(folderinfo, (str, Path)):
-        folderinfo = [folderinfo]
-
     fileinfo = []
 
     while folderinfo:

sgis/io/opener.py

@@ -1,19 +1,26 @@
+from collections.abc import Generator
 from contextlib import contextmanager
-
+from typing import Any
 
 try:
-    import dapla as dp
+    from dapla import FileClient
+    from dapla.gcs import GCSFileSystem
 except ImportError:
-    pass
+
+    class GCSFileSystem:  # type: ignore[no-redef]
+        """Placeholder."""
+
 
 from ._is_dapla import is_dapla
 
 
 @contextmanager
-def opener(path, mode="rb", file_system=None):
+def opener(
+    path, mode: str = "rb", file_system: GCSFileSystem | None = None
+) -> Generator[str | Any, None, None]:
     """Yields a gcs buffer if in Dapla, otherwise yields the path.
 
-    Example
+    Example:
     -------
     >>> with opener(path) as file:
     >>>     with rasterio.open(file) as src:
@@ -21,7 +28,7 @@ def opener(path, mode="rb", file_system=None):
     """
     if is_dapla():
         if file_system is None:
-            file_system = dp.FileClient.get_gcs_file_system()
+            file_system = FileClient.get_gcs_file_system()
         yield file_system.open(str(path), mode=mode)
     else:
         yield str(path)

sgis/io/read_parquet.py

@@ -14,7 +14,7 @@ def read_parquet_url(url: str) -> GeoDataFrame:
     Returns:
         A GeoDataFrame.
 
-    Examples
+    Examples:
     --------
     >>> from sgis import read_parquet_url
     >>> url = "https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/points_oslo.parquet"

sgis/maps/examine.py

@@ -4,7 +4,9 @@ import numpy as np
 from ..geopandas_tools.bounds import get_total_bounds
 from ..helpers import unit_is_degrees
 from .map import Map
-from .maps import clipmap, explore, samplemap
+from .maps import clipmap
+from .maps import explore
+from .maps import samplemap
 
 
 class Examine:
@@ -19,19 +21,7 @@ class Examine:
     first geometry in 'mask_gdf' (or the first speficied gdf). The 'next' method
     can then be repeated.
 
-    Args:
-        *gdfs: One or more GeoDataFrames. The rows of the first GeoDataFrame
-            will be used as masks, unless 'mask_gdf' is specified.
-        column: Column to use as colors.
-        mask_gdf: Optional GeoDataFrame to use as mask iterator. The geometries
-            of mask_gdf will not be shown.
-        size: Number of meters (or other crs unit) to buffer the mask geometry
-            before clipping.
-        sort_values: Optional sorting column(s) of the mask GeoDataFrame. Rows
-            will be iterated through from the top.
-        **kwargs: Additional keyword arguments passed to sgis.clipmap.
-
-    Examples
+    Examples:
     --------
     Create the examiner.
 
@@ -78,7 +68,24 @@ class Examine:
         size: int | float = 1000,
         only_show_mask: bool = True,
         **kwargs,
-    ):
+    ) -> None:
+        """Initialiser.
+
+        Args:
+            *gdfs: One or more GeoDataFrames. The rows of the first GeoDataFrame
+                will be used as masks, unless 'mask_gdf' is specified.
+            column: Column to use as colors.
+            mask_gdf: Optional GeoDataFrame to use as mask iterator. The geometries
+                of mask_gdf will not be shown.
+            size: Number of meters (or other crs unit) to buffer the mask geometry
+                before clipping.
+            sort_values: Optional sorting column(s) of the mask GeoDataFrame. Rows
+                will be iterated through from the top.
+            only_show_mask: If True (default), show only the mask GeoDataFrame by default.
+                The other layers can be toggled on.
+            **kwargs: Additional keyword arguments passed to sgis.clipmap.
+
+        """
         gdfs, column, kwargs = Map._separate_args(gdfs, column, kwargs)
 
         if mask_gdf is None:
@@ -87,7 +94,9 @@ class Examine:
             self.mask_gdf = mask_gdf
 
         m = Map(*gdfs, column=column, **kwargs)
-        self._gdfs: dict[str, gpd.GeoDataFrame] = dict(zip(m.labels, m.gdfs))
+        self._gdfs: dict[str, gpd.GeoDataFrame] = dict(
+            zip(m.labels, m.gdfs, strict=False)
+        )
 
         self.indices = list(range(len(self.mask_gdf)))
         self.i = 0
@@ -121,7 +130,7 @@ class Examine:
         elif not kwargs.get("show", True):
             self.kwargs["show"] = [False] * len(self._gdfs)
 
-    def next(self, i: int | None = None, **kwargs):
+    def next(self, i: int | None = None, **kwargs) -> None:
         """Displays a map of geometries within the next row of the mask gdf.
 
         Args:
@@ -151,7 +160,7 @@ class Examine:
         )
         self.i += 1
 
-    def sample(self, **kwargs):
+    def sample(self, **kwargs) -> None:
         """Takes a sample index of the mask and displays a map of this area.
 
         Args:
@@ -171,7 +180,7 @@ class Examine:
             **self.kwargs,
         )
 
-    def current(self, i: int | None = None, **kwargs):
+    def current(self, i: int | None = None, **kwargs) -> None:
         """Repeat the last shown map."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
@@ -190,7 +199,7 @@ class Examine:
             **self.kwargs,
         )
 
-    def explore(self, **kwargs):
+    def explore(self, **kwargs) -> None:
         """Show all rows like the function explore."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
@@ -202,7 +211,7 @@ class Examine:
             **self.kwargs,
         )
 
-    def clipmap(self, **kwargs):
+    def clipmap(self, **kwargs) -> None:
         """Show all rows like the function clipmap."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
@@ -214,7 +223,7 @@ class Examine:
             **self.kwargs,
         )
 
-    def samplemap(self, **kwargs):
+    def samplemap(self, **kwargs) -> None:
         """Show all rows like the function samplemap."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
@@ -241,21 +250,25 @@ class Examine:
         return gdfs
 
     @property
-    def bounds(self):
+    def bounds(self) -> tuple[float, float, float, float]:
+        """Total bounds of all GeoDataFrames."""
         return get_total_bounds(*list(self.gdfs.values()))
 
-    def _fix_kwargs(self, kwargs) -> dict:
+    def _fix_kwargs(self, kwargs: dict) -> dict:
         self.size = kwargs.pop("size", self.size)
         self.column = kwargs.pop("column", self.column)
         return kwargs
 
     def __repr__(self) -> str:
+        """Representation."""
         return f"{self.__class__.__name__}(indices={len(self.indices)}, current={self.i}, n_gdfs={len(self._gdfs)})"
 
-    def __add__(self, scalar):
+    def __add__(self, scalar: int) -> "Examine":
+        """Add a number to the index."""
         self.i += scalar
         return self
 
-    def __sub__(self, scalar):
+    def __sub__(self, scalar: int) -> "Examine":
+        """Subtract a number from the index."""
         self.i -= scalar
         return self

sgis/maps/explore.py

@@ -9,6 +9,8 @@ import warnings
 from collections.abc import Iterable
 from numbers import Number
 from statistics import mean
+from typing import Any
+from typing import ClassVar
 
 import branca as bc
 import folium
@@ -17,27 +19,30 @@ import numpy as np
 import pandas as pd
 import xyzservices
 from folium import plugins
-from geopandas import GeoDataFrame, GeoSeries
+from geopandas import GeoDataFrame
+from geopandas import GeoSeries
 from IPython.display import display
 from jinja2 import Template
 from pandas.api.types import is_datetime64_any_dtype
 from shapely import Geometry
 from shapely.geometry import LineString
 
-from ..geopandas_tools.conversion import from_4326, to_gdf
-from ..geopandas_tools.general import clean_geoms, make_all_singlepart
-from ..geopandas_tools.geometry_types import get_geom_type, to_single_geom_type
+from ..geopandas_tools.conversion import to_gdf
+from ..geopandas_tools.general import clean_geoms
+from ..geopandas_tools.general import make_all_singlepart
+from ..geopandas_tools.geometry_types import get_geom_type
+from ..geopandas_tools.geometry_types import to_single_geom_type
 from .httpserver import run_html_server
 from .map import Map
-from .tilesources import kartverket, xyz
-
+from .tilesources import kartverket
+from .tilesources import xyz
 
 try:
     from torchgeo.datasets.geo import RasterDataset
 except ImportError:
 
     class RasterDataset:
-        """Placeholder"""
+        """Placeholder."""
 
 
 # the geopandas._explore raises a deprication warning. Ignoring for now.
@@ -96,7 +101,10 @@ class MeasureControlFix(plugins.MeasureControl):
     """
     )
 
-    def __init__(self, active_color="red", completed_color="red", **kwargs):
+    def __init__(
+        self, active_color: str = "red", completed_color: str = "red", **kwargs
+    ) -> None:
+        """Run super __init__ after the new _template class attribute is made."""
         super().__init__(
             active_color=active_color, completed_color=completed_color, **kwargs
         )
@@ -146,8 +154,10 @@ def to_tile(tile: str | xyzservices.TileProvider, max_zoom: int) -> folium.TileL
 
 
 class Explore(Map):
+    """Class for displaying and saving html maps of multiple GeoDataFrames."""
+
     # class attribute that can be overridden locally
-    tiles = (
+    tiles: ClassVar[tuple[str]] = (
         "grunnkart",
         "norge_i_bilder",
         "dark",
@@ -161,17 +171,41 @@ class Explore(Map):
         column: str | None = None,
         popup: bool = True,
         max_zoom: int = 40,
-        smooth_factor: float = 1.5,
+        smooth_factor: float = 1.1,
         browser: bool = False,
         prefer_canvas: bool = True,
         measure_control: bool = True,
         geocoder: bool = False,
-        save=None,
+        save: str | None = None,
         show: bool | Iterable[bool] | None = None,
         text: str | None = None,
         decimals: int = 6,
         **kwargs,
-    ):
+    ) -> None:
+        """Initialiser.
+
+        Args:
+            *gdfs: One or more GeoDataFrames.
+            mask: Optional mask to clip to.
+            column: Optional column to color the data by.
+            popup: Whether to make the data popups clickable.
+            max_zoom: Max levels of zoom.
+            smooth_factor: Float of 1 or higher, 1 meaning no smoothing
+                of lines.
+            browser: Whether to open the map in a browser tab.
+            prefer_canvas: Option.
+            measure_control: Whether to include measurement box.
+            geocoder: Whether to include search bar for addresses.
+            save: Optional file path to an html file. The map will then
+                be saved instead of displayed.
+            show: Whether to show or hide the data upon creating the map.
+                If False, the data can be toggled on later. 'show' can also be
+                a sequence of boolean values the same length as the number of
+                GeoDataFrames.
+            text: Optional text for a text box in the map.
+            decimals: Number of decimals in the coordinates.
+            **kwargs: Additional keyword arguments passed to
+        """
         self.popup = popup
         self.max_zoom = max_zoom
         self.smooth_factor = smooth_factor
@@ -195,12 +229,12 @@ class Explore(Map):
         else:
             show_was_none = False
 
-        self.raster_datasets = tuple(
-            raster_dataset_to_background_map(x)
-            for x in gdfs
-            if isinstance(x, RasterDataset)
-        )
-        self.tiles  # += self.raster_datasets
+        self.raster_datasets = []  # tuple(
+        #     raster_dataset_to_background_map(x)
+        #     for x in gdfs
+        #     if isinstance(x, RasterDataset)
+        # )
+        # self.tiles  # += self.raster_datasets
 
         super().__init__(*gdfs, column=column, show=show, **kwargs)
 
@@ -262,14 +296,20 @@ class Explore(Map):
 
         self.original_crs = self.gdf.crs
 
-    def __repr__(self):
+    def __repr__(self) -> str:
+        """Representation."""
         return f"{self.__class__.__name__}()"
 
     def explore(
-        self, column: str | None = None, center=None, size=None, **kwargs
+        self,
+        column: str | None = None,
+        center: Any | None = None,
+        size: int | None = None,
+        **kwargs,
     ) -> None:
+        """Explore all the data."""
         if not any(len(gdf) for gdf in self._gdfs) and not len(self.raster_datasets):
-            warnings.warn("None of the GeoDataFrames have rows.")
+            warnings.warn("None of the GeoDataFrames have rows.", stacklevel=1)
             return
         if column:
             self._column = column
@@ -311,6 +351,7 @@ class Explore(Map):
         sample_from_first: bool = True,
         **kwargs,
     ) -> None:
+        """Explore a sample of the data."""
         if column:
             self._column = column
             self._update_column()
@@ -347,10 +388,11 @@ class Explore(Map):
 
     def clipmap(
         self,
-        mask,
+        mask: Any,
         column: str | None = None,
         **kwargs,
     ) -> None:
+        """Explore the data within a mask extent."""
         if column:
             self._column = column
             self._update_column()
@@ -368,7 +410,7 @@ class Explore(Map):
         self._gdf = pd.concat(gdfs, ignore_index=True)
         self._explore(**kwargs)
 
-    def _explore(self, **kwargs):
+    def _explore(self, **kwargs) -> None:
         self.kwargs = self.kwargs | kwargs
 
         if self._is_categorical:
@@ -384,7 +426,7 @@ class Explore(Map):
         else:
             display(self.map)
 
-    def _split_categories(self):
+    def _split_categories(self) -> None:
         new_gdfs, new_labels, new_shows = [], [], []
         for cat in self._unique_values:
             gdf = self.gdf.loc[self.gdf[self.column] == cat]
@@ -396,7 +438,7 @@ class Explore(Map):
         self.labels = new_labels
         self.show = new_shows
 
-    def _to_single_geom_type(self, gdf) -> GeoDataFrame:
+    def _to_single_geom_type(self, gdf: GeoDataFrame) -> GeoDataFrame:
         gdf = clean_geoms(gdf)
 
         if get_geom_type(gdf) != "mixed":
@@ -420,16 +462,16 @@ class Explore(Map):
 
         assert get_geom_type(gdf) != "mixed", gdf.geom_type.value_counts()
 
-        warnings.warn(mess)
+        warnings.warn(mess, stacklevel=1)
 
         return gdf
 
-    def _update_column(self):
+    def _update_column(self) -> None:
         self._is_categorical = self._check_if_categorical()
         self._fillna_if_col_is_missing()
         self._gdf = pd.concat(self._gdfs, ignore_index=True)
 
-    def _create_categorical_map(self):
+    def _create_categorical_map(self) -> None:
         self._get_categorical_colors()
 
         gdf = self._prepare_gdf_for_map(self._gdf)
@@ -478,7 +520,7 @@ class Explore(Map):
 
     def _add_tiles(
         self, mapobj: folium.Map, tiles: list[str, xyzservices.TileProvider]
-    ):
+    ) -> None:
         for tile in tiles:
             to_tile(tile, max_zoom=self.max_zoom).add_to(mapobj)
 
@@ -549,7 +591,7 @@ class Explore(Map):
         return [col for col in gdf.columns if col not in COLS_TO_DROP]
 
     @staticmethod
-    def _prepare_gdf_for_map(gdf):
+    def _prepare_gdf_for_map(gdf: GeoDataFrame) -> GeoDataFrame:
         if isinstance(gdf, GeoSeries):
             gdf = gdf.to_frame("geometry")
 
@@ -567,13 +609,13 @@ class Explore(Map):
 
     def _make_folium_map(
         self,
-        bounds,
-        attr=None,
-        tiles=None,
-        width="100%",
-        height="100%",
-        control_scale=True,
-        map_kwds=None,
+        bounds: tuple[float, float, float, float],
+        attr: Any = None,
+        tiles: Any = None,
+        width: str = "100%",
+        height: str = "100%",
+        control_scale: bool = True,
+        map_kwds: dict | None = None,
         **kwargs,
     ):
         if not map_kwds:
@@ -679,21 +721,29 @@ class Explore(Map):
 
     def _make_geojson(
         self,
-        df,
+        df: GeoDataFrame,
         show: bool,
-        color=None,
-        tooltip=True,
-        popup=False,
-        highlight=True,
-        marker_type=None,
-        marker_kwds={},
-        style_kwds={},
-        highlight_kwds={},
-        tooltip_kwds={},
-        popup_kwds={},
-        map_kwds={},
+        color: str | None = None,
+        tooltip: bool = True,
+        popup: bool = False,
+        highlight: bool = True,
+        marker_type: str | None = None,
+        marker_kwds: dict | None = None,
+        style_kwds: dict | None = None,
+        highlight_kwds: dict | None = None,
+        tooltip_kwds: dict | None = None,
+        popup_kwds: dict | None = None,
+        map_kwds: dict | None = None,
         **kwargs,
-    ):
+    ) -> folium.GeoJson:
+
+        marker_kwds = marker_kwds or {}
+        style_kwds = style_kwds or {}
+        highlight_kwds = highlight_kwds or {}
+        tooltip_kwds = tooltip_kwds or {}
+        popup_kwds = popup_kwds or {}
+        map_kwds = map_kwds or {}
+
         gdf = df.copy()
 
         # convert LinearRing to LineString
@@ -816,9 +866,10 @@ class Explore(Map):
         )
 
 
-def _tooltip_popup(type, fields, gdf, **kwds):
-    """get tooltip or popup"""
-
+def _tooltip_popup(
+    type_: str, fields: Any, gdf: GeoDataFrame, **kwargs
+) -> folium.GeoJsonTooltip | folium.GeoJsonPopup:
+    """Get tooltip or popup."""
     # specify fields to show in the tooltip
     if fields is False or fields is None or fields == 0:
         return None
@@ -836,20 +887,16 @@ def _tooltip_popup(type, fields, gdf, **kwds):
 
     # Cast fields to str
     fields = list(map(str, fields))
-    if type == "tooltip":
-        return folium.GeoJsonTooltip(fields, **kwds)
-    elif type == "popup":
-        return folium.GeoJsonPopup(fields, **kwds)
-
+    if type_ == "tooltip":
+        return folium.GeoJsonTooltip(fields, **kwargs)
+    elif type_ == "popup":
+        return folium.GeoJsonPopup(fields, **kwargs)
 
-def raster_dataset_to_background_map(dataset: RasterDataset):
-    crs = dataset.crs
-    bbox = dataset.bounds
 
-
-def _categorical_legend(m, title, categories, colors):
-    """
-    Add categorical legend to a map
+def _categorical_legend(
+    m: folium.Map, title: str, categories: list[str], colors: list[str]
+) -> None:
+    """Add categorical legend to a map.
 
     The implementation is using the code originally written by Michel Metran
     (@michelmetran) and released on GitHub
@@ -868,7 +915,6 @@ def _categorical_legend(m, title, categories, colors):
     colors : list-like
         list of colors (in the same order as categories)
     """
-
     # Header to Add
     head = """
     {% macro header(this, kwargs) %}