ssb-sgis 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

sgis/helpers.py

@@ -1,15 +1,29 @@
 """Small helper functions."""
+
 import glob
 import inspect
 import os
 import warnings
 from collections.abc import Callable
+from collections.abc import Generator
+from pathlib import Path
+from typing import Any
 
 import numpy as np
+import pandas as pd
 from geopandas import GeoDataFrame
 
 
-def get_numpy_func(text, error_message: str | None = None) -> Callable:
+def get_numpy_func(text: str, error_message: str | None = None) -> Callable:
+    """Fetch a numpy function based on its name.
+
+    Args:
+        text: The name of the numpy function to retrieve.
+        error_message: Custom error message if the function is not found.
+
+    Returns:
+        The numpy function corresponding to the provided text.
+    """
     f = getattr(np, text, None)
     if f is not None:
         return f
@@ -19,20 +33,36 @@ def get_numpy_func(text, error_message: str | None = None) -> Callable:
     raise ValueError(error_message)
 
 
-def get_func_name(func):
+def get_func_name(func: Callable) -> str:
+    """Return the name of a function.
+
+    Args:
+        func: The function object whose name is to be retrieved.
+
+    Returns:
+        The name of the function.
+    """
     try:
         return func.__name__
     except AttributeError:
         return str(func)
 
 
-def get_non_numpy_func_name(f):
+def get_non_numpy_func_name(f: Callable | str) -> str:
     if callable(f):
         return f.__name__
     return str(f).replace("np.", "").replace("numpy.", "")
 
 
-def to_numpy_func(text):
+def to_numpy_func(text: str) -> Callable:
+    """Convert a text identifier into a numpy function.
+
+    Args:
+        text: Name of the numpy function.
+
+    Returns:
+        The numpy function.
+    """
     f = getattr(np, text, None)
     if f is not None:
         return f
@@ -42,13 +72,22 @@ def to_numpy_func(text):
     raise ValueError
 
 
-def is_property(obj, attribute) -> bool:
+def is_property(obj: object, attribute: str) -> bool:
+    """Determine if a class attribute is a property.
+
+    Args:
+        obj: The object to check.
+        attribute: The attribute name to check on the object.
+
+    Returns:
+        True if the attribute is a property, False otherwise.
+    """
     return hasattr(obj.__class__, attribute) and isinstance(
         getattr(obj.__class__, attribute), property
     )
 
 
-def dict_zip_intersection(*dicts):
+def dict_zip_intersection(*dicts: dict) -> Generator[tuple[Any, ...], None, None]:
     """From mCoding (YouTube)."""
     if not dicts:
         return
@@ -58,7 +97,9 @@ def dict_zip_intersection(*dicts):
         yield key, *(d[key] for d in dicts)
 
 
-def dict_zip_union(*dicts, fillvalue=None):
+def dict_zip_union(
+    *dicts: dict, fillvalue: Any | None = None
+) -> Generator[tuple[Any, ...], None, None]:
     """From mCoding (YouTube)."""
     if not dicts:
         return
@@ -68,7 +109,7 @@ def dict_zip_union(*dicts, fillvalue=None):
         yield key, *(d.get(key, fillvalue) for d in dicts)
 
 
-def dict_zip(*dicts):
+def dict_zip(*dicts: dict) -> Generator[tuple[Any, ...], None, None]:
     """From mCoding (YouTube)."""
     if not dicts:
         return
@@ -81,17 +122,26 @@ def dict_zip(*dicts):
         yield key, first_val, *(other[key] for other in dicts[1:])
 
 
-def in_jupyter():
+def in_jupyter() -> bool:
     try:
-        get_ipython
+        get_ipython  # type: ignore[name-defined]
         return True
     except NameError:
         return False
 
 
-def get_all_files(root, recursive=True):
+def get_all_files(root: str, recursive: bool = True) -> list[str]:
+    """Fetch all files in a directory.
+
+    Args:
+        root: The root directory path.
+        recursive: Whether to include subdirectories.
+
+    Returns:
+        A list of file paths.
+    """
     if not recursive:
-        return [path for path in glob.glob(str(Path(root)) + "/*")]
+        return [path for path in glob.glob(str(Path(root)) + "/**")]
     paths = []
     for root_dir, _, files in os.walk(root):
         for file in files:
@@ -101,8 +151,8 @@ def get_all_files(root, recursive=True):
 
 
 def return_two_vals(
-    vals: tuple[str | None, str | None] | list[str] | str | int | float
-) -> tuple[str | int | float, str | int | float | None]:
+    vals: tuple[str, str] | list[str] | str | int | float
+) -> tuple[str | int | float, str | int | float]:
     """Return a two-length tuple from a str/int/float or list/tuple of length 1 or 2.
 
     Returns 'vals' as a 2-length tuple. If the input is a string, return
@@ -117,7 +167,7 @@ def return_two_vals(
     """
     if isinstance(vals, str):
         return vals, vals
-    if hasattr(vals, "__iter__"):
+    if isinstance(vals, (tuple, list)):
         if len(vals) == 2:
             return vals[0], vals[1]
         if len(vals) == 1:
@@ -155,45 +205,32 @@ def unit_is_degrees(gdf: GeoDataFrame) -> bool:
 
 def get_object_name(
     var: object, start: int = 2, stop: int = 7, ignore_self: bool = True
-) -> str | None:
-    """Searches through the local variables down one level at a time."""
-    frame = inspect.currentframe()
-
-    for _ in range(start):
-        frame = frame.f_back
-
-    for _ in np.arange(start, stop):
-        names = [
-            var_name for var_name, var_val in frame.f_locals.items() if var_val is var
-        ]
-        if names and len(names) == 1:
-            if ignore_self and names[0] == "self":
-                frame = frame.f_back
-                continue
-            return names[0]
-
-        names = [name for name in names if not name.startswith("_")]
-
-        if names and len(names) == 1:
-            if ignore_self and names[0] == "self":
-                frame = frame.f_back
-                continue
-
-            return names[0]
-
-        if names and len(names) > 1:
-            if ignore_self and names[0] == "self":
-                frame = frame.f_back
-                continue
-            warnings.warn(
-                "More than one local variable matches the object. Name might be wrong."
-            )
-            return names[0]
-
-        frame = frame.f_back
-
-        if not frame:
-            return
+) -> str:
+    frame = inspect.currentframe()  # frame can be FrameType or None
+    if frame:
+        try:
+            for _ in range(start):
+                frame = frame.f_back if frame else None
+            for _ in range(start, stop):
+                if frame:
+                    names = [
+                        var_name
+                        for var_name, var_val in frame.f_locals.items()
+                        if var_val is var and not (ignore_self and var_name == "self")
+                    ]
+                    names = [name for name in names if not name.startswith("_")]
+                    if names:
+                        if len(names) != 1:
+                            warnings.warn(
+                                "More than one local variable matches the object. Name might be wrong.",
+                                stacklevel=2,
+                            )
+                        return names[0]
+                frame = frame.f_back if frame else None
+        finally:
+            if frame:
+                del frame  # Explicitly delete frame reference to assist with garbage collection
+    raise ValueError(f"Couldn't find name for {var}")
 
 
 def make_namedict(gdfs: tuple[GeoDataFrame]) -> dict[int, str]:
@@ -207,7 +244,16 @@ def make_namedict(gdfs: tuple[GeoDataFrame]) -> dict[int, str]:
     return namedict
 
 
-def sort_nans_last(df, ignore_index: bool = False):
+def sort_nans_last(df: pd.DataFrame, ignore_index: bool = False) -> pd.DataFrame:
+    """Sort a DataFrame placing rows with the most NaNs last.
+
+    Args:
+        df: DataFrame to sort.
+        ignore_index: If True, the index will be reset.
+
+    Returns:
+        Sorted DataFrame with NaNs last.
+    """
     if not len(df):
         return df
     df["n_nan"] = df.isna().sum(axis=1).values
@@ -219,7 +265,15 @@ def sort_nans_last(df, ignore_index: bool = False):
     return df.reset_index(drop=True) if ignore_index else df
 
 
-def is_number(text) -> bool:
+def is_number(text: str) -> bool:
+    """Check if a string can be converted to a number.
+
+    Args:
+        text: The string to check.
+
+    Returns:
+        True if the string can be converted to a number, False otherwise.
+    """
     try:
         float(text)
         return True
@@ -228,10 +282,14 @@ def is_number(text) -> bool:
 
 
 class LocalFunctionError(ValueError):
-    def __init__(self, func: str):
+    """Exception for when a locally defined function is used in Jupyter, which is incompatible with multiprocessing."""
+
+    def __init__(self, func: Callable) -> None:
+        """Initialiser."""
         self.func = func.__name__
 
-    def __str__(self):
+    def __str__(self) -> str:
+        """Error message representation."""
         return (
             f"{self.func}. "
             "In Jupyter, functions to be parallelized must \n"

sgis/io/dapla_functions.py

@@ -1,11 +1,10 @@
-"""Functions for reading and writing GeoDataFrames in Statistics Norway's GCS Dapla.
-"""
+"""Functions for reading and writing GeoDataFrames in Statistics Norway's GCS Dapla."""
 
 from pathlib import Path
-from typing import Optional
 
 import dapla as dp
 import geopandas as gpd
+import joblib
 import pandas as pd
 from geopandas import GeoDataFrame
 from geopandas.io.arrow import _geopandas_to_arrow
@@ -14,9 +13,9 @@ from pyarrow import parquet
 
 
 def read_geopandas(
-    gcs_path: str | Path,
+    gcs_path: str | Path | list[str | Path],
     pandas_fallback: bool = False,
-    file_system: Optional[dp.gcs.GCSFileSystem] = None,
+    file_system: dp.gcs.GCSFileSystem | None = None,
     **kwargs,
 ) -> GeoDataFrame | DataFrame:
     """Reads geoparquet or other geodata from a file on GCS.
@@ -28,25 +27,35 @@ def read_geopandas(
         Does not currently read shapefiles or filegeodatabases.
 
     Args:
-        gcs_path: path to a file on Google Cloud Storage.
+        gcs_path: path to one or more files on Google Cloud Storage.
+            Multiple paths are read with threading.
         pandas_fallback: If False (default), an exception is raised if the file can
             not be read with geopandas and the number of rows is more than 0. If True,
-            the file will be read as
+            the file will be read with pandas if geopandas fails.
+        file_system: Optional file system.
         **kwargs: Additional keyword arguments passed to geopandas' read_parquet
             or read_file, depending on the file type.
 
-     Returns:
+    Returns:
          A GeoDataFrame if it has rows. If zero rows, a pandas DataFrame is returned.
     """
+    if file_system is None:
+        file_system = dp.FileClient.get_gcs_file_system()
+
+    if isinstance(gcs_path, (list, tuple)):
+        kwargs |= {"file_system": file_system, "pandas_fallback": pandas_fallback}
+        # recursive read with threads
+        with joblib.Parallel(n_jobs=len(gcs_path), backend="threading") as parallel:
+            dfs: list[GeoDataFrame] = parallel(
+                joblib.delayed(read_geopandas)(x, **kwargs) for x in gcs_path
+            )
+        return pd.concat(dfs)
 
     if not isinstance(gcs_path, str):
         try:
             gcs_path = str(gcs_path)
-        except TypeError:
-            raise TypeError(f"Unexpected type {type(gcs_path)}.")
-
-    if file_system is None:
-        file_system = dp.FileClient.get_gcs_file_system()
+        except TypeError as e:
+            raise TypeError(f"Unexpected type {type(gcs_path)}.") from e
 
     if "parquet" in gcs_path or "prqt" in gcs_path:
         with file_system.open(gcs_path, mode="rb") as file:
@@ -77,11 +86,11 @@ def read_geopandas(
 
 
 def write_geopandas(
-    df: gpd.GeoDataFrame,
+    df: GeoDataFrame,
     gcs_path: str | Path,
     overwrite: bool = True,
     pandas_fallback: bool = False,
-    file_system: Optional[dp.gcs.GCSFileSystem] = None,
+    file_system: dp.gcs.GCSFileSystem | None = None,
     **kwargs,
 ) -> None:
     """Writes a GeoDataFrame to the speficied format.
@@ -93,10 +102,13 @@ def write_geopandas(
         df: The GeoDataFrame to write.
         gcs_path: The path to the file you want to write to.
         overwrite: Whether to overwrite the file if it exists. Defaults to True.
+        pandas_fallback: If False (default), an exception is raised if the file can
+            not be written with geopandas and the number of rows is more than 0. If True,
+            the file will be written without geo-metadata if >0 rows.
+        file_system: Optional file sustem.
         **kwargs: Additional keyword arguments passed to parquet.write_table
             (for parquet) or geopandas' to_file method (if not parquet).
     """
-
     if not isinstance(gcs_path, str):
         try:
             gcs_path = str(gcs_path)
@@ -109,7 +121,8 @@ def write_geopandas(
     if file_system is None:
         file_system = dp.FileClient.get_gcs_file_system()
 
-    pd.io.parquet.BaseImpl.validate_dataframe(df)
+    if not isinstance(df, GeoDataFrame):
+        raise ValueError("DataFrame must be GeoDataFrame.")
 
     if not len(df):
         if pandas_fallback:
@@ -152,7 +165,6 @@ def exists(path: str | Path) -> bool:
     Returns:
         True if the path exists, False if not.
     """
-
     file_system = dp.FileClient.get_gcs_file_system()
     return file_system.exists(path)
 
@@ -185,7 +197,7 @@ def check_files(
     ]
     folderinfo = [x["name"] for x in info if x["storageClass"] == "DIRECTORY"]
 
-    fileinfo += get_files_in_subfolders(folderinfo)
+    fileinfo += _get_files_in_subfolders(folderinfo)
 
     df = pd.DataFrame(fileinfo, columns=["path", "kb", "updated"])
 
@@ -224,12 +236,9 @@ def check_files(
     return df.loc[lambda x: x.index > the_time, ["kb", "mb", "name", "child", "path"]]
 
 
-def get_files_in_subfolders(folderinfo: list[dict]) -> list[dict]:
+def _get_files_in_subfolders(folderinfo: list[dict]) -> list[tuple]:
     file_system = dp.FileClient.get_gcs_file_system()
 
-    if isinstance(folderinfo, (str, Path)):
-        folderinfo = [folderinfo]
-
     fileinfo = []
 
     while folderinfo:

sgis/io/opener.py

@@ -1,19 +1,26 @@
+from collections.abc import Generator
 from contextlib import contextmanager
-
+from typing import Any
 
 try:
-    import dapla as dp
+    from dapla import FileClient
+    from dapla.gcs import GCSFileSystem
 except ImportError:
-    pass
+
+    class GCSFileSystem:  # type: ignore[no-redef]
+        """Placeholder."""
+
 
 from ._is_dapla import is_dapla
 
 
 @contextmanager
-def opener(path, mode="rb", file_system=None):
+def opener(
+    path, mode: str = "rb", file_system: GCSFileSystem | None = None
+) -> Generator[str | Any, None, None]:
     """Yields a gcs buffer if in Dapla, otherwise yields the path.
 
-    Example
+    Example:
     -------
     >>> with opener(path) as file:
     >>>     with rasterio.open(file) as src:
@@ -21,7 +28,7 @@ def opener(path, mode="rb", file_system=None):
     """
     if is_dapla():
         if file_system is None:
-            file_system = dp.FileClient.get_gcs_file_system()
+            file_system = FileClient.get_gcs_file_system()
         yield file_system.open(str(path), mode=mode)
     else:
         yield str(path)

sgis/io/read_parquet.py

@@ -14,8 +14,8 @@ def read_parquet_url(url: str) -> GeoDataFrame:
     Returns:
         A GeoDataFrame.
 
-    Examples
-    --------
+    Examples:
+    ---------
     >>> from sgis import read_parquet_url
     >>> url = "https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/points_oslo.parquet"
     >>> points = read_parquet_url(url)

sgis/maps/examine.py

@@ -3,8 +3,14 @@ import numpy as np
 
 from ..geopandas_tools.bounds import get_total_bounds
 from ..helpers import unit_is_degrees
+from ..raster.image_collection import Band
+from ..raster.image_collection import Image
+from ..raster.image_collection import ImageCollection
+from .explore import Explore
 from .map import Map
-from .maps import clipmap, explore, samplemap
+from .maps import clipmap
+from .maps import explore
+from .maps import samplemap
 
 
 class Examine:
@@ -19,20 +25,8 @@ class Examine:
     first geometry in 'mask_gdf' (or the first speficied gdf). The 'next' method
     can then be repeated.
 
-    Args:
-        *gdfs: One or more GeoDataFrames. The rows of the first GeoDataFrame
-            will be used as masks, unless 'mask_gdf' is specified.
-        column: Column to use as colors.
-        mask_gdf: Optional GeoDataFrame to use as mask iterator. The geometries
-            of mask_gdf will not be shown.
-        size: Number of meters (or other crs unit) to buffer the mask geometry
-            before clipping.
-        sort_values: Optional sorting column(s) of the mask GeoDataFrame. Rows
-            will be iterated through from the top.
-        **kwargs: Additional keyword arguments passed to sgis.clipmap.
-
-    Examples
-    --------
+    Examples:
+    ---------
     Create the examiner.
 
     >>> import sgis as sg
@@ -78,7 +72,24 @@ class Examine:
         size: int | float = 1000,
         only_show_mask: bool = True,
         **kwargs,
-    ):
+    ) -> None:
+        """Initialiser.
+
+        Args:
+            *gdfs: One or more GeoDataFrames. The rows of the first GeoDataFrame
+                will be used as masks, unless 'mask_gdf' is specified.
+            column: Column to use as colors.
+            mask_gdf: Optional GeoDataFrame to use as mask iterator. The geometries
+                of mask_gdf will not be shown.
+            size: Number of meters (or other crs unit) to buffer the mask geometry
+                before clipping.
+            sort_values: Optional sorting column(s) of the mask GeoDataFrame. Rows
+                will be iterated through from the top.
+            only_show_mask: If True (default), show only the mask GeoDataFrame by default.
+                The other layers can be toggled on.
+            **kwargs: Additional keyword arguments passed to sgis.clipmap.
+
+        """
         gdfs, column, kwargs = Map._separate_args(gdfs, column, kwargs)
 
         if mask_gdf is None:
@@ -86,8 +97,14 @@ class Examine:
         else:
             self.mask_gdf = mask_gdf
 
-        m = Map(*gdfs, column=column, **kwargs)
-        self._gdfs: dict[str, gpd.GeoDataFrame] = dict(zip(m.labels, m.gdfs))
+        m = Explore(*gdfs, column=column, **kwargs)
+
+        # m = Map(*gdfs, column=column, **kwargs)
+        self._gdfs: dict[str, gpd.GeoDataFrame] = dict(
+            zip(m.labels, m.gdfs, strict=False)
+        )
+
+        self.rasters: dict[str, ImageCollection | Image | Band] = m.rasters
 
         self.indices = list(range(len(self.mask_gdf)))
         self.i = 0
@@ -121,7 +138,7 @@ class Examine:
         elif not kwargs.get("show", True):
             self.kwargs["show"] = [False] * len(self._gdfs)
 
-    def next(self, i: int | None = None, **kwargs):
+    def next(self, i: int | None = None, **kwargs) -> None:
         """Displays a map of geometries within the next row of the mask gdf.
 
         Args:
@@ -145,13 +162,14 @@ class Examine:
         print(f"i == {self.i} (of {len(self.mask_gdf)})")
         clipmap(
             self.column,
+            *list(self.rasters.values()),
             **self._gdfs,
             mask=self.mask_gdf.iloc[[self.i]].buffer(self.size),
             **self.kwargs,
         )
         self.i += 1
 
-    def sample(self, **kwargs):
+    def sample(self, **kwargs) -> None:
         """Takes a sample index of the mask and displays a map of this area.
 
         Args:
@@ -166,12 +184,13 @@ class Examine:
         print(f"Showing index {i}")
         clipmap(
             self.column,
+            *list(self.rasters.values()),
             **self._gdfs,
             mask=self.mask_gdf.iloc[[i]].buffer(self.size),
             **self.kwargs,
         )
 
-    def current(self, i: int | None = None, **kwargs):
+    def current(self, i: int | None = None, **kwargs) -> None:
         """Repeat the last shown map."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
@@ -185,42 +204,46 @@ class Examine:
         print(f"{self.i + 1} of {len(self.mask_gdf)}")
         clipmap(
             self.column,
+            *list(self.rasters.values()),
             **self._gdfs,
             mask=self.mask_gdf.iloc[[self.i]].buffer(self.size),
             **self.kwargs,
         )
 
-    def explore(self, **kwargs):
+    def explore(self, **kwargs) -> None:
         """Show all rows like the function explore."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
             self.kwargs = self.kwargs | kwargs
 
         explore(
+            *list(self.rasters.values()),
             **self._gdfs,
             column=self.column,
             **self.kwargs,
         )
 
-    def clipmap(self, **kwargs):
+    def clipmap(self, **kwargs) -> None:
         """Show all rows like the function clipmap."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
             self.kwargs = self.kwargs | kwargs
 
         clipmap(
+            *list(self.rasters.values()),
             **self._gdfs,
             column=self.column,
             **self.kwargs,
         )
 
-    def samplemap(self, **kwargs):
+    def samplemap(self, **kwargs) -> None:
         """Show all rows like the function samplemap."""
         if kwargs:
             kwargs = self._fix_kwargs(kwargs)
             self.kwargs = self.kwargs | kwargs
 
         samplemap(
+            *list(self.rasters.values()),
             **self._gdfs,
             column=self.column,
             **self.kwargs,
@@ -241,21 +264,25 @@ class Examine:
         return gdfs
 
     @property
-    def bounds(self):
+    def bounds(self) -> tuple[float, float, float, float]:
+        """Total bounds of all GeoDataFrames."""
         return get_total_bounds(*list(self.gdfs.values()))
 
-    def _fix_kwargs(self, kwargs) -> dict:
+    def _fix_kwargs(self, kwargs: dict) -> dict:
         self.size = kwargs.pop("size", self.size)
         self.column = kwargs.pop("column", self.column)
         return kwargs
 
     def __repr__(self) -> str:
+        """Representation."""
         return f"{self.__class__.__name__}(indices={len(self.indices)}, current={self.i}, n_gdfs={len(self._gdfs)})"
 
-    def __add__(self, scalar):
+    def __add__(self, scalar: int) -> "Examine":
+        """Add a number to the index."""
         self.i += scalar
         return self
 
-    def __sub__(self, scalar):
+    def __sub__(self, scalar: int) -> "Examine":
+        """Subtract a number from the index."""
         self.i -= scalar
         return self