ssb-sgis 0.3.13__py3-none-any.whl → 1.0.0__py3-none-any.whl

sgis/maps/maps.py

@@ -8,11 +8,11 @@ The 'qtm' function shows a simple static map of one or more GeoDataFrames.
 """
 
 import inspect
-import warnings
 from numbers import Number
 from typing import Any
 
 from geopandas import GeoDataFrame, GeoSeries
+from pyproj import CRS
 from shapely import Geometry
 
 from ..geopandas_tools.conversion import to_gdf as to_gdf_func
@@ -24,24 +24,23 @@ from .map import Map
 from .thematicmap import ThematicMap
 
 
+try:
+    from torchgeo.datasets.geo import RasterDataset
+except ImportError:
+
+    class RasterDataset:
+        """Placeholder"""
+
+
 def _get_location_mask(kwargs: dict, gdfs) -> tuple[GeoDataFrame | None, dict]:
     try:
         crs = get_common_crs(gdfs)
     except IndexError:
-        try:
-            crs = [x for x in kwargs.values() if hasattr(x, "crs")][0].crs
-        except IndexError:
-            crs = None
-        except Exception:
-            crs = set()
-            for x in kwargs.values():
-                try:
-                    crs.add(x.crs)
-                except Exception:
-                    pass
+        for x in kwargs.values():
             try:
-                crs = list(crs)[0]
-            except IndexError:
+                crs = CRS(x.crs) if hasattr(x, "crs") else CRS(x["crs"])
+                break
+            except Exception:
                 crs = None
 
     masks = {
@@ -73,6 +72,7 @@ def explore(
     *gdfs: GeoDataFrame | dict[str, GeoDataFrame],
     column: str | None = None,
     center: Any | None = None,
+    center_4326: Any | None = None,
     labels: tuple[str] | None = None,
     max_zoom: int = 40,
     browser: bool = False,
@@ -120,19 +120,23 @@ def explore(
 
     Examples
     --------
-    >>> from sgis import read_parquet_url, explore
-    >>> roads = read_parquet_url("https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/roads_eidskog_2022.parquet")
-    >>> points = read_parquet_url("https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/points_eidskog.parquet")
+    >>> import sgis as sg
+    >>> roads = sg.read_parquet_url("https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/roads_oslo_2022.parquet")
+    >>> points = sg.read_parquet_url("https://media.githubusercontent.com/media/statisticsnorway/ssb-sgis/main/tests/testdata/points_oslo.parquet")
 
-    Simple explore of two GeoDataFrames.
+    Explore the area 500 meter around a given point. Coordinates are in UTM 33 format (25833).
 
-    >>> explore(roads, points)
+    >>> sg.explore(roads, points, center=(262274.6528, 6650143.176, 500))
+
+    Same as above, but with coordinates given as WGS84, same as the coordinates displayed in the corner of the map.
+
+    >>> sg.explore(roads, points, center_4326=(10.7463, 59.92, 500))
 
     With additional arguments.
 
     >>> roads["meters"] = roads.length
     >>> points["meters"] = points.length
-    >>> explore(roads, points, column="meters", cmap="plasma", max_zoom=60)
+    >>> sg.explore(roads, points, column="meters", cmap="plasma", max_zoom=60, center_4326=(10.7463, 59.92, 500))
     """
 
     gdfs, column, kwargs = Map._separate_args(gdfs, column, kwargs)
@@ -154,18 +158,37 @@ def explore(
             **kwargs,
         )
 
+    try:
+        to_crs = gdfs[0].crs
+    except IndexError:
+        try:
+            to_crs = [x for x in kwargs.values() if hasattr(x, "crs")][0].crs
+        except IndexError:
+            to_crs = None
+
+    if center_4326 is not None:
+        from_crs = 4326
+        center = center_4326
+    elif "crs" in kwargs:
+        from_crs = kwargs.pop("crs")
+    else:
+        from_crs = to_crs
+
     if center is not None:
         size = size or 1000
         if isinstance(center, str) and not is_wkt(center):
-            mask = address_to_gdf(center, crs=gdfs[0].crs).buffer(size)
-        elif isinstance(center, GeoDataFrame):
+            mask = address_to_gdf(center, crs=from_crs)
+        elif isinstance(center, (GeoDataFrame, GeoSeries)):
             mask = center
         else:
-            try:
-                mask = to_gdf_func(center, crs=gdfs[0].crs)
-            except IndexError:
-                df = [x for x in kwargs.values() if hasattr(x, "crs")][0]
-                mask = to_gdf_func(center, crs=df.crs)
+            if isinstance(center, (tuple, list)) and len(center) == 3:
+                *center, size = center
+            mask = to_gdf_func(center, crs=from_crs)
+
+        try:
+            mask = mask.to_crs(to_crs)
+        except ValueError:
+            pass
 
         if get_geom_type(mask) in ["point", "line"]:
             mask = mask.buffer(size)
@@ -190,7 +213,7 @@ def explore(
         **kwargs,
     )
 
-    if m.gdfs is None:
+    if m.gdfs is None and not len(m.raster_datasets):
         return
 
     if not kwargs.pop("explore", True):
@@ -286,7 +309,7 @@ def samplemap(
             smooth_factor=smooth_factor,
             **kwargs,
         )
-        if m.gdfs is None:
+        if m.gdfs is None and not len(m.raster_datasets):
             return
         if mask is not None:
             m._gdfs = [gdf.clip(mask) for gdf in m._gdfs]
@@ -395,7 +418,7 @@ def clipmap(
             smooth_factor=smooth_factor,
             **kwargs,
         )
-        if m.gdfs is None:
+        if m.gdfs is None and not len(m.raster_datasets):
             return
 
         m._gdfs = [gdf.clip(mask) for gdf in m._gdfs]
@@ -421,36 +444,63 @@ def clipmap(
         qtm(m._gdf, column=m.column, cmap=m._cmap, k=m.k)
 
 
-def explore_locals(*gdfs, to_gdf: bool = True, **kwargs):
-    """Viser kart (explore) over alle lokale GeoDataFrames.
+def explore_locals(*gdfs, convert: bool = True, **kwargs):
+    """Displays all local variables with geometries (GeoDataFrame etc.).
 
-    Lokalt betyr enten inni funksjonen/metoden du er i, eller i notebooken
-    du jobber i.
+    Local means inside a function or file/notebook.
 
     Args:
-        *gdfs: Ekstra GeoDataFrames du vil legge til
-        **kwargs: keyword arguments som sendes til sg.explore.
+        *gdfs: Additional GeoDataFrames.
+        convert: If True (default), non-GeoDataFrames will be converted
+            to GeoDataFrames if possible.
+        **kwargs: keyword arguments passed to sg.explore.
     """
-    frame = inspect.currentframe()
 
+    def as_dict(obj):
+        if hasattr(obj, "__dict__"):
+            return obj.__dict__
+        elif isinstance(obj, dict):
+            return obj
+        raise TypeError
+
+    frame = inspect.currentframe().f_back
+
+    allowed_types = (GeoDataFrame, GeoSeries, Geometry, RasterDataset)
+
+    local_gdfs = {}
     while True:
-        local_gdfs = {}
         for name, value in frame.f_locals.items():
-            if isinstance(value, GeoDataFrame):
+            if isinstance(value, GeoDataFrame) and len(value):
                 local_gdfs[name] = value
                 continue
-            if not to_gdf:
+            if not convert:
                 continue
-            try:
-                if hasattr(value, "__len__") and not len(value):
-                    continue
-            except TypeError:
+
+            if isinstance(value, dict) or hasattr(value, "__dict__"):
+                # add dicts or classes with GeoDataFrames to kwargs
+                for key, value in as_dict(value).items():
+                    if isinstance(value, allowed_types):
+                        gdf = clean_geoms(to_gdf_func(value))
+                        if len(gdf):
+                            local_gdfs[key] = gdf
+
+                    elif isinstance(value, dict) or hasattr(value, "__dict__"):
+                        try:
+                            for k, v in value.items():
+                                if isinstance(v, allowed_types):
+                                    gdf = clean_geoms(to_gdf_func(v))
+                                    if len(gdf):
+                                        local_gdfs[k] = gdf
+                        except Exception:
+                            # no need to raise here
+                            pass
+
                 continue
-            #
             try:
                 gdf = clean_geoms(to_gdf_func(value))
                 if len(gdf):
                     local_gdfs[name] = gdf
+                continue
             except Exception:
                 pass
 
@@ -462,10 +512,6 @@ def explore_locals(*gdfs, to_gdf: bool = True, **kwargs):
         if not frame:
             break
 
-    mask = kwargs.pop("mask", None)
-    if mask is not None:
-        local_gdfs = {name: gdf.clip(mask) for name, gdf in local_gdfs.items()}
-
     explore(*gdfs, **local_gdfs, **kwargs)
 
 

sgis/parallel/parallel.py

@@ -2,6 +2,7 @@ import functools
 import inspect
 import itertools
 import multiprocessing
+import warnings
 from collections.abc import Callable, Collection, Iterable
 from pathlib import Path
 from typing import Any
@@ -102,6 +103,7 @@ class Parallel:
         iterable: Collection,
         args: tuple | None = None,
         kwargs: dict | None = None,
+        chunksize: int = 1,
     ) -> list[Any]:
         """Run functions in parallel with items of an iterable as 0th arguemnt.
 
@@ -185,7 +187,11 @@ class Parallel:
             with multiprocessing.get_context(self.context).Pool(
                 processes, maxtasksperchild=self.maxtasksperchild, **self.kwargs
             ) as pool:
-                return pool.map(func_with_kwargs, iterable)
+                try:
+                    return pool.map(func_with_kwargs, iterable, chunksize=chunksize)
+                except Exception as e:
+                    pool.terminate()
+                    raise e
 
         with joblib.Parallel(
             n_jobs=processes, backend=self.backend, **self.kwargs
@@ -198,6 +204,7 @@ class Parallel:
         iterable: Collection[Iterable[Any]],
         args: tuple | None = None,
         kwargs: dict | None = None,
+        chunksize: int = 1,
     ) -> list[Any]:
         """Run functions in parallel where items of the iterable are unpacked.
 
@@ -282,7 +289,11 @@ class Parallel:
             with multiprocessing.get_context(self.context).Pool(
                 processes, maxtasksperchild=self.maxtasksperchild, **self.kwargs
             ) as pool:
-                return pool.starmap(func_with_kwargs, iterable)
+                try:
+                    return pool.starmap(func_with_kwargs, iterable, chunksize=chunksize)
+                except Exception as e:
+                    pool.terminate()
+                    raise e
 
         with joblib.Parallel(
             n_jobs=processes, backend=self.backend, **self.kwargs
@@ -363,22 +374,30 @@ class Parallel:
 
         Args:
             in_data: Dictionary with dataset names as keys and file paths or
-                (Geo)DataFrames as values.
+                (Geo)DataFrames as values. Note that the files will be read
+                in parallel if file paths are used.
             out_data: Either a single folder path or a dictionary with same keys as
-                'in_data' and folder paths as values. If a single folder is given,
+                'in_data' and folder paths as values. If a single folder is passed,
                 the 'in_data' keys will be used as subfolders.
-            year: Year of the municipality numbers.
+            municipalities: GeoDataFrame of municipalities (or similar) of which to
+                split the data by.
+            with_neighbors: If True, the resulting data will include
+                neighbor municipalities, as well as the munipality itself.
+                Defaults to False.
             funcdict: Dictionary with the keys of 'in_data' and functions as values.
                 The functions should take a GeoDataFrame as input and return a
                 GeoDataFrame. The function will be excecuted before the right after
                 the data is read.
             file_type: Defaults to parquet.
             muni_number_col: String column name with municipality
-                number/identifier. Defaults to KOMMUNENR.
+                number/identifier. Defaults to KOMMUNENR. If the column is not present
+                in the data to be split, the data will be intersected with the
+                municipalities.
             strict: If False (default), the dictionaries 'out_data' and 'funcdict' does
                 not have to have the same length as 'in_data'.
             write_empty: If False (default), municipalities with no data will be skipped.
                 If True, an empty parquet file will be written.
+            clip: If True (default), the data will be clipped.
         """
         shared_kwds = {
             "municipalities": municipalities,
@@ -389,6 +408,7 @@ class Parallel:
             "clip": clip,
             "max_rows_per_chunk": max_rows_per_chunk,
             "processes_in_clip": processes_in_clip,
+            "strict": strict,
         }
 
         if isinstance(out_data, (str, Path)):
@@ -512,6 +532,7 @@ def write_municipality_data(
     clip: bool = True,
     max_rows_per_chunk: int = 150_000,
     processes_in_clip: int = 1,
+    strict: bool = True,
 ) -> None:
     write_func = (
         _write_neighbor_municipality_data
@@ -530,6 +551,7 @@ def write_municipality_data(
         clip=clip,
         max_rows_per_chunk=max_rows_per_chunk,
         processes_in_clip=processes_in_clip,
+        strict=strict,
     )
 
 
@@ -540,6 +562,7 @@ def _validate_data(data: str | list[str]) -> str:
         return data[0]
     elif not isinstance(data, GeoDataFrame):
         raise TypeError("'data' Must be a file path or a GeoDataFrame. Got", type(data))
+    return data
 
 
 def _get_out_path(out_folder, muni, file_type):
@@ -557,6 +580,7 @@ def _write_municipality_data(
     clip: bool = True,
     max_rows_per_chunk: int = 150_000,
     processes_in_clip: int = 1,
+    strict: bool = True,
 ) -> None:
     data = _validate_data(data)
 
@@ -583,6 +607,7 @@ def _write_municipality_data(
         clip,
         max_rows_per_chunk,
         processes_in_clip=processes_in_clip,
+        strict=strict,
     )
 
     for muni in municipalities[muni_number_col]:
@@ -612,6 +637,7 @@ def _write_neighbor_municipality_data(
     clip: bool = True,
     max_rows_per_chunk: int = 150_000,
     processes_in_clip: int = 1,
+    strict: bool = True,
 ) -> None:
     data = _validate_data(data)
 
@@ -628,6 +654,7 @@ def _write_neighbor_municipality_data(
         clip,
         max_rows_per_chunk,
         processes_in_clip,
+        strict=strict,
     )
 
     if municipalities.index.name != muni_number_col:
@@ -659,6 +686,7 @@ def _fix_missing_muni_numbers(
     clip,
     max_rows_per_chunk,
     processes_in_clip,
+    strict,
 ):
     if muni_number_col in gdf and gdf[muni_number_col].notna().all():
         if municipalities is None:
@@ -666,8 +694,13 @@ def _fix_missing_muni_numbers(
         if diffs := set(gdf[muni_number_col].values).difference(
             set(municipalities[muni_number_col].values)
         ):
-            raise ValueError(f"Different municipality numbers: {diffs}")
-
+            message = (
+                f"Different municipality numbers: {diffs}. Set 'strict=False' to ignore"
+            )
+            if strict:
+                raise ValueError(message)
+            else:
+                warnings.warn(message)
         return gdf
 
     if municipalities is None:
@@ -694,12 +727,11 @@ def _fix_missing_muni_numbers(
         if not clip:
             notna_anymore = isna.sjoin(municipalities).drop(columns="index_right")
         else:
-            notna_anymore = intersect_by_municipalities(
+            notna_anymore = parallel_overlay(
                 isna,
-                municipalities,
-                muni_number_col,
-                max_rows_per_chunk,
-                processes_in_clip,
+                municipalities[[muni_number_col, municipalities._geometry_column_name]],
+                processes=processes_in_clip,
+                max_rows_per_chunk=max_rows_per_chunk,
             )
 
         return pd.concat([notna, notna_anymore], ignore_index=True)
@@ -707,40 +739,46 @@ def _fix_missing_muni_numbers(
     if not clip:
         return gdf.sjoin(municipalities).drop(columns="index_right")
     else:
-        return intersect_by_municipalities(
+        return parallel_overlay(
             gdf,
-            municipalities,
-            muni_number_col,
-            max_rows_per_chunk,
-            processes_in_clip,
+            municipalities[[muni_number_col, municipalities._geometry_column_name]],
+            processes=processes_in_clip,
+            max_rows_per_chunk=max_rows_per_chunk,
         )
 
 
-def intersect_by_municipalities(
-    df: GeoDataFrame,
-    municipalities: GeoDataFrame,
-    muni_number_col: str,
+def parallel_overlay(
+    df1: GeoDataFrame,
+    df2: GeoDataFrame,
+    # muni_number_col: str,
+    processes: int,
     max_rows_per_chunk: int,
-    processes_in_clip: int,
+    backend: str = "loky",
+    **kwargs,
 ) -> GeoDataFrame:
-    if len(df) < max_rows_per_chunk:
-        return clean_overlay(df, municipalities)
+    # df2 = df2[[muni_number_col, df2._geometry_column_name]]
 
-    municipalities = municipalities.dissolve(by=muni_number_col, as_index=False)
+    if len(df1) < max_rows_per_chunk:
+        return clean_overlay(df1, df2, **kwargs)
 
-    n_chunks = len(df) // max_rows_per_chunk
-    chunks = np.array_split(np.arange(len(df)), n_chunks)
+    # df2 = df2.dissolve(by=muni_number_col, as_index=False)
 
-    x_mapper = dict(enumerate(df.centroid))
-    sorted_xs = dict(reversed(sorted(x_mapper.items(), key=lambda item: item[1])))
-    df = df.iloc[list(sorted_xs)]
+    n_chunks = len(df1) // max_rows_per_chunk
+    chunks = np.array_split(np.arange(len(df1)), n_chunks)
 
-    df_chunked: list[GeoDataFrame] = [df.iloc[chunk] for chunk in chunks]
+    try:
+        x_mapper = dict(enumerate(df1.centroid))
+        sorted_xs = dict(reversed(sorted(x_mapper.items(), key=lambda item: item[1])))
+        df1 = df1.iloc[list(sorted_xs)]
+    except TypeError:
+        pass
 
-    out = Parallel(processes_in_clip, backend="loky").map(
+    df1_chunked: list[GeoDataFrame] = [df1.iloc[chunk] for chunk in chunks]
+
+    out = Parallel(processes, backend=backend).map(
         _clean_intersection,
-        df_chunked,
-        args=(municipalities,),
+        df1_chunked,
+        args=(df2,),
     )
     return pd.concat(out, ignore_index=True)
 

sgis/raster/torchgeo.py

@@ -8,6 +8,7 @@ import rasterio.merge
 from rasterio.io import DatasetReader
 from rasterio.vrt import WarpedVRT
 from torchgeo.datasets.geo import RasterDataset
+from torchgeo.datasets.sentinel import Sentinel2 as TorchgeoSentinel2
 
 
 try:
@@ -28,28 +29,16 @@ from ..io.opener import opener
 from .bands import SENTINEL2_FILENAME_REGEX
 
 
-class DaplaRasterDataset(RasterDataset):
-    """Custom version of torchgeo's class that works in and outside of Dapla (stat norway)."""
+class GCSRasterDataset(RasterDataset):
+    """Wrapper around torchgeo's RasterDataset that works in and outside of Dapla (stat norway)."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         if is_dapla():
             [file.close() for file in self.files]
 
-    def _get_gcs_paths(self, paths: str | Iterable[str], fs=None) -> set[str]:
-        if fs is None:
-            fs = dp.FileClient.get_gcs_file_system()
-
-        # Using set to remove any duplicates if directories are overlapping
-        out_paths: set[str] = set()
-        for path in paths:
-            pathname = os.path.join(path, "**", self.filename_glob)
-            if is_dapla():
-                out_paths |= {x for x in fs.glob(pathname, recursive=True) if "." in x}
-        return out_paths
-
     @property
-    def files(self) -> set[str] | set[GCSFile]:
+    def files(self) -> set[GCSFile] | set[str]:
         """A list of all files in the dataset.
 
         Returns:
@@ -64,7 +53,12 @@ class DaplaRasterDataset(RasterDataset):
 
         if is_dapla():
             fs = dp.FileClient.get_gcs_file_system()
-            files = {fs.open(x) for x in self._get_gcs_paths(paths, fs)}
+            files: set[GCSFile] = {
+                fs.open(x)
+                for x in _get_gcs_paths(
+                    paths, filename_glob=self.filename_glob, file_system=fs
+                )
+            }
             return files
 
         # Using set to remove any duplicates if directories are overlapping
@@ -107,7 +101,24 @@ class DaplaRasterDataset(RasterDataset):
                 return src
 
 
-class Sentinel2(DaplaRasterDataset):
+def _get_gcs_paths(
+    paths: str | Iterable[str], filename_glob: str, file_system=None
+) -> set[str]:
+    if file_system is None:
+        file_system = dp.FileClient.get_gcs_file_system()
+
+    # Using set to remove any duplicates if directories are overlapping
+    out_paths: set[str] = set()
+    for path in paths:
+        pathname = os.path.join(path, "**", filename_glob)
+        if is_dapla():
+            out_paths |= {
+                x for x in file_system.glob(pathname, recursive=True) if "." in x
+            }
+    return out_paths
+
+
+class Sentinel2(GCSRasterDataset):
     """Works like torchgeo's Sentinel2, with custom regexes."""
 
     date_format: str = "%Y%m%d"
@@ -115,9 +126,6 @@ class Sentinel2(DaplaRasterDataset):
 
     filename_regex = SENTINEL2_FILENAME_REGEX
 
-    _indexes = 1
-    _nodata = 0
-
     all_bands = [
         # "B1",
         "B2",
@@ -138,3 +146,5 @@ class Sentinel2(DaplaRasterDataset):
     separate_files = True
 
     cmap: dict[int, tuple[int, int, int, int]] = {}
+
+    plot = TorchgeoSentinel2.plot

{ssb_sgis-0.3.13.dist-info → ssb_sgis-1.0.0.dist-info}/METADATA

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 0.3.13
+Version: 1.0.0
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT
 Author: Statistics Norway
 Author-email: ort@ssb.no
 Requires-Python: >=3.10,<4
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10