ssb-sgis 1.1.11__tar.gz → 1.1.14__tar.gz

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ssb-sgis
-Version: 1.1.11
+Version: 1.1.14
 Summary: GIS functions used at Statistics Norway.
 Home-page: https://github.com/statisticsnorway/ssb-sgis
 License: MIT

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-sgis"
-version = "1.1.11"
+version = "1.1.14"
 description = "GIS functions used at Statistics Norway."
 authors = ["Morten Letnes <morten.letnes@ssb.no>"]
 license = "MIT"

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/src/sgis/geopandas_tools/polygon_operations.py

@@ -425,12 +425,6 @@ def eliminate_by_longest(
     explore_locals(center=_DEBUG_CONFIG["center"])
 
     if not _recurse and len(isolated):
-        if 0:
-            isolated.geometry = isolated.buffer(
-                -PRECISION,
-                resolution=1,
-                join_style=2,
-            )
         out, isolated = _recursively_eliminate_new_neighbors(
             out,
             isolated,

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/src/sgis/io/dapla_functions.py

@@ -109,7 +109,7 @@ def read_geopandas(
     if single_eq_filter:
         try:
             expression = "".join(next(iter(filters))).replace("==", "=")
-            glob_func = _get_glob(file_system)
+            glob_func = _get_glob_func(file_system)
             paths = glob_func(str(Path(gcs_path) / expression))
             if paths:
                 return _read_geopandas_from_iterable(
@@ -543,7 +543,7 @@ def _write_partitioned_geoparquet(
         if df[col].isna().all() and not kwargs.get("schema"):
             raise ValueError("Must specify 'schema' when all rows are NA.")
 
-    glob_func = _get_glob(file_system)
+    glob_func = _get_glob_func(file_system)
 
     if file_system.exists(path) and file_system.isfile(path):
         _remove_file(path, file_system)
@@ -596,7 +596,7 @@ def _write_partitioned_geoparquet(
         executor.map(threaded_write, dfs, paths)
 
 
-def _get_glob(file_system) -> functools.partial:
+def _get_glob_func(file_system) -> functools.partial:
     try:
         return functools.partial(file_system.glob)
     except AttributeError:
@@ -724,9 +724,9 @@ def _read_partitioned_parquet(
     **kwargs,
 ):
     file_system = _get_file_system(file_system, kwargs)
+    glob_func = _get_glob_func(file_system)
 
     if child_paths is None:
-        glob_func = _get_glob(file_system)
         child_paths = list(glob_func(str(Path(path) / "**/*.parquet")))
 
     filters = _filters_to_expression(filters)
@@ -735,18 +735,29 @@ def _read_partitioned_parquet(
         bbox, _ = _get_bounds_parquet_from_open_file(file, file_system)
         return shapely.box(*bbox).intersects(to_shapely(mask))
 
-    def read(path: str) -> pyarrow.Table | None:
-        with file_system.open(path, "rb") as file:
-            if mask is not None and not intersects(file, mask):
-                return
+    def read(child_path: str) -> pyarrow.Table | None:
+        try:
+            with file_system.open(child_path, "rb") as file:
+                if mask is not None and not intersects(file, mask):
+                    return
 
-            # 'get' instead of 'pop' because dict is mutable
-            schema = kwargs.get("schema", pq.read_schema(file))
-            new_kwargs = {
-                key: value for key, value in kwargs.items() if key != "schema"
-            }
+                # 'get' instead of 'pop' because dict is mutable
+                schema = kwargs.get("schema", pq.read_schema(file))
+                new_kwargs = {
+                    key: value for key, value in kwargs.items() if key != "schema"
+                }
 
-            return read_func(file, schema=schema, filters=filters, **new_kwargs)
+                return read_func(file, schema=schema, filters=filters, **new_kwargs)
+        except ArrowInvalid as e:
+            if not len(
+                {
+                    x
+                    for x in glob_func(str(Path(child_path) / "**"))
+                    if not paths_are_equal(child_path, x)
+                }
+            ):
+                raise e
+            # allow not being able to read hard-to-delete empty directories
 
     with ThreadPoolExecutor() as executor:
         results = [
@@ -790,7 +801,7 @@ def paths_are_equal(path1: Path | str, path2: Path | str) -> bool:
 
 
 def get_child_paths(path, file_system) -> list[str]:
-    glob_func = _get_glob(file_system)
+    glob_func = _get_glob_func(file_system)
     return [
         x
         for x in glob_func(str(Path(path) / "**/*.parquet"))

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/src/sgis/maps/explore.py

@@ -828,7 +828,7 @@ class Explore(Map):
             if not len(gdf):
                 continue
 
-            gdf = self._to_single_geom_type(make_all_singlepart(gdf))
+            gdf = self._to_single_geom_type(make_all_singlepart(gdf, ignore_index=True))
 
             if not len(gdf):
                 continue

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/src/sgis/maps/map.py

@@ -729,6 +729,8 @@ class Map:
         """Place the column values into groups."""
         bins = bins.copy()
 
+        assert gdf.index.is_unique
+
         # if equal lenght, convert to integer and check for equality
         if len(bins) == len(self._unique_values):
             if gdf[self._column].isna().all():

{ssb_sgis-1.1.11 → ssb_sgis-1.1.14}/src/sgis/raster/image_collection.py

@@ -2,6 +2,7 @@ import datetime
 import functools
 import glob
 import itertools
+import json
 import os
 import random
 import re
@@ -214,13 +215,14 @@ class PixelwiseResults:
 
     def to_geopandas(self, column: str | list[str] = "value") -> GeoDataFrame:
         """Return GeoDataFrame with pixel geometries and values from the pixelwise operation."""
-        minx, miny = self.bounds[:2]
         resx, resy = _res_as_tuple(self.res)
 
-        minxs = np.full(self.row_indices.shape, minx) + (self.row_indices * resx)
-        minys = np.full(self.col_indices.shape, miny) + (self.col_indices * resy)
+        # work ourselves inwards from the bottom left and top right corners
+        minx, _, _, maxy = self.bounds
+        minxs = np.full(self.col_indices.shape, minx) + (self.col_indices * resx)
+        maxys = np.full(self.row_indices.shape, maxy) - (self.row_indices * resy)
         maxxs = minxs + resx
-        maxys = minys + resy
+        minys = maxys - resy
 
         return GeoDataFrame(
             {
@@ -467,8 +469,8 @@ class _ImageBase:
             regexes = (regexes,)
         return tuple(re.compile(regexes, flags=re.VERBOSE) for regexes in regexes)
 
-    @staticmethod
     def _metadata_to_nested_dict(
+        self,
         metadata: str | Path | os.PathLike | dict | pd.DataFrame | None,
     ) -> dict[str, dict[str, Any]]:
         """Construct metadata dict from dictlike, DataFrame or file path.
@@ -931,14 +933,20 @@ class Band(_ImageBandBase):
             }
 
         if self.metadata:
+            parent = _fix_path(str(Path(self.path).parent))
             if self.path is not None:
                 self.metadata = {
-                    key: value
-                    for key, value in self.metadata.items()
-                    if key == self.path
+                    key: value for key, value in self.metadata.items() if key == parent
                 }
-            this_metadata = self.metadata[self.path]
-            for key, value in this_metadata.items():
+            for key, value in self.metadata.get(parent, {}).items():
+                if key == "bands" and self.band_id in value:
+                    band_metadata = value[self.band_id]
+                    for band_key, band_value in band_metadata.items():
+                        if band_key in dir(self):
+                            setattr(self, f"_{band_key}", band_value)
+                        else:
+                            setattr(self, band_key, band_value)
+                    continue
                 if key in dir(self):
                     setattr(self, f"_{key}", value)
                 else:
@@ -1591,6 +1599,22 @@ class Image(_ImageBandBase):
         else:
             self._all_file_paths = None
 
+        if not self.metadata and "metadata.json" in {
+            Path(x).name for x in self._all_file_paths
+        }:
+            with _open_func(
+                next(
+                    iter(
+                        {
+                            x
+                            for x in self._all_file_paths
+                            if str(x).endswith("metadata.json")
+                        }
+                    )
+                )
+            ) as file:
+                self.metadata = json.load(file)
+
         if df is None:
             if not self._all_file_paths:
                 self._all_file_paths = {self.path}
@@ -1615,12 +1639,10 @@ class Image(_ImageBandBase):
                 key: value for key, value in self.metadata.items() if self.path in key
             }
 
-        if self.metadata:
-            try:
-                metadata = self.metadata[self.path]
-            except KeyError as e:
-                metadata = {}
-            for key, value in metadata.items():
+        if self.metadata.get(self.path, {}):
+            for key, value in self.metadata[self.path].items():
+                if key in {"bands"}:
+                    continue
                 if key in dir(self):
                     setattr(self, f"_{key}", value)
                 else:
@@ -1704,6 +1726,56 @@ class Image(_ImageBandBase):
 
         return self
 
+    def get_image_metadata_dict(self) -> dict:
+        """Creates a nested dict of metadata.
+
+        The dict structure will be:
+
+        {
+            image_path: {
+                image_attribute: value,
+                ...,
+                "bands": {
+                    band_id: {
+                        band_attribute: band_value,
+                    },
+                    ...,
+                }
+            }
+        }
+        """
+        path = self.path
+        metadata = {
+            path: {
+                "bounds": self.bounds,
+                "crs": str(pyproj.CRS(self.crs).to_string()),
+            }
+        }
+        for key in self.metadata_attributes:
+            metadata[path][key] = getattr(self, key)
+
+        metadata[path]["bands"] = {}
+        for band in self:
+            metadata[path]["bands"][band.band_id] = {}
+            for key in band.metadata_attributes:
+                if key in self.metadata_attributes:
+                    continue
+                metadata[path]["bands"][band.band_id][key] = getattr(band, key)
+        return metadata
+
+    def write_image_metadata(self) -> None:
+        """Write file 'metadata.json' under image path.
+
+        The file will be used to give the image attributes
+        and avoid the much slower metadata fetching with rasterio.
+
+        See method 'get_image_metadata_dict' for info on the structure of
+        the json file.
+        """
+        metadata = self.get_image_metadata_dict()
+        with _open_func(str(Path(self.path) / "metadata.json"), "w") as file:
+            json.dump(metadata, file)
+
     def _construct_image_from_bands(
         self, data: Sequence[Band], res: int | None
     ) -> None: