ssb-sgis 1.1.17__py3-none-any.whl → 1.2.1__py3-none-any.whl

sgis/raster/image_collection.py

@@ -41,12 +41,6 @@ from shapely.geometry import MultiPolygon
 from shapely.geometry import Point
 from shapely.geometry import Polygon
 
-try:
-    import dapla as dp
-except ImportError:
-    pass
-
-
 try:
     from google.auth import exceptions
 except ImportError:
@@ -59,12 +53,16 @@ except ImportError:
 
 
 try:
+    from gcsfs import GCSFileSystem
     from gcsfs.core import GCSFile
 except ImportError:
 
     class GCSFile:
         """Placeholder."""
 
+    class GCSFileSystem:
+        """Placeholder."""
+
 
 try:
     from rioxarray.exceptions import NoDataInBounds
@@ -103,7 +101,6 @@ from ..geopandas_tools.conversion import to_geoseries
 from ..geopandas_tools.conversion import to_shapely
 from ..geopandas_tools.general import get_common_crs
 from ..helpers import _fix_path
-from ..helpers import get_all_files
 from ..helpers import get_numpy_func
 from ..helpers import is_method
 from ..helpers import is_property
@@ -132,22 +129,23 @@ from .zonal import _zonal_post
 if is_dapla():
 
     def _ls_func(*args, **kwargs) -> list[str]:
-        return dp.FileClient.get_gcs_file_system().ls(*args, **kwargs)
+        return GCSFileSystem().ls(*args, **kwargs)
 
     def _glob_func(*args, **kwargs) -> list[str]:
-        return dp.FileClient.get_gcs_file_system().glob(*args, **kwargs)
+        return GCSFileSystem().glob(*args, **kwargs)
 
     def _open_func(*args, **kwargs) -> GCSFile:
-        return dp.FileClient.get_gcs_file_system().open(*args, **kwargs)
-
-    def _read_parquet_func(*args, **kwargs) -> list[str]:
-        return dp.read_pandas(*args, **kwargs)
+        return GCSFileSystem().open(*args, **kwargs)
 
 else:
-    _ls_func = functools.partial(get_all_files, recursive=False)
+
+    def _ls_func(path):
+        return glob.glob(str(Path(path) / "**"), recursive=False)
+
+    def _glob_func(path, **kwargs):
+        return glob.glob(path, recursive=True, **kwargs)
+
     _open_func = open
-    _glob_func = glob.glob
-    _read_parquet_func = pd.read_parquet
 
 DATE_RANGES_TYPE = (
     tuple[str | pd.Timestamp | None, str | pd.Timestamp | None]
@@ -181,15 +179,12 @@ ALLOWED_INIT_KWARGS = [
 _LOAD_COUNTER: int = 0
 
 
-def _get_child_paths_threaded(data: Sequence[str]) -> set[str]:
-    with ThreadPoolExecutor() as executor:
-        all_paths: Iterator[set[str]] = executor.map(_ls_func, data)
-    return set(itertools.chain.from_iterable(all_paths))
-
-
 @dataclass
 class PixelwiseResults:
-    """Container of pixelwise results to be converted to numpy/geopandas."""
+    """Container of pixelwise results to be converted to dict/tuple/numpy/geopandas.
+
+    Not to be initialised by user.
+    """
 
     row_indices: np.ndarray
     col_indices: np.ndarray
@@ -213,8 +208,34 @@ class PixelwiseResults:
             )
         }
 
+    def to_pandas(self, column: str | list[str] = "value") -> GeoDataFrame:
+        """Return DataFrame with 2 dim index and values from the pixelwise operation."""
+        return pd.DataFrame(
+            {
+                **(
+                    {col: [x[i] for x in self.results] for i, col in enumerate(column)}
+                    if not isinstance(column, str)
+                    else {column: self.results}
+                )
+            },
+            index=[self.row_indices, self.col_indices],
+        )
+
+    @property
+    def is_empty(self) -> bool:
+        """Returns True if all band arrays in all images have shape (0,)."""
+        return not any(x for x in self.row_indices.shape) and not any(
+            x for x in self.col_indices.shape
+        )
+
     def to_geopandas(self, column: str | list[str] = "value") -> GeoDataFrame:
         """Return GeoDataFrame with pixel geometries and values from the pixelwise operation."""
+        if self.is_empty:
+            if isinstance(column, str):
+                return GeoDataFrame({"geometry": [], column: []}, crs=self.crs)
+            return GeoDataFrame(
+                {"geometry": [], **{col: [] for col in column}}, crs=self.crs
+            )
         resx, resy = _res_as_tuple(self.res)
 
         # work ourselves inwards from the bottom left and top right corners
@@ -223,7 +244,6 @@ class PixelwiseResults:
         maxys = np.full(self.row_indices.shape, maxy) - (self.row_indices * resy)
         maxxs = minxs + resx
         minys = maxys - resy
-
         return GeoDataFrame(
             {
                 **(
@@ -244,6 +264,8 @@ class PixelwiseResults:
 
     def to_numpy(self) -> np.ndarray | tuple[np.ndarray, ...]:
         """Reshape pixelwise results to 2d numpy arrays in the shape of the full arrays of the image bands."""
+        if self.is_empty:
+            return tuple(np.array([]) for _ in range(len(self.results)))
         try:
             n_out_arrays = len(next(iter(self.results)))
         except TypeError:
@@ -310,7 +332,7 @@ class ImageCollectionGroupBy:
     ) -> "ImageCollection":
         """Merge each group into separate Bands per band_id, returned as an ImageCollection."""
         images = self._run_func_for_collection_groups(
-            _merge_by_band,
+            _merge_by_band_as_func,
             method=method,
             bounds=bounds,
             as_int=as_int,
@@ -342,7 +364,7 @@ class ImageCollectionGroupBy:
     ) -> "Image":
         """Merge each group into a single Band, returned as combined Image."""
         bands: list[Band] = self._run_func_for_collection_groups(
-            _merge,
+            _merge_as_func,
             method=method,
             bounds=bounds,
             as_int=as_int,
@@ -405,7 +427,10 @@ class BandMasking:
 
 
 class None_:
-    """Default None for args that are not allowed to be None."""
+    """Default None for args that should not be None.
+
+    In order to raise error only in some cases.
+    """
 
     def __new__(cls) -> None:
         """Always returns None."""
@@ -449,6 +474,27 @@ class _ImageBase:
         self._from_array = False
         self._from_geopandas = False
 
+    @property
+    def path(self) -> str:
+        try:
+            return self._path
+        except AttributeError as e:
+            raise PathlessImageError(self) from e
+
+    @property
+    def res(self) -> int:
+        """Pixel resolution."""
+        return self._res
+
+    @abstractmethod
+    def union_all(self) -> Polygon | MultiPolygon:
+        pass
+
+    def assign(self, **kwargs) -> "_ImageBase":
+        for key, value in kwargs.items():
+            self._safe_setattr(key, value)
+        return self
+
     def _safe_setattr(
         self, key: str, value: Any, error_obj: Exception | None = None
     ) -> None:
@@ -471,16 +517,15 @@ class _ImageBase:
 
     def _metadata_to_nested_dict(
         self,
-        metadata: str | Path | os.PathLike | dict | pd.DataFrame | None,
+        metadata: dict | pd.DataFrame | None,
     ) -> dict[str, dict[str, Any]]:
-        """Construct metadata dict from dictlike, DataFrame or file path.
+        """Construct metadata dict from dictlike or DataFrame.
+
+        First level keys are are file paths, second level keys are attributes.
 
         Extract metadata value:
         >>> self.metadata[self.path]['cloud_cover_percentage']
         """
-        if isinstance(metadata, (str | Path | os.PathLike)):
-            metadata = _read_parquet_func(metadata)
-
         if isinstance(metadata, pd.DataFrame):
 
             def is_scalar(x) -> bool:
@@ -491,7 +536,7 @@ class _ImageBase:
                 """Convert to None rowwise because pandas doesn't always."""
                 return x if not (is_scalar(x) and pd.isna(x)) else None
 
-            # to nested dict because pandas indexing gives rare KeyError with long strings
+            # to nested dict instead of pandas because pandas indexing gives rare KeyError with long strings
             return {
                 _fix_path(path): {
                     attr: na_to_none(value) for attr, value in row.items()
@@ -516,40 +561,13 @@ class _ImageBase:
 
     @property
     def _common_init_kwargs_after_load(self) -> dict:
+        """Some attributes can be wrong after loading the image."""
         return {
             k: v
             for k, v in self._common_init_kwargs.items()
             if k not in {"res", "metadata"}
         }
 
-    @property
-    def path(self) -> str:
-        try:
-            return self._path
-        except AttributeError as e:
-            raise PathlessImageError(self) from e
-
-    @property
-    def res(self) -> int:
-        """Pixel resolution."""
-        # if self._res is None:
-        #     if self.has_array:
-        #         self._res = _get_res_from_bounds(self.bounds, self.values.shape)
-        #     else:
-        #         with opener(self.path) as file:
-        #             with rasterio.open(file) as src:
-        #                 self._res = src.res
-        return self._res
-
-    @abstractmethod
-    def union_all(self) -> Polygon | MultiPolygon:
-        pass
-
-    def assign(self, **kwargs) -> "_ImageBase":
-        for key, value in kwargs.items():
-            self._safe_setattr(key, value)
-        return self
-
     def _name_regex_searcher(
         self, group: str, patterns: tuple[re.Pattern]
     ) -> str | None:
@@ -648,21 +666,22 @@ class _ImageBase:
 class _ImageBandBase(_ImageBase):
     """Common parent class of Image and Band."""
 
-    def intersects(
-        self, geometry: GeoDataFrame | GeoSeries | Geometry | tuple | _ImageBase
-    ) -> bool:
-        if hasattr(geometry, "crs") and not pyproj.CRS(self.crs).equals(
-            pyproj.CRS(geometry.crs)
-        ):
-            raise ValueError(f"crs mismatch: {self.crs} and {geometry.crs}")
-        return self.union_all().intersects(to_shapely(geometry))
-
     def union_all(self) -> Polygon:
         try:
             return box(*self.bounds)
         except TypeError:
             return Polygon()
 
+    def intersects(
+        self, geometry: GeoDataFrame | GeoSeries | Geometry | tuple | _ImageBase
+    ) -> bool:
+        crs_mismatch: bool = hasattr(geometry, "crs") and not pyproj.CRS(
+            self.crs
+        ).equals(pyproj.CRS(geometry.crs))
+        if crs_mismatch:
+            raise ValueError(f"crs mismatch: {self.crs} and {geometry.crs}")
+        return self.union_all().intersects(to_shapely(geometry))
+
     @property
     def centroid(self) -> Point:
         """Centerpoint of the object."""
@@ -672,19 +691,19 @@ class _ImageBandBase(_ImageBase):
     def year(self) -> str:
         if hasattr(self, "_year") and self._year:
             return self._year
-        return str(self.date)[:4]
+        return pd.to_datetime(self.date).year
 
     @property
     def month(self) -> str:
         if hasattr(self, "_month") and self._month:
             return self._month
-        return str(self.date).replace("-", "").replace("/", "")[4:6]
+        return pd.to_datetime(self.date).month
 
     @property
     def day(self) -> str:
         if hasattr(self, "_day") and self._day:
             return self._day
-        return str(self.date).replace("-", "").replace("/", "")[6:8]
+        return pd.to_datetime(self.date).day
 
     @property
     def name(self) -> str | None:
@@ -744,7 +763,7 @@ class _ImageBandBase(_ImageBase):
             results = None
             for i, file_content in enumerate(file_contents.values()):
                 if isinstance(value, str) and value in dir(self):
-                    # method or a hardcoded value
+                    # is method or a hardcoded value
                     value: Callable | Any = getattr(self, value)
 
                 if callable(value):
@@ -1305,13 +1324,14 @@ class Band(_ImageBandBase):
 
         self._path = _fix_path(str(path))
 
-    def apply(self, func: Callable, **kwargs) -> "Band":
-        """Apply a function to the Band."""
-        results = func(self, **kwargs)
+    def apply(self, func_: Callable, copy: bool = True, **kwargs) -> "Band":
+        """Apply a function to the array."""
+        copied = self.copy() if copy else self
+        results = func_(copied, **kwargs)
         if isinstance(results, Band):
             return results
-        self.values = results
-        return self
+        copied.values = results
+        return copied
 
     def sample(self, size: int = 1000, mask: Any = None, **kwargs) -> "Image":
         """Take a random spatial sample area of the Band."""
@@ -1580,8 +1600,8 @@ class Image(_ImageBandBase):
                 f"'data' must be string, Path-like or a sequence of Band. Got {data}"
             )
 
-        self._res = res if not (callable(res) and res() is None) else None
         self._path = _fix_path(data)
+        self._res = res if not (callable(res) and res() is None) else None
 
         if all_file_paths is None and self.path:
             self._all_file_paths = _get_all_file_paths(self.path)
@@ -1592,9 +1612,11 @@ class Image(_ImageBandBase):
         else:
             self._all_file_paths = None
 
-        if not self.metadata and "metadata.json" in {
-            Path(x).name for x in self._all_file_paths
-        }:
+        if (
+            self.metadata is None
+            or not len(self.metadata)
+            and "metadata.json" in {Path(x).name for x in self._all_file_paths}
+        ):
             with _open_func(
                 next(
                     iter(
@@ -1697,7 +1719,7 @@ class Image(_ImageBandBase):
 
         with joblib.Parallel(n_jobs=self.processes, backend="threading") as parallel:
             parallel(
-                joblib.delayed(_load_band)(
+                joblib.delayed(_load_as_func)(
                     band,
                     bounds=bounds,
                     indexes=indexes,
@@ -1787,10 +1809,11 @@ class Image(_ImageBandBase):
             if len(band_values) > 1:
                 raise ValueError(f"Different {key} values in bands: {band_values}")
             elif len(band_values):
+                value = next(iter(band_values))
                 try:
-                    setattr(self, key, next(iter(band_values)))
+                    setattr(self, key, value)
                 except AttributeError:
-                    setattr(self, f"_{key}", next(iter(band_values)))
+                    setattr(self, f"_{key}", value)
 
     def copy(self) -> "Image":
         """Copy the instance and its attributes."""
@@ -1799,12 +1822,18 @@ class Image(_ImageBandBase):
             band._mask = copied._mask
         return copied
 
-    def apply(self, func: Callable, **kwargs) -> "Image":
+    def apply(self, func_: Callable, copy: bool = True, **kwargs) -> "Image":
         """Apply a function to each band of the Image."""
-        with joblib.Parallel(n_jobs=self.processes, backend="loky") as parallel:
-            parallel(joblib.delayed(_band_apply)(band, func, **kwargs) for band in self)
+        copied = self.copy() if copy else self
+        with joblib.Parallel(n_jobs=copied.processes, backend="loky") as parallel:
+            results = parallel(joblib.delayed(func_)(band, **kwargs) for band in copied)
+        if all(isinstance(x, Band) for x in results):
+            copied._bands = results
+        elif all(hasattr(x, "shape") for x in results):
+            for band, arr in zip(copied.bands, results, strict=True):
+                band.values = arr
 
-        return self
+        return copied
 
     def ndvi(
         self, red_band: str, nir_band: str, padding: int = 0, copy: bool = True
@@ -2132,40 +2161,36 @@ class ImageCollection(_ImageBase):
 
         if hasattr(data, "__iter__") and not isinstance(data, str):
             self._path = None
-            if all(isinstance(x, Image) for x in data):
+            data_is_images: bool = all(isinstance(x, Image) for x in data)
+            if data_is_images:
                 self.images = [x.copy() for x in data]
                 return
-            elif all(isinstance(x, (str | Path | os.PathLike)) for x in data):
-                # adding band paths (asuming 'data' is a sequence of image paths)
+            data_is_paths: bool = all(
+                isinstance(x, (str | Path | os.PathLike)) for x in data
+            )
+            if data_is_paths:
                 try:
+                    # adding band paths (asuming 'data' is a sequence of image paths)
                     self._all_file_paths = _get_child_paths_threaded(data) | {
                         _fix_path(x) for x in data
                     }
                 except FileNotFoundError as e:
                     if _from_root:
-                        raise TypeError(
+                        raise ValueError(
                             "When passing 'root', 'data' must be a sequence of image file names that have 'root' as parent path."
                         ) from e
                     raise e
-                if self.level:
-                    self._all_file_paths = {
-                        path for path in self._all_file_paths if self.level in path
-                    }
-                self._df = self._create_metadata_df(self._all_file_paths)
-                return
 
-        if not isinstance(data, (str | Path | os.PathLike)):
+        elif isinstance(data, (str | Path | os.PathLike)):
+            self._path = _fix_path(str(data))
+            self._all_file_paths = _get_all_file_paths(self.path)
+        else:
             raise TypeError("'data' must be string, Path-like or a sequence of Image.")
 
-        self._path = _fix_path(str(data))
-
-        self._all_file_paths = _get_all_file_paths(self.path)
-
         if self.level:
             self._all_file_paths = {
                 path for path in self._all_file_paths if self.level in path
             }
-
         self._df = self._create_metadata_df(self._all_file_paths)
 
     def groupby(
@@ -2230,16 +2255,15 @@ class ImageCollection(_ImageBase):
                 pass
         return copied
 
-    def apply(self, func: Callable, **kwargs) -> "ImageCollection":
-        """Apply a function to all bands in each image of the collection."""
-        with joblib.Parallel(n_jobs=self.processes, backend="loky") as parallel:
-            parallel(
-                joblib.delayed(_band_apply)(band, func, **kwargs)
-                for img in self
-                for band in img
+    def apply(self, func_: Callable, copy: bool = True, **kwargs) -> "ImageCollection":
+        """Apply a function to each image of the collection."""
+        copied = self.copy() if copy else self
+        with joblib.Parallel(n_jobs=copied.processes, backend="loky") as parallel:
+            copied.images = parallel(
+                joblib.delayed(func_)(img, **kwargs) for img in copied
             )
 
-        return self
+        return copied
 
     def pixelwise(
         self,
@@ -2254,6 +2278,18 @@ class ImageCollection(_ImageBase):
         The function should take a 1d array as first argument. This will be
         the pixel values for all bands in all images in the collection.
         """
+        if not len(self):
+            return PixelwiseResults(
+                np.array([]),
+                np.array([]),
+                np.array([]),
+                shape=(0,),
+                res=self.res,
+                bounds=None,
+                crs=None,
+                nodata=self.nodata or np.nan,
+            )
+
         values = np.array([band.values for img in self for band in img])
 
         if (
@@ -2594,7 +2630,7 @@ class ImageCollection(_ImageBase):
                 )
 
             parallel(
-                joblib.delayed(_load_band)(
+                joblib.delayed(_load_as_func)(
                     band,
                     bounds=bounds,
                     indexes=indexes,
@@ -2798,25 +2834,6 @@ class ImageCollection(_ImageBase):
         return combine_by_coords(list(xarrs.values()))
         # return Dataset(xarrs)
 
-    def to_geopandas(self, column: str = "value") -> dict[str, GeoDataFrame]:
-        """Convert each band in each Image to a GeoDataFrame."""
-        out = {}
-        i = 0
-        for img in self:
-            for band in img:
-                i += 1
-                try:
-                    name = band.name
-                except AttributeError:
-                    name = None
-
-                if name is None:
-                    name = f"{self.__class__.__name__}({i})"
-
-                if name not in out:
-                    out[name] = band.to_geopandas(column=column)
-        return out
-
     def sample(self, n: int = 1, size: int = 500) -> "ImageCollection":
         """Sample one or more areas of a given size and set this as mask for the images."""
         unioned = self.union_all()
@@ -2894,6 +2911,11 @@ class ImageCollection(_ImageBase):
         if isinstance(item, int):
             return self.images[item]
 
+        if isinstance(item, str):
+            return self._metadata_attribute_collection_type(
+                [getattr(img, item) for img in self]
+            )
+
         if isinstance(item, slice):
             copied = self.copy()
             copied.images = copied.images[item]
@@ -2922,7 +2944,6 @@ class ImageCollection(_ImageBase):
         # check for base bool and numpy bool
         if all("bool" in str(type(x)) for x in item):
             copied.images = [img for x, img in zip(item, copied, strict=True) if x]
-
         else:
             copied.images = [copied.images[i] for i in item]
         return copied
@@ -2981,7 +3002,7 @@ class ImageCollection(_ImageBase):
         )
 
     @images.setter
-    def images(self, new_value: list["Image"]) -> list["Image"]:
+    def images(self, new_value: list["Image"]) -> None:
         new_value = list(new_value)
         if not new_value:
             self._images = new_value
@@ -2994,10 +3015,20 @@ class ImageCollection(_ImageBase):
                 img._bands = [new_value[i]]
                 new_images.append(img)
             self._images = new_images
-            return
-        if not all(isinstance(x, Image) for x in new_value):
+        elif all(hasattr(x, "shape") for x in new_value):
+            for img, arr in zip(self._images, new_value, strict=True):
+                img._bands = [
+                    Band(
+                        arr,
+                        bounds=img.bounds,
+                        crs=img.crs,
+                        **img._common_init_kwargs_after_load,
+                    )
+                ]
+        elif not all(isinstance(x, Image) for x in new_value):
             raise TypeError("images should be a sequence of Image.")
-        self._images = new_value
+        else:
+            self._images = new_value
 
     def union_all(self) -> Polygon | MultiPolygon:
         """(Multi)Polygon representing the union of all image bounds."""
@@ -3098,7 +3129,7 @@ class ImageCollection(_ImageBase):
                 root = f" root='{next(iter(parents))}',"
         else:
             data = [img for img in self]
-        return f"{self.__class__.__name__}({data},{root} res={self.res}, level='{self.level}')"
+        return f"{self.__class__.__name__}({data},{root} res={self.res}, level='{self.level}', processes={self.processes})"
 
 
 class Sentinel2Config:
@@ -3171,7 +3202,6 @@ class Sentinel2Config:
                 xml_file,
                 (
                     r'<BOA_QUANTIFICATION_VALUE unit="none">(\d+)</BOA_QUANTIFICATION_VALUE>',
-                    # r'<BOA_QUANTIFICATION_VALUE unit="none">-?(\d+)</BOA_QUANTIFICATION_VALUE>',
                     r'<QUANTIFICATION_VALUE unit="none">?(\d+)</QUANTIFICATION_VALUE>',
                 ),
             )
@@ -3390,21 +3420,13 @@ def _clip_xarray(
 
 
 def _get_all_file_paths(path: str) -> set[str]:
-    if is_dapla():
-        return {_fix_path(x) for x in sorted(set(_glob_func(path + "/**")))}
-    else:
-        return {
-            _fix_path(x)
-            for x in sorted(
-                set(
-                    _glob_func(path + "/**")
-                    + _glob_func(path + "/**/**")
-                    + _glob_func(path + "/**/**/**")
-                    + _glob_func(path + "/**/**/**/**")
-                    + _glob_func(path + "/**/**/**/**/**")
-                )
-            )
-        }
+    return {_fix_path(x) for x in sorted(set(_glob_func(path + "/**")))}
+
+
+def _get_child_paths_threaded(data: Sequence[str]) -> set[str]:
+    with ThreadPoolExecutor() as executor:
+        all_paths: Iterator[set[str]] = executor.map(_ls_func, data)
+    return set(itertools.chain.from_iterable(all_paths))
 
 
 def _get_images(
@@ -3584,19 +3606,19 @@ def _read_mask_array(self: Band | Image, **kwargs) -> np.ndarray:
     return boolean_mask
 
 
-def _load_band(band: Band, **kwargs) -> Band:
+def _load_as_func(band: Band, **kwargs) -> Band:
     return band.load(**kwargs)
 
 
-def _band_apply(band: Band, func: Callable, **kwargs) -> Band:
-    return band.apply(func, **kwargs)
+def _apply_as_func(band: Band | Image, func_: Callable, **kwargs) -> Band:
+    return band.apply(func_, **kwargs)
 
 
-def _merge_by_band(collection: ImageCollection, **kwargs) -> Image:
+def _merge_by_band_as_func(collection: ImageCollection, **kwargs) -> Image:
     return collection.merge_by_band(**kwargs)
 
 
-def _merge(collection: ImageCollection, **kwargs) -> Band:
+def _merge_as_func(collection: ImageCollection, **kwargs) -> Band:
     return collection.merge(**kwargs)
 
 
@@ -3749,4 +3771,4 @@ def pixelwise(
             )
         )
 
-    return nonmissing_row_indices, nonmissing_col_indices, results
+    return (nonmissing_row_indices, nonmissing_col_indices, results)

sgis/raster/indices.py

@@ -4,5 +4,4 @@ import numpy as np
 def ndvi(red: np.ndarray, nir: np.ndarray, padding: int = 0) -> np.ndarray:
     ndvi_values = (nir - red + padding) / (nir + red + padding)
     ndvi_values[(red + nir) == 0] = 0
-
     return ndvi_values