cars 1.0.0rc1__cp313-cp313-musllinux_1_2_i686.whl

cars/applications/dsm_filling/abstract_dsm_filling_app.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+this module contains the abstract dsm filling application class.
+"""
+import logging
+from abc import ABCMeta, abstractmethod
+from typing import Dict
+
+from cars.applications.application import Application
+from cars.applications.application_template import ApplicationTemplate
+
+
+@Application.register("dsm_filling")
+class DsmFilling(ApplicationTemplate, metaclass=ABCMeta):
+    """
+    DsmFilling
+    """
+
+    available_applications: Dict = {}
+    default_application = "bulldozer"
+
+    def __new__(cls, conf=None):  # pylint: disable=W0613
+        """
+        Return the required application
+        :raises:
+         - KeyError when the required application is not registered
+
+        :param orchestrator: orchestrator used
+        :param conf: configuration for filling
+        :return: an application_to_use object
+        """
+
+        dsm_filling_method = cls.default_application
+        if bool(conf) is False:
+            logging.info(
+                "dsm_filling method not specified, default"
+                " {} is used".format(dsm_filling_method)
+            )
+        else:
+            dsm_filling_method = conf["method"]
+
+        if dsm_filling_method not in cls.available_applications:
+            logging.error(
+                "No dsm_filling application named {} registered".format(
+                    dsm_filling_method
+                )
+            )
+            raise KeyError(
+                "No dsm_filling application named {} registered".format(
+                    dsm_filling_method
+                )
+            )
+
+        logging.info(
+            "The DsmFilling {} application will be used".format(
+                dsm_filling_method
+            )
+        )
+
+        return super(DsmFilling, cls).__new__(
+            cls.available_applications[dsm_filling_method]
+        )
+
+    def __init_subclass__(cls, short_name, **kwargs):  # pylint: disable=E0302
+        super().__init_subclass__(**kwargs)
+        cls.available_applications[short_name] = cls
+
+    def __init__(self, conf=None):
+        """
+        Init function of DSM Filling
+
+        :param conf: configuration
+        :return: an application_to_use object
+        """
+
+        super().__init__(conf=conf)
+
+    @abstractmethod
+    def run(self):
+        """
+        Run dsm filling using initial elevation and the current dsm
+        """

cars/applications/dsm_filling/border_interpolation_app.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module contains the border interpolation dsm filling application class.
+"""
+
+import logging
+import os
+import shutil
+import warnings
+
+import numpy as np
+import rasterio as rio
+import scipy
+import skimage
+from json_checker import Checker, Or
+from pyproj import CRS
+from rasterio.errors import NodataShadowWarning
+from shapely import Polygon
+
+from cars.core import inputs, projection
+from cars.orchestrator.cluster.log_wrapper import cars_profile
+
+from .abstract_dsm_filling_app import DsmFilling
+
+
+class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
+    """
+    Border interpolation
+    """
+
+    def __init__(self, conf=None):
+        """
+        Init function of BorderInterpolation
+
+        :param conf: configuration for BulldozerFilling
+        :return: an application_to_use object
+        """
+        super().__init__(conf=conf)
+
+        # check conf
+        self.used_method = self.used_config["method"]
+        self.classification = self.used_config["classification"]
+        self.component_min_size = self.used_config["component_min_size"]
+        self.border_size = self.used_config["border_size"]
+        self.percentile = self.used_config["percentile"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
+
+    def check_conf(self, conf):
+
+        # init conf
+        if conf is not None:
+            overloaded_conf = conf.copy()
+        else:
+            conf = {}
+            overloaded_conf = {}
+
+        # Overload conf
+        overloaded_conf["method"] = conf.get("method", "border_interpolation")
+        overloaded_conf["classification"] = conf.get("classification", "nodata")
+        if isinstance(overloaded_conf["classification"], str):
+            overloaded_conf["classification"] = [
+                overloaded_conf["classification"]
+            ]
+        overloaded_conf["component_min_size"] = conf.get(
+            "component_min_size", 5
+        )
+        overloaded_conf["border_size"] = conf.get("border_size", 10)
+        overloaded_conf["percentile"] = conf.get("percentile", 10)
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
+        )
+
+        rectification_schema = {
+            "method": str,
+            "classification": Or(None, [str]),
+            "component_min_size": int,
+            "border_size": int,
+            "percentile": Or(int, float),
+            "save_intermediate_data": bool,
+        }
+
+        # Check conf
+        checker = Checker(rectification_schema)
+        checker.validate(overloaded_conf)
+
+        return overloaded_conf
+
+    @cars_profile(name="Border interpolation filling")
+    def run(  # pylint: disable=too-many-positional-arguments  # noqa C901
+        self,
+        dsm_file,
+        classif_file,
+        filling_file,
+        dtm_file,
+        dump_dir,
+        roi_polys,
+        roi_epsg,
+    ):
+        """
+        Run dsm filling using initial elevation and the current dsm
+        Replaces dsm.tif by the filled dsm. Adds a new band
+        to filling.tif if it exists.
+        The old dsm is saved in dump_dir.
+
+        roi_poly can any of these objects :
+            - a list of Shapely Polygons
+            - a Shapely Polygon
+        """
+
+        if self.classification is None:
+            self.classification = ["nodata"]
+            logging.error(
+                "Filling method 'border_interpolation' needs a classification"
+            )
+
+        if not os.path.exists(dump_dir):
+            os.makedirs(dump_dir)
+
+        old_dsm_path = os.path.join(dump_dir, "dsm_not_filled.tif")
+        new_dsm_path = os.path.join(dump_dir, "dsm_filled.tif")
+
+        # get dsm to be filled and its metadata
+        with rio.open(dsm_file) as in_dsm:
+            dsm = in_dsm.read(1)
+            dsm_tr = in_dsm.transform
+            dsm_crs = in_dsm.crs
+            dsm_meta = in_dsm.meta
+            dsm_nodata = in_dsm.nodata
+
+        roi_raster = np.ones(dsm.shape)
+
+        if isinstance(roi_polys, list):
+            roi_polys_outepsg = []
+            for poly in roi_polys:
+                if isinstance(poly, Polygon):
+                    roi_poly_outepsg = projection.polygon_projection_crs(
+                        poly, CRS(roi_epsg), dsm_crs
+                    )
+                    roi_polys_outepsg.append(roi_poly_outepsg)
+
+            roi_raster = rio.features.rasterize(
+                roi_polys_outepsg, out_shape=roi_raster.shape, transform=dsm_tr
+            )
+        elif isinstance(roi_polys, Polygon):
+            roi_poly_outepsg = projection.polygon_projection_crs(
+                roi_polys, CRS(roi_epsg), dsm_crs
+            )
+            roi_raster = rio.features.rasterize(
+                [roi_poly_outepsg], out_shape=roi_raster.shape, transform=dsm_tr
+            )
+
+        # get dtm to fill the dsm
+        if dtm_file is not None:
+            logging.info(
+                "Use DTM file {} for border interpolation".format(dtm_file)
+            )
+            with rio.open(dtm_file) as in_dtm:
+                dtm = in_dtm.read(1)
+                dtm_nodata = in_dtm.nodata
+        else:
+            logging.info(
+                "No DTM provided : DSM {} will be used for "
+                "border interpolation".format(dsm_file)
+            )
+            dtm = dsm.copy()
+            dtm_nodata = dsm_nodata
+        dtm[dtm == dtm_nodata] = np.nan
+
+        if self.save_intermediate_data:
+            with rio.open(old_dsm_path, "w", **dsm_meta) as out_dsm:
+                out_dsm.write(dsm, 1)
+
+        if classif_file is not None:
+            classif_descriptions = inputs.get_descriptions_bands(classif_file)
+        else:
+            classif_descriptions = []
+        combined_mask = np.zeros_like(dsm).astype(np.uint8)
+        for label in self.classification:
+            if label in classif_descriptions:
+                index_classif = classif_descriptions.index(label) + 1
+                with rio.open(classif_file) as in_classif:
+                    classif = in_classif.read(index_classif)
+
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore", NodataShadowWarning)
+                        classif_msk = in_classif.read_masks(1)
+
+                classif[classif_msk == 0] = 0
+                filling_mask = np.logical_and(classif, roi_raster > 0)
+            else:
+                logging.error(
+                    "Label {} not found in classification "
+                    "descriptions {}".format(label, classif_descriptions)
+                )
+                continue
+            logging.info(
+                "Filling of {} with Bulldozer DTM using "
+                "border interpolation".format(label)
+            )
+            filling_mask[classif_msk == 0] = 0
+            filling_mask = skimage.morphology.binary_opening(
+                filling_mask,
+                footprint=[
+                    (np.ones((self.component_min_size, 1)), 1),
+                    (np.ones((1, self.component_min_size)), 1),
+                ],
+            )
+            features, num_features = scipy.ndimage.label(filling_mask)
+            logging.info("Filling of {} features".format(num_features))
+            features_boundaries = skimage.morphology.dilation(
+                features,
+                footprint=[
+                    (np.ones((self.border_size, 1)), 1),
+                    (np.ones((1, self.border_size)), 1),
+                ],
+            )
+            features_boundaries[filling_mask] = 0
+            borders_file_path = os.path.join(
+                dump_dir, "borders_of_{}.tif".format(label)
+            )
+            if self.save_intermediate_data:
+                with rio.open(
+                    borders_file_path, "w", **dsm_meta
+                ) as out_borders:
+                    out_borders.write(features_boundaries, 1)
+            for feature_id in range(1, num_features + 1):
+                altitude = np.nanpercentile(
+                    dtm[features_boundaries == feature_id], self.percentile
+                )
+                if altitude is not None:
+                    dsm[features == feature_id] = altitude
+            combined_mask = np.logical_or(combined_mask, filling_mask)
+
+        with rio.open(dsm_file, "w", **dsm_meta) as out_dsm:
+            out_dsm.write(dsm, 1)
+        if self.save_intermediate_data:
+            shutil.copy2(dsm_file, new_dsm_path)
+
+        if filling_file is not None:
+            with rio.open(filling_file, "r") as src:
+                fill_meta = src.meta
+                bands = [src.read(i + 1) for i in range(src.count)]
+                bands_desc = [src.descriptions[i] for i in range(src.count)]
+            fill_meta["count"] += 1
+            bands.append(combined_mask.astype(np.uint8))
+            bands_desc.append("border_interpolation")
+
+            with rio.open(filling_file, "w", **fill_meta) as out:
+                for i, band in enumerate(bands):
+                    out.write(band, i + 1)
+                    out.set_band_description(i + 1, bands_desc[i])

cars/applications/dsm_filling/bulldozer_config/base_config.yaml

@@ -0,0 +1,44 @@
+# both paths will be overwritten in the filling application
+dsm_path: "path/to/dsm"
+output_dir: "path/to/output"
+
+# everything else will be used directly as-is
+
+#-------------------------#
+#         Options         #
+#-------------------------#
+
+
+max_object_size : 16
+# If True, keep the intermediate results
+developer_mode : False
+
+
+#-------------------------#
+#    Advanced settings    #
+#-------------------------#
+# [Optional] - Altimetric height accuracy of the input DSM (m). If null, use the default value: 2*planimetric resolution
+dsm_z_accuracy: null
+# [Optional] - Maximum slope of the observed landscape terrain (%)
+# former slope_threshold
+max_ground_slope: 2.0
+# [Optional] - If True, activate the ground pre-detection
+activate_ground_anchors : False
+# [Optional] - If True, keep the intermediate results
+developer_mode : False
+
+
+#-------------------------#
+# Bulldozer core settings #
+#-------------------------#
+# /!\ Modify those data at your own risk (it is suggested to keep the default values) /!\
+
+# [Optional] - DtmExtraction parameters
+# former uniform_filter_size
+cloth_tension_force : 3
+prevent_unhook_iter : 10
+num_outer_iter : 50
+num_inner_iter : 10
+
+
+

cars/applications/dsm_filling/bulldozer_filling_app.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module contains the bulldozer dsm filling application class.
+"""
+
+import contextlib
+import logging
+import os
+import shutil
+import warnings
+
+import numpy as np
+import rasterio as rio
+import yaml
+from bulldozer.pipeline.bulldozer_pipeline import dsm_to_dtm
+from json_checker import Checker, Or
+from pyproj import CRS
+from rasterio.errors import NodataShadowWarning
+from shapely import Polygon
+
+from cars.applications.dem_generation.bulldozer_memory import (
+    can_allocate_shared_memory,
+)
+from cars.core import inputs, projection
+from cars.orchestrator.cluster.log_wrapper import cars_profile
+
+from .abstract_dsm_filling_app import DsmFilling
+
+
+class BulldozerFilling(DsmFilling, short_name="bulldozer"):
+    """
+    Bulldozer filling
+    """
+
+    def __init__(self, conf=None):
+        """
+        Init function of BulldozerFilling
+
+        :param conf: configuration for BulldozerFilling
+        :return: an application_to_use object
+        """
+        super().__init__(conf=conf)
+
+        # check conf
+        self.used_method = self.used_config["method"]
+        self.classification = self.used_config["classification"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
+
+    def check_conf(self, conf):
+
+        # init conf
+        if conf is not None:
+            overloaded_conf = conf.copy()
+        else:
+            conf = {}
+            overloaded_conf = {}
+
+        # Check if can use bulldozer
+        can_allocate_sm, log_message = can_allocate_shared_memory()
+        if not can_allocate_sm:
+            logging.error(log_message)
+            logging.error("DSM filling with bulldozer might crash")
+
+        # Overload conf
+        overloaded_conf["method"] = conf.get("method", "bulldozer")
+        overloaded_conf["classification"] = conf.get("classification", "nodata")
+
+        if isinstance(overloaded_conf["classification"], str):
+            overloaded_conf["classification"] = [
+                overloaded_conf["classification"]
+            ]
+
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
+        )
+
+        rectification_schema = {
+            "method": str,
+            "classification": Or(None, [str]),
+            "save_intermediate_data": bool,
+        }
+
+        # Check conf
+        checker = Checker(rectification_schema)
+        checker.validate(overloaded_conf)
+
+        return overloaded_conf
+
+    @cars_profile(name="Bulldozer filling")
+    def run(  # pylint: disable=too-many-positional-arguments # noqa C901
+        self,
+        dsm_file,
+        classif_file,
+        filling_file,
+        dump_dir,
+        roi_polys,
+        roi_epsg,
+        orchestrator,
+    ):
+        """
+        Run dsm filling using initial elevation and the current dsm
+        Replaces dsm.tif by the filled dsm. Adds a new band
+        to filling.tif if it exists.
+        The old dsm is saved in dump_dir.
+
+        roi_poly can any of these objects :
+            - a list of Shapely Polygons
+            - a Shapely Polygon
+        """
+
+        if self.classification is None:
+            self.classification = ["nodata"]
+
+        if not os.path.exists(dump_dir):
+            os.makedirs(dump_dir)
+
+        old_dsm_path = os.path.join(dump_dir, "dsm_not_filled.tif")
+        new_dsm_path = os.path.join(dump_dir, "dsm_filled.tif")
+
+        # create the config for the bulldozer execution
+        bull_conf_path = os.path.join(
+            os.path.dirname(__file__), "bulldozer_config/base_config.yaml"
+        )
+        with open(bull_conf_path, "r", encoding="utf8") as bull_conf_file:
+            bull_conf = yaml.safe_load(bull_conf_file)
+
+        bull_conf["dsm_path"] = dsm_file
+        bull_conf["output_dir"] = os.path.join(dump_dir, "bulldozer")
+
+        if orchestrator is not None:
+            if (
+                orchestrator.get_conf()["mode"] == "multiprocessing"
+                or orchestrator.get_conf()["mode"] == "local_dask"
+            ):
+                bull_conf["nb_max_workers"] = orchestrator.get_conf()[
+                    "nb_workers"
+                ]
+
+        bull_conf_path = os.path.join(dump_dir, "bulldozer_config.yaml")
+        with open(bull_conf_path, "w", encoding="utf8") as bull_conf_file:
+            yaml.dump(bull_conf, bull_conf_file)
+
+        dtm_path = os.path.join(bull_conf["output_dir"], "dtm.tif")
+
+        # get dsm to be filled and its metadata
+        with rio.open(dsm_file) as in_dsm:
+            dsm = in_dsm.read(1)
+            dsm_tr = in_dsm.transform
+            dsm_crs = in_dsm.crs
+            dsm_meta = in_dsm.meta
+
+        roi_raster = np.ones(dsm.shape)
+
+        if isinstance(roi_polys, list):
+            roi_polys_outepsg = []
+            for poly in roi_polys:
+                if isinstance(poly, Polygon):
+                    roi_poly_outepsg = projection.polygon_projection_crs(
+                        poly, CRS(roi_epsg), dsm_crs
+                    )
+                    roi_polys_outepsg.append(roi_poly_outepsg)
+
+            roi_raster = rio.features.rasterize(
+                roi_polys_outepsg, out_shape=roi_raster.shape, transform=dsm_tr
+            )
+        elif isinstance(roi_polys, Polygon):
+            roi_poly_outepsg = projection.polygon_projection_crs(
+                roi_polys, CRS(roi_epsg), dsm_crs
+            )
+            roi_raster = rio.features.rasterize(
+                [roi_poly_outepsg], out_shape=roi_raster.shape, transform=dsm_tr
+            )
+
+        try:
+            try:
+                # suppress prints in bulldozer by redirecting stdout&stderr
+                with open(os.devnull, "w", encoding="utf8") as devnull:
+                    with (
+                        contextlib.redirect_stdout(devnull),
+                        contextlib.redirect_stderr(devnull),
+                    ):
+                        dsm_to_dtm(bull_conf_path)
+            except Exception:
+                logging.info(
+                    "Bulldozer failed on its first execution. Retrying"
+                )
+                # suppress prints in bulldozer by redirecting stdout&stderr
+                with open(os.devnull, "w", encoding="utf8") as devnull:
+                    with (
+                        contextlib.redirect_stdout(devnull),
+                        contextlib.redirect_stderr(devnull),
+                    ):
+                        dsm_to_dtm(bull_conf_path)
+        except Exception:
+            logging.warning(
+                "Bulldozer failed on its second execution."
+                + " The DSM could not be filled."
+            )
+            return None
+        with rio.open(dtm_path) as in_dtm:
+            dtm = in_dtm.read(1)
+
+        if self.save_intermediate_data:
+            with rio.open(old_dsm_path, "w", **dsm_meta) as out_dsm:
+                out_dsm.write(dsm, 1)
+
+        if classif_file is not None and os.path.exists(classif_file):
+            classif_descriptions = inputs.get_descriptions_bands(classif_file)
+        else:
+            classif_descriptions = []
+        combined_mask = np.zeros_like(dsm).astype(np.uint8)
+        for label in self.classification:
+            if label in classif_descriptions:
+                index_classif = classif_descriptions.index(label) + 1
+                with rio.open(classif_file) as in_classif:
+                    classif = in_classif.read(index_classif)
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore", NodataShadowWarning)
+                        classif_msk = in_classif.read_masks(1)
+                classif[classif_msk == 0] = 0
+                filling_mask = np.logical_and(classif, roi_raster > 0)
+            elif label == "nodata":
+                if classif_file is not None and os.path.exists(classif_file):
+                    with rio.open(classif_file) as in_classif:
+                        classif_msk = in_classif.read_masks(1)
+                    classif = ~classif_msk
+                else:
+                    with rio.open(dsm_file) as in_dsm:
+                        dsm_msk = in_dsm.read_masks(1)
+                    classif = ~dsm_msk
+                filling_mask = np.logical_and(classif, roi_raster > 0)
+            else:
+                logging.error(
+                    "Label {} not found in classification "
+                    "descriptions {}".format(label, classif_descriptions)
+                )
+                continue
+            logging.info("Filling of {} with Bulldozer DTM".format(label))
+            dsm[filling_mask] = dtm[filling_mask]
+            combined_mask = np.logical_or(combined_mask, filling_mask)
+
+        with rio.open(dsm_file, "w", **dsm_meta) as out_dsm:
+            out_dsm.write(dsm, 1)
+        if self.save_intermediate_data:
+            shutil.copy2(dsm_file, new_dsm_path)
+
+        if filling_file is not None:
+            with rio.open(filling_file, "r") as src:
+                fill_meta = src.meta
+                bands = [src.read(i + 1) for i in range(src.count)]
+                bands_desc = [src.descriptions[i] for i in range(src.count)]
+            fill_meta["count"] += 1
+            bands.append(combined_mask.astype(np.uint8))
+            bands_desc.append("bulldozer")
+
+            with rio.open(filling_file, "w", **fill_meta) as out:
+                for i, band in enumerate(bands):
+                    out.write(band, i + 1)
+                    out.set_band_description(i + 1, bands_desc[i])
+        return dtm_path