cars 1.0.0rc1__cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl

cars/pipelines/parameters/advanced_parameters.py

@@ -0,0 +1,417 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+CARS module containing functions to check advanced parameters configuration
+"""
+import logging
+import os
+
+import numpy as np
+import rasterio as rio
+from json_checker import And, Checker, OptionalKey, Or
+
+from cars.pipelines.parameters import advanced_parameters_constants as adv_cst
+from cars.pipelines.parameters import dsm_inputs
+from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst
+from cars.pipelines.parameters import sensor_inputs
+from cars.pipelines.parameters import sensor_inputs_constants as sens_cst
+from cars.pipelines.parameters.sensor_inputs import CARS_GEOID_PATH
+from cars.pipelines.pipeline_constants import ADVANCED
+
+
+def get_epipolar_resolutions(conf):
+    """
+    Get the epipolar resolutions from the configuration
+
+    :param conf: configuration of advanced parameters
+    :type conf: dict
+
+    :return: list of epipolar resolutions
+    :rtype: list
+    """
+    if adv_cst.EPIPOLAR_RESOLUTIONS in conf:
+        return conf[adv_cst.EPIPOLAR_RESOLUTIONS]
+
+    return [16, 4, 1]
+
+
+def check_advanced_parameters(
+    inputs, conf, check_epipolar_a_priori=True, output_dem_dir=None
+):
+    """
+    Check the advanced parameters consistency
+
+    :param inputs: configuration of inputs
+    :type inputs: dict
+    :param conf: configuration of advanced parameters
+    :type conf: dict
+    :param check_epipolar_a_priori: use epipolar a priori parameters
+    :type check_epipolar_a_priori: bool
+
+    :return: overloaded configuration
+    :rtype: dict
+    """
+
+    overloaded_conf = conf.copy()
+
+    overloaded_conf[adv_cst.SAVE_INTERMEDIATE_DATA] = conf.get(
+        adv_cst.SAVE_INTERMEDIATE_DATA, False
+    )
+
+    overloaded_conf[adv_cst.LAND_COVER_MAP] = conf.get(
+        adv_cst.LAND_COVER_MAP, "global_land_cover_map.tif"
+    )
+
+    overloaded_conf[adv_cst.KEEP_LOW_RES_DIR] = conf.get(
+        adv_cst.KEEP_LOW_RES_DIR,
+        bool(overloaded_conf[adv_cst.SAVE_INTERMEDIATE_DATA]),
+    )
+
+    overloaded_conf[adv_cst.DEBUG_WITH_ROI] = conf.get(
+        adv_cst.DEBUG_WITH_ROI, False
+    )
+
+    overloaded_conf[adv_cst.CLASSIFICATION_TO_CONFIGURATION_MAPPING] = conf.get(
+        adv_cst.CLASSIFICATION_TO_CONFIGURATION_MAPPING, "config_mapping.json"
+    )
+
+    overloaded_conf[adv_cst.PHASING] = conf.get(adv_cst.PHASING, None)
+
+    overloaded_conf[adv_cst.EPIPOLAR_RESOLUTIONS] = get_epipolar_resolutions(
+        conf
+    )
+
+    # use endogenous dm when generated
+    overloaded_conf[adv_cst.USE_ENDOGENOUS_DEM] = conf.get(
+        adv_cst.USE_ENDOGENOUS_DEM,
+        inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] is None,
+    )
+
+    overloaded_conf[adv_cst.DSM_MERGING_TILE_SIZE] = conf.get(
+        adv_cst.DSM_MERGING_TILE_SIZE, 4000
+    )
+
+    overloaded_conf[adv_cst.GROUND_TRUTH_DSM] = conf.get(
+        adv_cst.GROUND_TRUTH_DSM, {}
+    )
+
+    # Validate ground truth DSM
+    if overloaded_conf[adv_cst.GROUND_TRUTH_DSM]:
+        overloaded_conf[adv_cst.GROUND_TRUTH_DSM][adv_cst.INPUT_AUX_PATH] = (
+            conf[adv_cst.GROUND_TRUTH_DSM].get(adv_cst.INPUT_AUX_PATH, None)
+        )
+        overloaded_conf[adv_cst.GROUND_TRUTH_DSM][adv_cst.INPUT_AUX_INTERP] = (
+            conf[adv_cst.GROUND_TRUTH_DSM].get(adv_cst.INPUT_AUX_INTERP, None)
+        )
+        check_ground_truth_dsm_data(overloaded_conf[adv_cst.GROUND_TRUTH_DSM])
+
+    if check_epipolar_a_priori:
+        # Retrieve epipolar_a_priori if it is provided
+        overloaded_conf[adv_cst.EPIPOLAR_A_PRIORI] = conf.get(
+            adv_cst.EPIPOLAR_A_PRIORI, None
+        )
+        # Retrieve terrain_a_priori if it is provided
+        overloaded_conf[adv_cst.TERRAIN_A_PRIORI] = conf.get(
+            adv_cst.TERRAIN_A_PRIORI, None
+        )
+
+    # Check geometry plugin
+    geom_plugin_without_dem_and_geoid = None
+    geom_plugin_with_dem_and_geoid = None
+
+    scaling_coeff = None
+    # Get last resolution for scaling
+    if isinstance(overloaded_conf[adv_cst.EPIPOLAR_RESOLUTIONS], list):
+        epipolar_resolution = overloaded_conf[adv_cst.EPIPOLAR_RESOLUTIONS][-1]
+    else:
+        epipolar_resolution = overloaded_conf[adv_cst.EPIPOLAR_RESOLUTIONS]
+    if inputs[sens_cst.SENSORS] is not None:
+        # Check geometry plugin and overwrite geomodel in conf inputs
+        (
+            inputs,
+            overloaded_conf[adv_cst.GEOMETRY_PLUGIN],
+            geom_plugin_without_dem_and_geoid,
+            geom_plugin_with_dem_and_geoid,
+            scaling_coeff,
+        ) = sensor_inputs.check_geometry_plugin(
+            inputs,
+            conf.get(adv_cst.GEOMETRY_PLUGIN, None),
+            epipolar_resolution,
+            output_dem_dir,
+        )
+    elif dsm_cst.DSMS in inputs:
+        # assume the input comes from 0.5m sensor images
+        scaling_coeff = 1
+        # If there's an initial elevation with
+        # point clouds as inputs, generate a plugin (used in dsm_filling)
+        (
+            overloaded_conf[adv_cst.GEOMETRY_PLUGIN],
+            geom_plugin_with_dem_and_geoid,
+        ) = dsm_inputs.check_geometry_plugin(
+            inputs, conf.get(adv_cst.GEOMETRY_PLUGIN, None)
+        )
+
+    # Check pipeline
+    overloaded_conf[adv_cst.PIPELINE] = conf.get(adv_cst.PIPELINE, "default")
+
+    # Validate inputs
+    schema = {
+        adv_cst.DEBUG_WITH_ROI: bool,
+        adv_cst.SAVE_INTERMEDIATE_DATA: Or(dict, bool),
+        adv_cst.KEEP_LOW_RES_DIR: bool,
+        adv_cst.GROUND_TRUTH_DSM: Or(dict, str),
+        adv_cst.PHASING: Or(dict, None),
+        adv_cst.GEOMETRY_PLUGIN: Or(str, dict),
+        adv_cst.PIPELINE: str,
+        adv_cst.DSM_MERGING_TILE_SIZE: And(int, lambda x: x > 0),
+        adv_cst.EPIPOLAR_RESOLUTIONS: Or(int, list),
+        adv_cst.LAND_COVER_MAP: str,
+        adv_cst.CLASSIFICATION_TO_CONFIGURATION_MAPPING: str,
+        adv_cst.USE_ENDOGENOUS_DEM: bool,
+    }
+    if check_epipolar_a_priori:
+        schema[adv_cst.EPIPOLAR_A_PRIORI] = Or(None, dict)
+        schema[adv_cst.TERRAIN_A_PRIORI] = Or(None, dict)
+
+    checker_advanced_parameters = Checker(schema)
+    checker_advanced_parameters.validate(overloaded_conf)
+
+    # Validate epipolar schema
+    epipolar_schema = {
+        adv_cst.GRID_CORRECTION: Or(list, None),
+        adv_cst.DISPARITY_RANGE: list,
+        adv_cst.REFERENCE_DEM: Or(str, None),
+    }
+    checker_epipolar = Checker(epipolar_schema)
+
+    # Check terrain a priori
+    if check_epipolar_a_priori and overloaded_conf[adv_cst.TERRAIN_A_PRIORI]:
+        overloaded_conf[adv_cst.TERRAIN_A_PRIORI][adv_cst.DEM_MEDIAN] = (
+            overloaded_conf[adv_cst.TERRAIN_A_PRIORI].get(
+                adv_cst.DEM_MEDIAN, None
+            )
+        )
+        overloaded_conf[adv_cst.TERRAIN_A_PRIORI][adv_cst.DEM_MIN] = (
+            overloaded_conf[adv_cst.TERRAIN_A_PRIORI].get(adv_cst.DEM_MIN, None)
+        )
+        overloaded_conf[adv_cst.TERRAIN_A_PRIORI][adv_cst.DEM_MAX] = (
+            overloaded_conf[adv_cst.TERRAIN_A_PRIORI].get(adv_cst.DEM_MAX, None)
+        )
+        terrain_a_priori_schema = {
+            adv_cst.DEM_MEDIAN: str,
+            adv_cst.DEM_MIN: Or(str, None),  # TODO mandatory with local disp
+            adv_cst.DEM_MAX: Or(str, None),
+        }
+        checker_terrain = Checker(terrain_a_priori_schema)
+        checker_terrain.validate(overloaded_conf[adv_cst.TERRAIN_A_PRIORI])
+
+    # check epipolar a priori for each image pair
+    if check_epipolar_a_priori and (
+        not overloaded_conf[adv_cst.TERRAIN_A_PRIORI] in (None, {})
+        or not overloaded_conf[adv_cst.EPIPOLAR_A_PRIORI] in (None, {})
+    ):
+        validate_epipolar_a_priori(overloaded_conf, checker_epipolar)
+
+    return (
+        inputs,
+        overloaded_conf,
+        overloaded_conf[adv_cst.GEOMETRY_PLUGIN],
+        geom_plugin_without_dem_and_geoid,
+        geom_plugin_with_dem_and_geoid,
+        scaling_coeff,
+        overloaded_conf[adv_cst.LAND_COVER_MAP],
+        overloaded_conf[adv_cst.CLASSIFICATION_TO_CONFIGURATION_MAPPING],
+    )
+
+
+def validate_epipolar_a_priori(
+    overloaded_conf,
+    checker_epipolar,
+):
+    """
+    Validate inner epipolar configuration
+
+    :param conf : input configuration json
+    :type conf: dict
+    :param overloaded_conf : overloaded configuration json
+    :type overloaded_conf: dict
+    :param checker_epipolar : json checker
+    :type checker_epipolar: Checker
+    """
+
+    if overloaded_conf[adv_cst.EPIPOLAR_A_PRIORI] is not None:
+        for key_image_pair in overloaded_conf[adv_cst.EPIPOLAR_A_PRIORI]:
+            checker_epipolar.validate(
+                overloaded_conf[adv_cst.EPIPOLAR_A_PRIORI][key_image_pair]
+            )
+
+
+def check_ground_truth_dsm_data(conf):
+    """
+    Check data of the image ground truth
+
+    :param conf: ground truth dsm configuration
+    :type conf: str
+    """
+    if isinstance(conf, str):
+        with rio.open(conf) as img_reader:
+            trans = img_reader.transform
+            if trans.e < 0:
+                logging.warning(
+                    "{} seems to have an incoherent pixel size. "
+                    "Input images has to be in sensor geometry.".format(conf)
+                )
+
+    conf[adv_cst.INPUT_GEOID] = conf.get(adv_cst.INPUT_GEOID, None)
+
+    if isinstance(conf, dict):
+        ground_truth_dsm_schema = {
+            adv_cst.INPUT_GROUND_TRUTH_DSM: str,
+            OptionalKey(adv_cst.INPUT_AUX_PATH): Or(dict, None),
+            OptionalKey(adv_cst.INPUT_AUX_INTERP): Or(dict, None),
+            adv_cst.INPUT_GEOID: Or(None, str, bool),
+        }
+
+        checker_ground_truth_dsm_schema = Checker(ground_truth_dsm_schema)
+        checker_ground_truth_dsm_schema.validate(conf)
+
+        gt_dsm_path = conf[adv_cst.INPUT_GROUND_TRUTH_DSM]
+        with rio.open(gt_dsm_path) as img_reader:
+            trans = img_reader.transform
+            if trans.e < 0:
+                logging.warning(
+                    "{} seems to have an incoherent pixel size. "
+                    "Input images has to be in sensor geometry.".format(
+                        gt_dsm_path
+                    )
+                )
+
+        # Update geoid
+        if isinstance(conf[adv_cst.INPUT_GEOID], bool):
+            if conf[adv_cst.INPUT_GEOID]:
+                # Use CARS geoid
+                logging.info(
+                    "CARS will use its own internal file as geoid reference"
+                )
+                package_path = os.path.dirname(__file__)
+                geoid_path = os.path.join(
+                    package_path, "..", "..", "conf", CARS_GEOID_PATH
+                )
+                conf[adv_cst.INPUT_GEOID] = geoid_path
+            else:
+                conf[adv_cst.INPUT_GEOID] = None
+
+        path_dict = conf.get(adv_cst.INPUT_AUX_PATH, None)
+        if path_dict is not None:
+            for key in path_dict.keys():
+                if not isinstance(path_dict[key], str):
+                    raise RuntimeError("Path should be a string")
+                if not os.path.exists(path_dict[key]):
+                    raise RuntimeError("Path doesn't exist")
+
+        path_interp = conf.get(adv_cst.INPUT_AUX_INTERP, None)
+        if path_interp is not None:
+            for key in path_interp.keys():
+                if not isinstance(path_interp[key], str):
+                    raise RuntimeError("interpolator should be a string")
+                if path_interp[key] not in (
+                    "nearest",
+                    "linear",
+                    "cubic",
+                    "slinear",
+                    "quintic",
+                ):
+                    raise RuntimeError("interpolator does not exist")
+
+
+def update_conf(  # pylint: disable=too-many-positional-arguments  # noqa: C901
+    conf,
+    grid_correction_coef=None,
+    reference_dem=None,
+    dmin=None,
+    dmax=None,
+    pair_key=None,
+    dem_median=None,
+    dem_min=None,
+    dem_max=None,
+):
+    """
+    Update the conf with grid correction and disparity range
+    :param grid_correction_coef: grid correction coefficient
+    :type grid_correction_coef: list
+    :param reference_dem: Dem used to compute epipolar grids
+    :type reference_dem: str
+    :param dmin: disparity range minimum
+    :type dmin: float
+    :param dmax: disparity range maximum
+    :type dmax: float
+    :param pair_key: name of the inputs key pair
+    :type pair_key: str
+    """
+
+    if pair_key is not None:
+        if ADVANCED not in conf:
+            conf[ADVANCED] = {}
+        if adv_cst.EPIPOLAR_A_PRIORI not in conf[ADVANCED]:
+            conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI] = {}
+        if conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI] is None:
+            conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI] = {}
+        if pair_key not in conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI]:
+            conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI][pair_key] = {}
+        if grid_correction_coef is not None and reference_dem is not None:
+            conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI][pair_key][
+                "reference_dem"
+            ] = reference_dem
+            if len(grid_correction_coef) == 2:
+                conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI][pair_key][
+                    "grid_correction"
+                ] = (
+                    np.concatenate(grid_correction_coef[0], axis=0).tolist()[
+                        :-1
+                    ]
+                    + np.concatenate(grid_correction_coef[1], axis=0).tolist()[
+                        :-1
+                    ]
+                )
+            else:
+                conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI][pair_key][
+                    "grid_correction"
+                ] = list(grid_correction_coef)
+        if None not in (dmin, dmax):
+            conf[ADVANCED][adv_cst.EPIPOLAR_A_PRIORI][pair_key][
+                "disparity_range"
+            ] = [
+                dmin,
+                dmax,
+            ]
+
+    if adv_cst.TERRAIN_A_PRIORI not in conf[ADVANCED]:
+        conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] = {}
+    if conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] is None:
+        conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI] = {}
+
+    if dem_median is not None:
+        conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI]["dem_median"] = dem_median
+    if dem_min is not None:
+        conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI]["dem_min"] = dem_min
+    if dem_max is not None:
+        conf[ADVANCED][adv_cst.TERRAIN_A_PRIORI]["dem_max"] = dem_max

cars/pipelines/parameters/advanced_parameters_constants.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+This module contains the advanced parameter definitions
+"""
+
+SAVE_INTERMEDIATE_DATA = "save_intermediate_data"
+KEEP_LOW_RES_DIR = "keep_low_res_dir"
+PHASING = "phasing"
+DEBUG_WITH_ROI = "debug_with_roi"
+LAND_COVER_MAP = "land_cover_map"
+CLASSIFICATION_TO_CONFIGURATION_MAPPING = (
+    "classification_to_configuration_mapping"
+)
+
+EPIPOLAR_A_PRIORI = "epipolar_a_priori"
+RESOLUTION_A_PRIORI = "resolution_a_priori"
+GROUND_TRUTH_DSM = "ground_truth_dsm"
+
+EPIPOLAR_RESOLUTIONS = "epipolar_resolutions"
+
+DSM_MERGING_TILE_SIZE = "dsm_merging_tile_size"
+
+
+# inner epipolar a priori constants
+GRID_CORRECTION = "grid_correction"
+DISPARITY_RANGE = "disparity_range"
+REFERENCE_DEM = "reference_dem"
+
+TERRAIN_A_PRIORI = "terrain_a_priori"
+USE_ENDOGENOUS_DEM = "use_endogenous_dem"
+DEM_MEDIAN = "dem_median"
+DEM_MIN = "dem_min"
+DEM_MAX = "dem_max"
+
+# ground truth dsm
+INPUT_GROUND_TRUTH_DSM = "dsm"
+INPUT_CLASSIFICATION = "classification"
+INPUT_AUX_PATH = "auxiliary_data"
+INPUT_AUX_INTERP = "auxiliary_data_interpolation"
+INPUT_GEOID = "geoid"
+INPUT_EPSG = "epsg"
+
+PERFORMANCE_MAP_CLASSES = "performance_map_classes"
+
+TEXTURE_BANDS = "texture_bands"
+
+GEOMETRY_PLUGIN = "geometry_plugin"
+
+PIPELINE = "pipeline"

cars/pipelines/parameters/application_parameters.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+General application configuration module
+"""
+
+
+def get_needed_apps(
+    sensors_in_inputs, save_output_dsm, save_output_point_cloud, conf
+):
+    """
+    This function returns the apps needed by the CARS pipeline,
+    depending on overall parameters and the user configuration
+    """
+    needed_applications = []
+    if sensors_in_inputs:
+        needed_applications += [
+            "grid_generation",
+            "resampling",
+            "ground_truth_reprojection",
+            "dense_match_filling",
+            "sparse_matching",
+            "dense_matching",
+            "triangulation",
+            "dem_generation",
+        ]
+
+        add_default_pc_outlier_removal = True
+        for key in conf:
+            if key.startswith("point_cloud_outlier_removal"):
+                add_default_pc_outlier_removal = False
+                needed_applications += [key]
+
+        if add_default_pc_outlier_removal:
+            needed_applications += [
+                "point_cloud_outlier_removal.1",
+                "point_cloud_outlier_removal.2",
+            ]
+
+    if save_output_dsm or save_output_point_cloud:
+        needed_applications += ["pc_denoising"]
+
+        if save_output_dsm:
+            needed_applications += [
+                "point_cloud_rasterization",
+                "auxiliary_filling",
+            ]
+
+            for key in conf:
+                if key.startswith("dsm_filling"):
+                    needed_applications += [key]
+
+    return needed_applications