cars 1.0.0a3__cp39-cp39-win_amd64.whl → 1.0.0a4__cp39-cp39-win_amd64.whl

cars/applications/dense_matching/dense_matching_algo.py

@@ -144,7 +144,7 @@ def compute_disparity_grid(
     return disp_min_grid, disp_max_grid
 
 
-def compute_disparity(
+def compute_disparity(  # pylint: disable=too-many-positional-arguments
     left_dataset,
     right_dataset,
     corr_cfg,
@@ -257,19 +257,53 @@ def compute_disparity(
         right_dataset = right_dataset.drop_vars([cst.EPI_MSK])
         right_dataset[cst.EPI_MSK] = right_msk
 
-    # Instantiate pandora state machine
-    pandora_machine = PandoraMachine()
-
-    # check datasets
-    check_datasets(left_dataset, right_dataset)
-
-    # Run the Pandora pipeline
-    ref, _ = pandora.run(
-        pandora_machine,
-        left_dataset,
-        right_dataset,
-        corr_cfg,
-    )
+    # Check that the datasets are not full of nan (would crash later)
+    if left_dataset["msk"].all() or right_dataset["msk"].all():
+        height = left_dataset.sizes["row"]
+        width = left_dataset.sizes["col"]
+        ref = xr.Dataset(
+            coords={
+                "row": left_dataset.coords["row"],
+                "col": left_dataset.coords["col"],
+                "indicator": [
+                    "confidence_from_ambiguity.cars_1",
+                    "confidence_from_risk_max.cars_2",
+                    "confidence_from_risk_min.cars_2",
+                    "confidence_from_disp_sup_from_risk.cars_2",
+                    "confidence_from_disp_inf_from_risk.cars_2",
+                    "confidence_from_interval_bounds_inf.cars_3",
+                    "confidence_from_interval_bounds_sup.cars_3",
+                    "confidence_from_left_right_consistency",
+                ],
+            },
+            data_vars={
+                "disparity_map": (
+                    ("row", "col"),
+                    np.full((height, width), np.nan, dtype=np.float32),
+                ),
+                "validity_mask": (
+                    ("row", "col"),
+                    np.ones((height, width), dtype=np.int64),
+                ),
+                "confidence_measure": (
+                    ("row", "col", "indicator"),
+                    np.full((height, width, 8), np.nan, dtype=np.float32),
+                ),
+            },
+        )
+    else:
+        # Instantiate pandora state machine
+        pandora_machine = PandoraMachine()
+        # check datasets
+        check_datasets(left_dataset, right_dataset)
+
+        # Run the Pandora pipeline
+        ref, _ = pandora.run(
+            pandora_machine,
+            left_dataset,
+            right_dataset,
+            corr_cfg,
+        )
 
     disp_dataset = dm_wrap.create_disp_dataset(
         ref,

cars/applications/dense_matching/dense_matching_wrappers.py

@@ -21,10 +21,11 @@
 This module is responsible for the dense matching algorithms:
 - thus it creates a disparity map from a pair of images
 """
-# pylint: disable=too-many-lines
+# pylint: disable=C0302
 
 # Standard imports
 import logging
+import math
 import warnings
 from typing import Dict
 
@@ -281,6 +282,7 @@ def compute_cropped_roi(
     )
 
 
+# pylint: disable=too-many-positional-arguments
 def create_disp_dataset(  # noqa: C901
     disp: xr.Dataset,
     ref_dataset: xr.Dataset,
@@ -513,6 +515,7 @@ def add_crop_info(disp_ds, cropped_range):
     return disp_ds
 
 
+# pylint: disable=too-many-positional-arguments
 def estimate_right_classif_on_left(
     right_classif,
     disp_map,
@@ -752,6 +755,7 @@ def estimate_right_grid_disp(disp_min_grid, disp_max_grid):
     )
 
 
+# pylint: disable=too-many-positional-arguments
 def optimal_tile_size_pandora_plugin_libsgm(
     disp_min: int,
     disp_max: int,
@@ -779,7 +783,7 @@ def optimal_tile_size_pandora_plugin_libsgm(
     """
 
     memory = max_ram_per_worker
-    disp = disp_max - disp_min
+    disp = max(3, abs(disp_max - disp_min))
 
     image = 32 * 2
     disp_ref = 32
@@ -809,13 +813,17 @@ def optimal_tile_size_pandora_plugin_libsgm(
         # but sqrt(nb_pixels + (disp/2)**2) - disp/2
         tile_size = np.sqrt(row_or_col + (disp / 2) ** 2) - disp / 2
         tile_size = (1.0 - margin / 100.0) * tile_size
-        tile_size = tile_size_rounding * int(tile_size / tile_size_rounding)
+
+    if math.isinf(tile_size):
+        logging.warning("Tile size infinite")
 
     if tile_size > max_tile_size:
         tile_size = max_tile_size
     elif tile_size < min_tile_size:
         tile_size = min_tile_size
 
+    tile_size = tile_size_rounding * int(tile_size / tile_size_rounding)
+
     return tile_size
 
 

cars/applications/dense_matching/disparity_grid_algo.py

@@ -27,7 +27,6 @@ import itertools
 import logging
 
 # Third party imports
-import affine
 import numpy as np
 import rasterio
 import xarray as xr
@@ -45,6 +44,7 @@ from cars.core.projection import point_cloud_conversion
 from cars.data_structures import cars_dataset, cars_dict
 
 
+# pylint: disable=too-many-positional-arguments
 def generate_disp_grids_dataset(
     grid_min,
     grid_max,
@@ -103,6 +103,7 @@ def generate_disp_grids_dataset(
     return disp_range_tile
 
 
+# pylint: disable=too-many-positional-arguments
 def generate_disp_range_const_tile_wrapper(
     row_range,
     col_range,
@@ -152,6 +153,7 @@ def generate_disp_range_const_tile_wrapper(
     return disp_range, global_infos
 
 
+# pylint: disable=too-many-positional-arguments
 def generate_disp_range_from_dem_wrapper(
     epipolar_grid_array_window,
     full_epi_row_range,
@@ -162,8 +164,6 @@ def generate_disp_range_from_dem_wrapper(
     dem_median,
     dem_min,
     dem_max,
-    altitude_delta_min,
-    altitude_delta_max,
     raster_profile,
     saving_info,
     saving_info_global_infos,
@@ -193,10 +193,6 @@ def generate_disp_range_from_dem_wrapper(
     :type dem_min: str
     :param dem_max: Path of dem max.
     :type dem_max: srt
-    :param altitude_delta_min: The minimum altitude delta.
-    :type altitude_delta_min: float
-    :param altitude_delta_max: The maximum altitude delta.
-    :type altitude_delta_max: float
     :param raster_profile: The raster profile.
     :type raster_profile: dict
     :param saving_info: The disp range grid saving information.
@@ -217,23 +213,11 @@ def generate_disp_range_from_dem_wrapper(
     """
 
     # compute reverse matrix
-    transform_sensor = rasterio.Affine(
-        *np.abs(
-            inputs.rasterio_get_transform(
-                sensor_image_right["image"]["main_file"]
-            )
-        )
+    transform_sensor = inputs.rasterio_get_transform(
+        sensor_image_right["image"]["main_file"], convention="north"
     )
 
     trans_inv_sensor = ~transform_sensor
-    # Transform to positive values
-    trans_inv_sensor = np.array(trans_inv_sensor)
-    trans_inv_sensor = np.reshape(trans_inv_sensor, (3, 3))
-    if trans_inv_sensor[0, 0] < 0:
-        trans_inv_sensor[0, :] *= -1
-    if trans_inv_sensor[1, 1] < 0:
-        trans_inv_sensor[1, :] *= -1
-    trans_inv_sensor = affine.Affine(*list(trans_inv_sensor.flatten()))
 
     # Geometry plugin
     geo_plugin = geom_plugin_with_dem_and_geoid
@@ -294,7 +278,9 @@ def generate_disp_range_from_dem_wrapper(
         (np.max(col_range_with_margin), np.min(row_range_with_margin)),
         (np.max(col_range_with_margin), np.max(row_range_with_margin)),
     ]
-    sensor_bbox = geo_plugin.sensor_position_from_grid(grid_right, epi_bbox)
+    sensor_bbox = geo_plugin.sensor_position_from_grid(
+        grid_right, epi_bbox, interpolation_method="linear"
+    )
     row_sensor_bbox, col_sensor_bbox = transform_physical_point_to_index(
         trans_inv_sensor, sensor_bbox[:, 1], sensor_bbox[:, 0]
     )
@@ -356,18 +342,26 @@ def generate_disp_range_from_dem_wrapper(
     lon_mean = terrain_position_lon_lat[:, 0]
     lat_mean = terrain_position_lon_lat[:, 1]
 
-    if None not in (dem_min, dem_max, dem_median):
-        # dem min and max are in 4326
-        dem_min_list = inputs.rasterio_get_values(
-            dem_min, lon_mean, lat_mean, point_cloud_conversion
-        )
-        dem_max_list = inputs.rasterio_get_values(
-            dem_max, lon_mean, lat_mean, point_cloud_conversion
+    # dem min and max are in 4326
+    dem_min_list = inputs.rasterio_get_values(
+        dem_min, lon_mean, lat_mean, point_cloud_conversion
+    )
+    dem_max_list = inputs.rasterio_get_values(
+        dem_max, lon_mean, lat_mean, point_cloud_conversion
+    )
+    if dem_min_list is None or dem_max_list is None:
+        logging.warning("DEM min and DEM max does not cover this tile")
+        disp_range, global_infos = empty_disparity_grids(
+            row_range_no_margin,
+            col_range_no_margin,
+            epipolar_grid_array_window,
+            raster_profile,
+            saving_info,
+            saving_info_global_infos,
         )
-        nan_mask = nan_mask & ~np.isnan(dem_min_list) & ~np.isnan(dem_max_list)
-    else:
-        dem_min_list = dem_median_list - altitude_delta_min
-        dem_max_list = dem_median_list + altitude_delta_max
+        return disp_range, global_infos
+
+    nan_mask = nan_mask & ~np.isnan(dem_min_list) & ~np.isnan(dem_max_list)
 
     # filter nan value from input points
     lon_mean = lon_mean[nan_mask]
@@ -410,6 +404,7 @@ def generate_disp_range_from_dem_wrapper(
                     2,
                 ),
             ),
+            interpolation_method="linear",
         )
     )
 
@@ -422,37 +417,14 @@ def generate_disp_range_from_dem_wrapper(
 
     if len(ind_rows_sensor) < 5:
         # QH6214 needs at least 4 points for interpolation
-
-        grid_min = np.empty(
-            (len(row_range_no_margin), len(col_range_no_margin))
-        )
-        grid_max = np.empty(
-            (len(row_range_no_margin), len(col_range_no_margin))
-        )
-        grid_min[:, :] = 0
-        grid_max[:, :] = 0
-
-        disp_range = generate_disp_grids_dataset(
-            grid_min,
-            grid_max,
-            saving_info,
+        disp_range, global_infos = empty_disparity_grids(
+            row_range_no_margin,
+            col_range_no_margin,
+            epipolar_grid_array_window,
             raster_profile,
-            window=epipolar_grid_array_window,
-            row_coords=row_range_no_margin,
-            col_coords=col_range_no_margin,
-        )
-
-        # Generate infos on global min and max
-        global_infos = cars_dict.CarsDict(
-            {
-                "global_min": 0,
-                "global_max": 0,
-            }
-        )
-        cars_dataset.fill_dict(
-            global_infos, saving_info=saving_info_global_infos
+            saving_info,
+            saving_info_global_infos,
         )
-
         return disp_range, global_infos
 
     # Interpolate disparity
@@ -542,6 +514,56 @@ def generate_disp_range_from_dem_wrapper(
     return disp_range, global_infos
 
 
+def empty_disparity_grids(  # pylint: disable=too-many-positional-arguments
+    row_range_no_margin,
+    col_range_no_margin,
+    epipolar_grid_array_window,
+    raster_profile,
+    saving_info,
+    saving_info_global_infos,
+):
+    """
+    Return empty disparity grids
+    :param row_range_no_margin: Rows id in grid
+    :type row_range_no_margin: int
+    :param col_range_no_margin: Cols id in grid
+    :type col_range_no_margin: int
+    :param epipolar_grid_array_window: ROI of grid
+    :type epipolar_grid_array_window: dict
+    :param raster_profile: The raster profile.
+    :type raster_profile: dict
+    :param saving_info: The disp range grid saving information.
+    :type saving_info: dict
+    :param saving_info_global_infos: Global info saving infos.
+    :type saving_info_global_infos: dict
+    """
+    grid_min = np.empty((len(row_range_no_margin), len(col_range_no_margin)))
+    grid_max = np.empty((len(row_range_no_margin), len(col_range_no_margin)))
+    grid_min[:, :] = 0
+    grid_max[:, :] = 0
+
+    disp_range = generate_disp_grids_dataset(
+        grid_min,
+        grid_max,
+        saving_info,
+        raster_profile,
+        window=epipolar_grid_array_window,
+        row_coords=row_range_no_margin,
+        col_coords=col_range_no_margin,
+    )
+
+    # Generate infos on global min and max
+    global_infos = cars_dict.CarsDict(
+        {
+            "global_min": 0,
+            "global_max": 0,
+        }
+    )
+    cars_dataset.fill_dict(global_infos, saving_info=saving_info_global_infos)
+
+    return disp_range, global_infos
+
+
 def create_circular_mask(height, width):
     """
     Create a circular mask for footprint around pixel

cars/applications/dense_matching/loaders/pandora_loader.py

@@ -32,6 +32,7 @@ from typing import Dict
 import numpy as np
 import pandora
 import rasterio
+import xarray as xr
 from json_checker import Checker, Or
 from pandora.check_configuration import (
     check_pipeline_section,
@@ -39,12 +40,12 @@ from pandora.check_configuration import (
     get_config_pipeline,
     update_conf,
 )
-from pandora.img_tools import get_metadata
 from pandora.state_machine import PandoraMachine
 from rasterio.mask import mask
 from shapely.geometry import mapping
 
 from cars.core.projection import polygon_projection
+from cars.orchestrator.cluster.log_wrapper import cars_profile
 
 
 class PandoraLoader:
@@ -53,7 +54,7 @@ class PandoraLoader:
 
     """
 
-    def __init__(  # noqa: C901
+    def __init__(  # pylint: disable=too-many-positional-arguments  # noqa: C901
         self,
         conf=None,
         method_name=None,
@@ -271,26 +272,6 @@ class PandoraLoader:
                 if "band" not in filter_conf:
                     conf["pipeline"]["filter"]["band"] = used_band
 
-        for key in list(conf.get("pipeline")):
-            if key.startswith("filter"):
-                if (
-                    conf["pipeline"][key]["filter_method"]
-                    == "median_for_intervals"
-                    and "validation" in conf["pipeline"]
-                ):
-                    if (
-                        conf["pipeline"]["validation"]["validation_method"]
-                        == "cross_checking_fast"
-                    ):
-                        conf["pipeline"]["validation"][
-                            "validation_method"
-                        ] = "cross_checking_accurate"
-                        logging.warning(
-                            "You can not use median_for_intervals with "
-                            "the fast cross checking validation for now. "
-                            "It therefore has been overrided to accurate"
-                        )
-
         # Check conf
         self.pandora_config = conf
 
@@ -305,6 +286,35 @@ class PandoraLoader:
 
         return self.pandora_config
 
+    def get_classif_bands(self):
+        """
+        Get the classification bands used in the pandora configuration
+
+        :return: list of classification bands
+        """
+
+        classif_bands = []
+
+        def search_classes_recursive(obj):
+            """
+            Recursive search of keys containing 'classes' in the configuration
+            """
+            if isinstance(obj, dict):
+                for key, value in obj.items():
+                    if "classes" in key.lower():
+                        if isinstance(value, list):
+                            classif_bands.extend(value)
+                        elif value is not None:
+                            classif_bands.append(value)
+                    # Continue recursive search
+                    search_classes_recursive(value)
+
+        search_classes_recursive(self.pandora_config)
+
+        # Remove duplicates and return the list
+        return list(set(classif_bands))
+
+    @cars_profile(name="Find auto conf")
     def find_auto_conf(
         self, intersection_poly, land_cover_map, classif_to_config_mapping, epsg
     ):
@@ -377,15 +387,15 @@ class PandoraLoader:
 
         return conf
 
-    def check_conf(
+    def check_conf(  # pylint: disable=too-many-positional-arguments
         self,
         user_cfg,
-        img_left,
-        img_right,
+        nodata_left,
+        nodata_right,
         bands_left,
         bands_right,
-        classif_left=None,
-        classif_right=None,
+        bands_classif_left=None,
+        bands_classif_right=None,
     ):
         """
         Check configuration
@@ -397,15 +407,18 @@ class PandoraLoader:
         :rtype: dict
 
         """
-
         # Import plugins before checking configuration
         pandora.import_plugin()
         # Check configuration and update the configuration with default values
         # Instantiate pandora state machine
         pandora_machine = PandoraMachine()
         # check pipeline
-        metadata_left = get_metadata(img_left, classif=classif_left)
-        metadata_right = get_metadata(img_right, classif=classif_right)
+        metadata_left = overide_pandora_get_metadata(
+            bands_left, classif_bands=bands_classif_left
+        )
+        metadata_right = overide_pandora_get_metadata(
+            bands_right, classif_bands=bands_classif_right
+        )
 
         metadata_left = metadata_left.assign_coords(band_im=bands_left)
         metadata_right = metadata_right.assign_coords(band_im=bands_right)
@@ -422,7 +435,9 @@ class PandoraLoader:
             saved_schema
         )
         # check a part of input section
-        user_cfg_input = get_config_input_custom_cars(user_cfg)
+        user_cfg_input = get_config_input_custom_cars(
+            user_cfg, nodata_left, nodata_right
+        )
         cfg_input = check_input_section_custom_cars(user_cfg_input)
         # concatenate updated config
         cfg = concat_conf([cfg_input, cfg_pipeline])
@@ -447,7 +462,46 @@ default_short_configuration_input_custom_cars = {
 }
 
 
-def get_config_input_custom_cars(user_cfg: Dict[str, dict]) -> Dict[str, dict]:
+def overide_pandora_get_metadata(
+    im_bands: list, classif_bands: list = None
+) -> xr.Dataset:
+    """
+    Read metadata from image, and return the corresponding xarray.DataSet
+
+    :param im_bands: list of band names
+    :param classif_bands: list of classification band names
+    :return: partial xarray.DataSet (attributes and coordinates)
+    :rtype: xarray.DataSet
+    """
+
+    coords = {
+        "band_im": list(im_bands),
+        "row": np.arange(10),
+        "col": np.arange(10),
+    }
+
+    data_vars = {
+        "image": (["row", "col"], np.zeros((10, 10))),
+    }
+
+    if classif_bands is not None:
+        coords["band_classif"] = list(classif_bands)
+        data_vars["classif"] = (
+            ["row", "col", "band_classif"],
+            np.zeros((10, 10, len(classif_bands)), dtype=np.int32),
+        )
+
+    # create the dataset
+    dataset = xr.Dataset(data_vars=data_vars, coords=coords)
+
+    dataset.attrs["disparity_source"] = None
+
+    return dataset
+
+
+def get_config_input_custom_cars(
+    user_cfg: Dict[str, dict], nodata_left, nodata_right
+) -> Dict[str, dict]:
     """
     Get the input configuration
 
@@ -464,9 +518,13 @@ def get_config_input_custom_cars(user_cfg: Dict[str, dict]) -> Dict[str, dict]:
 
         if "nodata_left" in user_cfg["input"]:
             cfg["input"]["nodata_left"] = user_cfg["input"]["nodata_left"]
+        else:
+            cfg["input"]["nodata_left"] = nodata_left
 
         if "nodata_right" in user_cfg["input"]:
             cfg["input"]["nodata_right"] = user_cfg["input"]["nodata_right"]
+        else:
+            cfg["input"]["nodata_right"] = nodata_right
 
     return cfg
 
@@ -516,7 +574,7 @@ def overload_pandora_conf_with_confidence(conf, confidence_conf):
 
     for key in confidence_conf_keys:
         if key in conf_keys:
-            logging.error("{} pandora key already in configuration".format(key))
+            logging.info("{} pandora key already in configuration".format(key))
 
     # update confidence
     out_dict.update(confidence_conf)

cars/applications/dsm_filling/border_interpolation_app.py

@@ -56,7 +56,6 @@ class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
 
         # check conf
         self.used_method = self.used_config["method"]
-        self.activated = self.used_config["activated"]
         self.classification = self.used_config["classification"]
         self.component_min_size = self.used_config["component_min_size"]
         self.border_size = self.used_config["border_size"]
@@ -74,7 +73,6 @@ class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
 
         # Overload conf
         overloaded_conf["method"] = conf.get("method", "bulldozer")
-        overloaded_conf["activated"] = conf.get("activated", False)
         overloaded_conf["classification"] = conf.get("classification", "nodata")
         if isinstance(overloaded_conf["classification"], str):
             overloaded_conf["classification"] = [
@@ -91,7 +89,6 @@ class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
 
         rectification_schema = {
             "method": str,
-            "activated": bool,
             "classification": Or(None, [str]),
             "component_min_size": int,
             "border_size": int,
@@ -106,7 +103,7 @@ class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
         return overloaded_conf
 
     @cars_profile(name="Border interpolation filling")
-    def run(  # noqa C901
+    def run(  # pylint: disable=too-many-positional-arguments  # noqa C901
         self,
         dsm_file,
         classif_file,
@@ -127,9 +124,6 @@ class BorderInterpolation(DsmFilling, short_name="border_interpolation"):
             - a Shapely Polygon
         """
 
-        if not self.activated:
-            return
-
         if self.classification is None:
             self.classification = ["nodata"]
             logging.error(

cars/applications/dsm_filling/bulldozer_filling_app.py

@@ -57,7 +57,6 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
 
         # check conf
         self.used_method = self.used_config["method"]
-        self.activated = self.used_config["activated"]
         self.classification = self.used_config["classification"]
         self.save_intermediate_data = self.used_config["save_intermediate_data"]
 
@@ -72,7 +71,6 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
 
         # Overload conf
         overloaded_conf["method"] = conf.get("method", "bulldozer")
-        overloaded_conf["activated"] = conf.get("activated", False)
         overloaded_conf["classification"] = conf.get("classification", "nodata")
 
         if isinstance(overloaded_conf["classification"], str):
@@ -86,7 +84,6 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
 
         rectification_schema = {
             "method": str,
-            "activated": bool,
             "classification": Or(None, [str]),
             "save_intermediate_data": bool,
         }
@@ -98,7 +95,7 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
         return overloaded_conf
 
     @cars_profile(name="Bulldozer filling")
-    def run(  # noqa C901
+    def run(  # pylint: disable=too-many-positional-arguments # noqa C901
         self,
         dsm_file,
         classif_file,
@@ -119,9 +116,6 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
             - a Shapely Polygon
         """
 
-        if not self.activated:
-            return None
-
         if self.classification is None:
             self.classification = ["nodata"]
 
@@ -206,7 +200,7 @@ class BulldozerFilling(DsmFilling, short_name="bulldozer"):
                     ):
                         dsm_to_dtm(bull_conf_path)
         except Exception:
-            logging.error(
+            logging.warning(
                 "Bulldozer failed on its second execution."
                 + " The DSM could not be filled."
             )