cars 1.0.0a3__cp311-cp311-win_amd64.whl → 1.0.0a4__cp311-cp311-win_amd64.whl

cars/applications/point_cloud_outlier_removal/statistical_app.py

@@ -27,9 +27,7 @@ import copy
 
 # Standard imports
 import logging
-import math
 import os
-import time
 
 import numpy as np
 
@@ -40,14 +38,13 @@ from pyproj import CRS
 # CARS imports
 import cars.orchestrator.orchestrator as ocht
 from cars.applications import application_constants
-from cars.applications.point_cloud_fusion import (
-    pc_fusion_algo,
-    pc_fusion_wrappers,
-)
 from cars.applications.point_cloud_outlier_removal import (
     abstract_outlier_removal_app as pc_removal,
 )
-from cars.applications.point_cloud_outlier_removal import outlier_removal_algo
+from cars.applications.point_cloud_outlier_removal import (
+    outlier_removal_algo,
+)
+from cars.applications.triangulation import pc_transform
 from cars.applications.triangulation.triangulation_wrappers import (
     generate_point_cloud_file_names,
 )
@@ -83,7 +80,6 @@ class Statistical(
         self.used_method = self.used_config["method"]
 
         # statistical outliers
-        self.activated = self.used_config["activated"]
         self.k = self.used_config["k"]
         self.filtering_constant = self.used_config["filtering_constant"]
         self.mean_factor = self.used_config["mean_factor"]
@@ -126,10 +122,6 @@ class Statistical(
         overloaded_conf["use_median"] = conf.get("use_median", True)
 
         # statistical outlier filtering
-        overloaded_conf["activated"] = conf.get(
-            "activated", True
-        )  # if false, the following
-        # parameters are unused
         # k: number of neighbors
         overloaded_conf["k"] = conf.get("k", 50)
         # filtering_constant: constant to apply in the distance threshold
@@ -154,7 +146,6 @@ class Statistical(
         point_cloud_outlier_removal_schema = {
             "method": str,
             "save_by_pair": bool,
-            "activated": bool,
             "k": And(int, lambda x: x > 0),
             "filtering_constant": And(Or(float, int), lambda x: x >= 0),
             "mean_factor": And(Or(float, int), lambda x: x >= 0),
@@ -191,17 +182,12 @@ class Statistical(
 
         """
 
-        if not self.activated:
-            # if not activated, this tile size must not be taken into acount
-            # during the min(*tile_sizes) operations
-            tile_size = math.inf
-        else:
-            tot = 10000 * superposing_point_clouds / point_cloud_resolution
+        tot = 10000 * superposing_point_clouds / point_cloud_resolution
 
-            import_ = 200  # MiB
-            tile_size = int(
-                np.sqrt(float(((max_ram_per_worker - import_) * 2**23)) / tot)
-            )
+        import_ = 200  # MiB
+        tile_size = int(
+            np.sqrt(float(((max_ram_per_worker - import_) * 2**23)) / tot)
+        )
 
         logging.info(
             "Estimated optimal tile size for statistical "
@@ -228,11 +214,7 @@ class Statistical(
         :return: margin
         :rtype: int
         """
-
-        margin = 0
-
-        if self.activated:
-            margin = self.half_epipolar_size
+        margin = self.half_epipolar_size
 
         return margin
 
@@ -247,7 +229,7 @@ class Statistical(
 
         return 0
 
-    def run(
+    def run(  # pylint: disable=too-many-positional-arguments
         self,
         merged_point_cloud,
         orchestrator=None,
@@ -300,9 +282,6 @@ class Statistical(
         :rtype : CarsDataset filled with xr.Dataset
         """
 
-        if not self.activated:
-            return merged_point_cloud
-
         # Default orchestrator
         if orchestrator is None:
             # Create default sequential orchestrator for current application
@@ -314,263 +293,103 @@ class Statistical(
         else:
             self.orchestrator = orchestrator
 
-        if merged_point_cloud.dataset_type == "points":
-            (
-                filtered_point_cloud,
-                laz_pc_file_name,
-                csv_pc_file_name,
-                saving_info,
-            ) = self.__register_pc_dataset__(
-                merged_point_cloud,
-                point_cloud_dir,
-                dump_dir,
-                app_name="statistical",
-            )
-
-            logging.info(
-                "Cloud filtering: Filtered points number: {}".format(
-                    filtered_point_cloud.shape[1]
-                    * filtered_point_cloud.shape[0]
-                )
-            )
+        if dump_dir is None:
+            dump_dir = self.generate_unknown_dump_dir(self.orchestrator)
 
-            # Generate rasters
-            for col in range(filtered_point_cloud.shape[1]):
-                for row in range(filtered_point_cloud.shape[0]):
-                    # update saving infos  for potential replacement
-                    full_saving_info = ocht.update_saving_infos(
-                        saving_info, row=row, col=col
-                    )
-                    if merged_point_cloud.tiles[row][col] is not None:
-                        # Delayed call to cloud filtering
-                        filtered_point_cloud[
-                            row, col
-                        ] = self.orchestrator.cluster.create_task(
-                            statistical_removal_wrapper
-                        )(
-                            merged_point_cloud[row, col],
-                            self.k,
-                            self.filtering_constant,
-                            self.mean_factor,
-                            self.std_dev_factor,
-                            self.use_median,
-                            save_by_pair=(self.save_by_pair),
-                            point_cloud_csv_file_name=csv_pc_file_name,
-                            point_cloud_laz_file_name=laz_pc_file_name,
-                            saving_info=full_saving_info,
-                        )
-
-        elif merged_point_cloud.dataset_type == "arrays":
-            prefix = os.path.basename(dump_dir)
-            # Save as depth map
-            filtered_point_cloud, saving_info_epipolar = (
-                self.__register_epipolar_dataset__(
-                    merged_point_cloud,
-                    depth_map_dir,
-                    dump_dir,
-                    app_name="statistical",
-                    pair_key=prefix,
-                )
+        if merged_point_cloud.dataset_type != "arrays":
+            raise RuntimeError(
+                "Only arrays is supported in statistical removal"
             )
 
-            # Save as point cloud
-            (
-                flatten_filtered_point_cloud,
-                laz_pc_dir_name,
-                csv_pc_dir_name,
-                saving_info_flatten,
-            ) = self.__register_pc_dataset__(
+        prefix = os.path.basename(dump_dir)
+        # Save as depth map
+        filtered_point_cloud, saving_info_epipolar = (
+            self.__register_epipolar_dataset__(
                 merged_point_cloud,
-                point_cloud_dir,
+                depth_map_dir,
                 dump_dir,
                 app_name="statistical",
+                pair_key=prefix,
             )
+        )
 
-            # initialize empty index file for point cloud product if official
-            # product is requested
-            pc_index = None
-            if point_cloud_dir:
-                pc_index = {}
+        # Save as point cloud
+        (
+            flatten_filtered_point_cloud,
+            laz_pc_dir_name,
+            csv_pc_dir_name,
+            saving_info_flatten,
+        ) = self.__register_pc_dataset__(
+            merged_point_cloud,
+            point_cloud_dir,
+            dump_dir,
+            app_name="statistical",
+        )
+
+        # initialize empty index file for point cloud product if official
+        # product is requested
+        pc_index = None
+        if point_cloud_dir:
+            pc_index = {}
 
-            # Generate rasters
-            for col in range(filtered_point_cloud.shape[1]):
-                for row in range(filtered_point_cloud.shape[0]):
+        # Generate rasters
+        for col in range(filtered_point_cloud.shape[1]):
+            for row in range(filtered_point_cloud.shape[0]):
 
-                    # update saving infos  for potential replacement
-                    full_saving_info_epipolar = ocht.update_saving_infos(
-                        saving_info_epipolar, row=row, col=col
+                # update saving infos  for potential replacement
+                full_saving_info_epipolar = ocht.update_saving_infos(
+                    saving_info_epipolar, row=row, col=col
+                )
+                full_saving_info_flatten = None
+                if saving_info_flatten is not None:
+                    full_saving_info_flatten = ocht.update_saving_infos(
+                        saving_info_flatten, row=row, col=col
                     )
-                    full_saving_info_flatten = None
-                    if saving_info_flatten is not None:
-                        full_saving_info_flatten = ocht.update_saving_infos(
-                            saving_info_flatten, row=row, col=col
-                        )
 
-                    if merged_point_cloud[row][col] is not None:
-                        csv_pc_file_name, laz_pc_file_name = (
-                            generate_point_cloud_file_names(
-                                csv_pc_dir_name,
-                                laz_pc_dir_name,
-                                row,
-                                col,
-                                pc_index,
-                                pair_key=prefix,
-                            )
+                if merged_point_cloud[row][col] is not None:
+                    csv_pc_file_name, laz_pc_file_name = (
+                        generate_point_cloud_file_names(
+                            csv_pc_dir_name,
+                            laz_pc_dir_name,
+                            row,
+                            col,
+                            pc_index,
+                            pair_key=prefix,
                         )
-                        window = merged_point_cloud.tiling_grid[row, col]
-                        overlap = filtered_point_cloud.overlaps[row, col]
-                        # Delayed call to cloud filtering
-                        (
-                            filtered_point_cloud[row, col],
-                            flatten_filtered_point_cloud[row, col],
-                        ) = self.orchestrator.cluster.create_task(
-                            epipolar_statistical_removal_wrapper, nout=2
-                        )(
-                            merged_point_cloud[row, col],
-                            self.k,
-                            self.filtering_constant,
-                            self.mean_factor,
-                            self.std_dev_factor,
-                            self.use_median,
-                            self.half_epipolar_size,
-                            window,
-                            overlap,
-                            epsg=epsg,
-                            point_cloud_csv_file_name=csv_pc_file_name,
-                            point_cloud_laz_file_name=laz_pc_file_name,
-                            saving_info_epipolar=full_saving_info_epipolar,
-                            saving_info_flatten=full_saving_info_flatten,
-                        )
-
-            # update point cloud index
-            if point_cloud_dir:
-                self.orchestrator.update_index(pc_index)
+                    )
+                    window = merged_point_cloud.tiling_grid[row, col]
+                    overlap = filtered_point_cloud.overlaps[row, col]
+                    # Delayed call to cloud filtering
+                    (
+                        filtered_point_cloud[row, col],
+                        flatten_filtered_point_cloud[row, col],
+                    ) = self.orchestrator.cluster.create_task(
+                        epipolar_statistical_removal_wrapper, nout=2
+                    )(
+                        merged_point_cloud[row, col],
+                        self.k,
+                        self.filtering_constant,
+                        self.mean_factor,
+                        self.std_dev_factor,
+                        self.use_median,
+                        self.half_epipolar_size,
+                        window,
+                        overlap,
+                        epsg=epsg,
+                        point_cloud_csv_file_name=csv_pc_file_name,
+                        point_cloud_laz_file_name=laz_pc_file_name,
+                        saving_info_epipolar=full_saving_info_epipolar,
+                        saving_info_flatten=full_saving_info_flatten,
+                    )
 
-        else:
-            logging.error(
-                "PointCloudOutlierRemoval application doesn't support"
-                "this input data "
-                "format"
-            )
+        # update point cloud index
+        if point_cloud_dir:
+            self.orchestrator.update_index(pc_index)
 
         return filtered_point_cloud
 
 
-def statistical_removal_wrapper(
-    cloud,
-    statistical_k,
-    filtering_constant,
-    mean_factor,
-    std_dev_factor,
-    use_median,
-    save_by_pair: bool = False,
-    point_cloud_csv_file_name=None,
-    point_cloud_laz_file_name=None,
-    saving_info=None,
-):
-    """
-    Statistical outlier removal
-
-    :param cloud: cloud to filter
-    :type cloud: pandas DataFrame
-    :param statistical_k: k
-    :type statistical_k: int
-    :param filtering_constant: constant applied to the threshold
-    :type filtering_constant: float
-    :param mean_factor: mean factor
-    :type mean_factor: float
-    :param std_dev_factor: std factor
-    :type std_dev_factor: float
-    :param use_median: use median and quartile instead of mean and std
-    :type use median: bool
-    :param save_by_pair: save point cloud as pair
-    :type save_by_pair: bool
-    :param point_cloud_csv_file_name: write point cloud as CSV in filename
-        (if None, the point cloud is not written as csv)
-    :type point_cloud_csv_file_name: str
-    :param point_cloud_laz_file_name: write point cloud as laz in filename
-        (if None, the point cloud is not written as laz)
-    :type point_cloud_laz_file_name: str
-    :param saving_info: saving infos
-    :type saving_info: dict
-
-    :return: filtered cloud
-    :rtype: pandas DataFrame
-
-    """
-
-    # Copy input cloud
-    new_cloud = cloud.copy()
-    new_cloud.attrs = copy.deepcopy(cloud.attrs)
-
-    # Get current epsg
-    cloud_attributes = cars_dataset.get_attributes(new_cloud)
-    cloud_epsg = cloud_attributes["epsg"]
-    current_epsg = cloud_epsg
-
-    # Check if can be used to filter
-    spatial_ref = CRS.from_epsg(cloud_epsg)
-    if spatial_ref.is_geographic:
-        logging.debug(
-            "The point cloud to filter is not in a cartographic system. "
-            "The filter's default parameters might not be adapted "
-            "to this referential. Convert the points "
-            "cloud to ECEF to ensure a proper point_cloud."
-        )
-        # Convert to epsg = 4978
-        cartographic_epsg = 4978
-        projection.point_cloud_conversion_dataframe(
-            new_cloud, current_epsg, cartographic_epsg
-        )
-        current_epsg = cartographic_epsg
-
-    # Filter point cloud
-    tic = time.process_time()
-    (new_cloud, _) = outlier_removal_algo.statistical_outlier_filtering(
-        new_cloud,
-        statistical_k,
-        filtering_constant,
-        mean_factor,
-        std_dev_factor,
-        use_median,
-    )
-    toc = time.process_time()
-    logging.debug(
-        "Statistical cloud filtering done in {} seconds".format(toc - tic)
-    )
-
-    # Conversion to UTM
-    projection.point_cloud_conversion_dataframe(
-        new_cloud, cloud_epsg, current_epsg
-    )
-    # Update attributes
-    cloud_attributes["epsg"] = current_epsg
-
-    cars_dataset.fill_dataframe(
-        new_cloud, saving_info=saving_info, attributes=cloud_attributes
-    )
-
-    # save point cloud in worker
-    if point_cloud_csv_file_name:
-        cars_dataset.run_save_points(
-            new_cloud,
-            point_cloud_csv_file_name,
-            save_by_pair=save_by_pair,
-            overwrite=True,
-            point_cloud_format="csv",
-        )
-    if point_cloud_laz_file_name:
-        cars_dataset.run_save_points(
-            new_cloud,
-            point_cloud_laz_file_name,
-            save_by_pair=save_by_pair,
-            overwrite=True,
-            point_cloud_format="laz",
-        )
-
-    return new_cloud
-
-
+# pylint: disable=too-many-positional-arguments
 def epipolar_statistical_removal_wrapper(
     epipolar_ds,
     statistical_k,
@@ -619,9 +438,29 @@ def epipolar_statistical_removal_wrapper(
     # Copy input cloud
     filtered_cloud = copy.copy(epipolar_ds)
 
+    # Get current epsg
+    cloud_epsg = filtered_cloud.attrs["epsg"]
+    current_epsg = cloud_epsg
+
+    # Check if can be used to filter
+    spatial_ref = CRS.from_epsg(cloud_epsg)
+    if spatial_ref.is_geographic:
+        logging.debug(
+            "The point cloud to filter is not in a cartographic system. "
+            "The filter's default parameters might not be adapted "
+            "to this referential. Please, convert the point "
+            "cloud to ECEF to ensure a proper point_cloud."
+        )
+        # Convert to epsg = 4978
+        cartographic_epsg = 4978
+
+        projection.point_cloud_conversion_dataset(
+            filtered_cloud, cartographic_epsg
+        )
+        current_epsg = cartographic_epsg
+
     outlier_removal_algo.epipolar_statistical_filtering(
         filtered_cloud,
-        epsg,
         k=statistical_k,
         filtering_constant=filtering_constant,
         mean_factor=mean_factor,
@@ -645,9 +484,11 @@ def epipolar_statistical_removal_wrapper(
     if point_cloud_csv_file_name or point_cloud_laz_file_name:
         # Convert epipolar array into point cloud
         flatten_filtered_cloud, cloud_epsg = (
-            pc_fusion_algo.create_combined_cloud([filtered_cloud], ["0"], epsg)
+            pc_transform.depth_map_dataset_to_dataframe(
+                filtered_cloud, current_epsg
+            )
         )
-        # Convert to UTM
+        # Convert to wanted epsg
         if epsg is not None and cloud_epsg != epsg:
             projection.point_cloud_conversion_dataframe(
                 flatten_filtered_cloud, cloud_epsg, epsg
@@ -655,7 +496,7 @@ def epipolar_statistical_removal_wrapper(
             cloud_epsg = epsg
 
         # Fill attributes for LAZ saving
-        color_type = pc_fusion_wrappers.get_color_type([filtered_cloud])
+        color_type = pc_transform.get_color_type([filtered_cloud])
         attributes = {
             "epsg": cloud_epsg,
             "color_type": color_type,

cars/applications/rasterization/abstract_pc_rasterization_app.py

@@ -127,7 +127,7 @@ class PointCloudRasterization(ApplicationTemplate, metaclass=ABCMeta):
         """
 
     @abstractmethod
-    def run(
+    def run(  # pylint: disable=too-many-positional-arguments
         self,
         point_clouds,
         epsg,

cars/applications/rasterization/rasterization_algo.py

@@ -45,6 +45,7 @@ from cars.core import constants as cst
 from cars.data_structures import cars_dataset
 
 
+# pylint: disable=too-many-positional-arguments
 def simple_rasterization_dataset_wrapper(
     cloud: pandas.DataFrame,
     resolution: float,
@@ -61,6 +62,7 @@ def simple_rasterization_dataset_wrapper(
     list_computed_layers: List[str] = None,
     source_pc_names: List[str] = None,
     performance_map_classes: List[float] = None,
+    cloud_global_id: int = None,
 ) -> xr.Dataset:
     """
     Wrapper of simple_rasterization
@@ -91,6 +93,8 @@ def simple_rasterization_dataset_wrapper(
         name of sensors pair or name of point cloud file
     :param performance_map_classes: list for step defining border of class
     :type performance_map_classes: list or None
+    :param cloud_global_id: global id of pair
+    :type cloud_global_id: int
     :return: Rasterized cloud
     """
 
@@ -129,11 +133,13 @@ def simple_rasterization_dataset_wrapper(
         list_computed_layers=list_computed_layers,
         source_pc_names=source_pc_names,
         performance_map_classes=performance_map_classes,
+        cloud_global_id=cloud_global_id,
     )
 
     return raster
 
 
+# pylint: disable=too-many-positional-arguments
 def compute_vector_raster_and_stats(
     cloud: pandas.DataFrame,
     x_start: float,
@@ -144,6 +150,7 @@ def compute_vector_raster_and_stats(
     sigma: float,
     radius: int,
     list_computed_layers: List[str] = None,
+    cloud_global_id: int = None,
 ) -> Tuple[
     np.ndarray,
     np.ndarray,
@@ -168,6 +175,7 @@ def compute_vector_raster_and_stats(
     :param sigma: Sigma for gaussian interpolation. If None, set to resolution
     :param radius: Radius for hole filling.
     :param list_computed_layers: list of computed output data
+    :param cloud_global_id: global id of pair
     :return: a tuple with rasterization results and statistics.
     """
     # get points corresponding to (X, Y positions) + data_valid
@@ -225,8 +233,8 @@ def compute_vector_raster_and_stats(
     # Fill the dataframe with additional columns :
     # each column refers to a point cloud id
     number_of_pc = cars_dataset.get_attributes(cloud)["number_of_pc"]
-    if cst.POINT_CLOUD_GLOBAL_ID in cloud.columns and (
-        (list_computed_layers is None)
+    if (cloud_global_id is not None) and (
+        list_computed_layers is None
         or rast_wrap.substring_in_list(
             list_computed_layers, cst.POINT_CLOUD_SOURCE_KEY_ROOT
         )
@@ -234,9 +242,10 @@ def compute_vector_raster_and_stats(
         for pc_id in range(number_of_pc):
             # Create binary list that indicates from each point whether it comes
             # from point cloud number "pc_id"
-            point_is_from_pc = list(
-                map(int, cloud[cst.POINT_CLOUD_GLOBAL_ID] == pc_id)
-            )
+            if pc_id == cloud_global_id:
+                point_is_from_pc = np.ones(cloud.shape[0], dtype=int)
+            else:
+                point_is_from_pc = np.zeros(cloud.shape[0], dtype=int)
             pc_key = "{}{}".format(cst.POINT_CLOUD_SOURCE_KEY_ROOT, pc_id)
             cloud[pc_key] = point_is_from_pc
 
@@ -346,7 +355,7 @@ def compute_vector_raster_and_stats(
     )
 
 
-def rasterize(
+def rasterize(  # pylint: disable=too-many-positional-arguments
     cloud: pandas.DataFrame,
     resolution: float,
     epsg: int,
@@ -362,6 +371,7 @@ def rasterize(
     list_computed_layers: List[str] = None,
     source_pc_names: List[str] = None,
     performance_map_classes: List[float] = None,
+    cloud_global_id: int = None,
 ) -> Union[xr.Dataset, None]:
     """
     Rasterize a point cloud with its color bands to a Dataset
@@ -385,6 +395,7 @@ def rasterize(
     :param source_pc_names: list of source pc names
     :param performance_map_classes: list for step defining border of class
     :type performance_map_classes: list or None
+    :param cloud_global_id: global id of pair
     :return: Rasterized cloud color and statistics.
     """
 
@@ -432,6 +443,7 @@ def rasterize(
         sigma,
         radius,
         list_computed_layers,
+        cloud_global_id=cloud_global_id,
     )
 
     # reshape data as a 2d grid.

cars/applications/rasterization/rasterization_wrappers.py

@@ -154,6 +154,7 @@ def find_indexes_in_point_cloud(
     return indexes
 
 
+# pylint: disable=too-many-positional-arguments
 def create_raster_dataset(  # noqa: C901
     raster: np.ndarray,
     weights_sum: np.ndarray,
@@ -548,7 +549,7 @@ def update_weights(old_weights, weights):
     return new_weights
 
 
-def update_data(
+def update_data(  # pylint: disable=too-many-positional-arguments
     old_data, current_data, weights, old_weights, nodata, method="basic"
 ):
     """