cars 1.0.0a3__cp39-cp39-win_amd64.whl → 1.0.0a4__cp39-cp39-win_amd64.whl

cars/data_structures/corresponding_tiles_tools.py

@@ -27,8 +27,6 @@ Contains functions for array reconstructions and crop for multiple tiles
 import numpy as np
 import xarray as xr
 
-import cars.core.constants as cst
-
 
 def reconstruct_data(tiles, window, overlap):  # noqa: C901
     """
@@ -186,104 +184,3 @@ def reconstruct_data(tiles, window, overlap):  # noqa: C901
             dims=dims,
         ).astype(data_type)
     return new_dataset, row_min - ol_row_min, col_min - ol_col_min
-
-
-def find_tile_dataset(corresponding_tiles, window):
-    """
-    Find the dataset corresponding to window, in the list of tiles.
-
-    :param corresponding_tiles: list of tiles
-    :type corresponding_tiles: list(tuple)
-    :param window: window of base tile [row min, row max, col min col max]
-    :type window: list
-
-    :return: dataset corresponding to window
-    :rtype: xr.Dataset
-
-    """
-
-    dataset = None
-    for tile_window, _, tile_dataset in corresponding_tiles:
-        if tuple(tile_window) == tuple(window):
-            dataset = tile_dataset
-
-    return dataset
-
-
-def crop_dataset(full_dataset, in_dataset, window, overlap, row_min, col_min):
-    """
-    Crop full dataset to fit with a given tile dataset
-
-    :param full_dataset: Combined dataset
-    :type full_dataset: xr.Dataset
-    :param in_dataset: dataset to use as template dataset
-    :type in_dataset: xr.Dataset
-    :param window: window of base tile [row min, row max, col min col max]
-    :type window: list
-    :param overlap: overlap of base tile [row min, row max, col min col max]
-    :type overlap: list
-    :param row_min: position of row min in full image
-    :type row_min: int
-    :param col_min: position of col min in full image
-    :type col_min: int
-
-    :return: cropped dataset
-    :rtype: xr.Dataset
-
-    """
-    cropped = xr.Dataset()
-
-    coords = {}
-    if cst.ROW in in_dataset:
-        coords[cst.ROW] = in_dataset.coords[cst.ROW]
-    if cst.COL in in_dataset:
-        coords[cst.COL] = in_dataset.coords[cst.COL]
-
-    list_tags = list(full_dataset.keys())
-
-    for tag in list_tags:
-        full_data = full_dataset[tag].values
-
-        offset_row = int(window[0] - overlap[0] - row_min)
-        offset_col = int(window[2] - overlap[2] - col_min)
-
-        if len(full_data.shape) == 2:
-            nb_row = int(in_dataset[tag].values.shape[0])
-            nb_col = int(in_dataset[tag].values.shape[1])
-
-            values = np.ascontiguousarray(
-                full_data[
-                    offset_row : offset_row + nb_row,
-                    offset_col : offset_col + nb_col,
-                ]
-            )
-
-        else:
-            nb_row = int(in_dataset[tag].values.shape[1])
-            nb_col = int(in_dataset[tag].values.shape[2])
-
-            values = np.ascontiguousarray(
-                full_data[
-                    :,
-                    offset_row : offset_row + nb_row,
-                    offset_col : offset_col + nb_col,
-                ]
-            )
-
-        xarray_coords = {}
-        for coord_tag in full_dataset[tag].coords:
-            if coord_tag in coords:
-                xarray_coords[coord_tag] = coords[coord_tag]
-            else:
-                xarray_coords[coord_tag] = full_dataset[tag].coords[coord_tag]
-
-        dims = full_dataset[tag].dims
-
-        data_array = xr.DataArray(values, coords=xarray_coords, dims=dims)
-        if tag in in_dataset:
-            data_array.attrs = in_dataset[tag].attrs
-        cropped[tag] = data_array
-
-    cropped.attrs = in_dataset.attrs
-
-    return cropped

cars/data_structures/format_transformation.py

@@ -30,9 +30,12 @@ import math
 import numpy as np
 import xarray as xr
 
+from cars.orchestrator.cluster.log_wrapper import cars_profile
+
 # CARS imports
 
 
+@cars_profile(name="Grid margins 2 overlaps", interval=0.5)
 def grid_margins_2_overlaps(grid, margins_fun):
     """
     Convert margins to overlap grid format used in CarsDatasets
@@ -48,7 +51,7 @@ def grid_margins_2_overlaps(grid, margins_fun):
 
     """
 
-    def fill_overlap(
+    def fill_overlap(  # pylint: disable=too-many-positional-arguments
         cars_ds_overlaps,
         margins,
         row_up,

cars/devibrate.py

@@ -95,7 +95,7 @@ def acquisition_direction(
     return time_direction_vector, vec1, vec2
 
 
-def get_time_ground_direction(
+def get_time_ground_direction(  # pylint: disable=too-many-positional-arguments
     sensor,
     geomodel,
     geometry_plugin,
@@ -180,6 +180,7 @@ def project_coordinates_on_line(
     return dist_to_origin * np.cos(proj_angle)
 
 
+# pylint: disable=too-many-positional-arguments
 def lowres_initial_dem_splines_fit(
     lowres_dsm_from_matches: xr.Dataset,
     lowres_initial_dem: xr.Dataset,
@@ -379,7 +380,7 @@ def read_lowres_dsm(srtm_path, startx, starty, endx, endy):
     return dsm_as_ds, newstartx, newstarty, sizex, sizey, resolution
 
 
-def compute_splines(
+def compute_splines(  # pylint: disable=too-many-positional-arguments
     sensor1,
     geomodel1,
     sensor2,
@@ -507,7 +508,7 @@ def compute_splines(
     }
 
 
-def cars_devibrate(
+def cars_devibrate(  # pylint: disable=too-many-positional-arguments
     used_conf,
     srtm_path,
     geoid_path,
@@ -594,6 +595,8 @@ def cars_devibrate(
         )
 
         x_values_2d, y_values_2d = np.meshgrid(x_values_1d, y_values_1d)
+
+        positions = None
         if src.crs != "EPSG:4326":
             transformer = pyproj.Transformer.from_crs(
                 src.crs, "EPSG:4326", always_xy=True

cars/extractroi.py

@@ -29,6 +29,7 @@ import numpy as np
 import rasterio as rio
 from affine import Affine
 from shapely.geometry import box
+from shareloc.geomodels.rpc_writers import write_rio_rpc_as_rpb
 
 
 def is_bbx_in_image(bbx, image_dataset):
@@ -75,7 +76,7 @@ def get_slices_from_bbx(image_dataset, bbx, rpc_options):
 
 
 def process_image_file(
-    bbx, input_image_path, output_image_path, geom_file_path, rpc_options
+    bbx, input_image_path, output_image_path, rpb_file_path, rpc_options
 ):
     """
     Processes an image file by extracting a region based on the given geometry.
@@ -84,7 +85,7 @@ def process_image_file(
         region_geometry (dict): GeoJSON-like dictionary defining the region.
         input_image_path (str): Path to the input image file.
         output_image_path (str): Path to save the output image.
-        geom_file_path (str): Path to save the .geom file.
+        rpb_file_path (str): Path to save the .RPB file.
         rpc_options (dict): Options for GDALCreateRPCTransformer.
     """
 
@@ -110,7 +111,8 @@ def process_image_file(
             # copy rpc
             dst.rpcs = image_dataset.rpcs
 
-        create_geom_file(image_dataset, geom_file_path)
+        if rpb_file_path is not None:
+            create_rpb_file(image_dataset, rpb_file_path)
 
 
 def get_human_readable_bbox(image_dataset, rpc_options):
@@ -167,34 +169,23 @@ def validate_bounding_box(bbx, image_dataset, rpc_options):
             image_dataset, rpc_options
         )
         raise ValueError(
-            f"Coordinates must between "
+            f"Coordinates must be between "
             f"({min_x}, {min_y}) and ({max_x}, {max_y})"
         )
 
 
-def create_geom_file(image_dataset, geom_filename):
+def create_rpb_file(image_dataset, rpb_filename):
     """
-    Create and save a .geom file from a rasterio dataset
+    Create and save a .RPB file from a rasterio dataset
 
     Parameters:
         image_dataset (rio.DatasetReader): Opened image dataset.
-        geom_filename (str): Path to save the .geom file.
+        rpb_filename (str): Path to save the .RPB file.
     """
     if not image_dataset.rpcs:
         raise ValueError("Image dataset has no RPCs")
     rpcs_as_dict = image_dataset.rpcs.to_dict()
-    with open(geom_filename, "w", encoding="utf-8") as writer:
-        for key in rpcs_as_dict:
-            if isinstance(rpcs_as_dict[key], list):
-                for idx, coef in enumerate(rpcs_as_dict[key]):
-                    writer.write(key + "_%02d" % idx + ": " + str(coef))
-                    writer.write("\n")
-            else:
-                writer.write(key + ": " + str(rpcs_as_dict[key]))
-                writer.write("\n")
-
-        writer.write("type:  ossimRpcModel\n")
-        writer.write("polynomial_format:  B\n")
+    write_rio_rpc_as_rpb(rpcs_as_dict, rpb_filename)
 
 
 def main():
@@ -240,6 +231,12 @@ def main():
         help="Digital Elevation Model used for projection",
     )
 
+    parser.add_argument(
+        "--generate_rpb",
+        action="store_true",
+        help="Generate RPB file",
+    )
+
     args = parser.parse_args()
     if not os.path.exists(args.out):
         os.makedirs(args.out)
@@ -254,10 +251,12 @@ def main():
     # check first input in list to determine pipeline
     for idx, image_path in enumerate(args.il):
         output_image_path = os.path.join(args.out, "ext_%03d.tif" % idx)
-        geom_file_path = os.path.splitext(output_image_path)[0] + ".geom"
+        rpb_file_path = None
+        if args.generate_rpb:
+            rpb_file_path = os.path.splitext(output_image_path)[0] + ".RPB"
 
         process_image_file(
-            args.bbx, image_path, output_image_path, geom_file_path, rpc_options
+            args.bbx, image_path, output_image_path, rpb_file_path, rpc_options
         )
 
 

cars/orchestrator/cluster/abstract_cluster.py

@@ -48,8 +48,13 @@ class AbstractCluster(metaclass=ABCMeta):
     # cluster mode output directory
     out_dir: str
 
-    def __new__(  # pylint: disable=W0613
-        cls, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __new__(  # pylint: disable=too-many-positional-arguments
+        cls,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):
         """
         Return the required cluster
@@ -98,8 +103,13 @@ class AbstractCluster(metaclass=ABCMeta):
 
         return decorator
 
-    def __init__(
-        self, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):  # pylint: disable=W0613
         """
         Init function of AbstractCluster
@@ -114,7 +124,7 @@ class AbstractCluster(metaclass=ABCMeta):
         # data to propagate
         self.data_to_propagate = data_to_propagate
 
-        self.worker_log_dir = os.path.join(out_dir, "logs", "workers_log")
+        self.worker_log_dir = os.path.join(log_dir, "workers_log")
         if not os.path.exists(self.worker_log_dir):
             os.makedirs(self.worker_log_dir)
 

cars/orchestrator/cluster/abstract_dask_cluster.py

@@ -56,7 +56,7 @@ class AbstractDaskCluster(
     AbstractDaskCluster
     """
 
-    def __init__(self, conf_cluster, out_dir, launch_worker=True):
+    def __init__(self, conf_cluster, out_dir, log_dir, launch_worker=True):
         """
         Init function of AbstractDaskCluster
 
@@ -64,8 +64,12 @@ class AbstractDaskCluster(
 
         """
 
+        print("cluster log_dir", log_dir)
+
         # call parent init
-        super().__init__(conf_cluster, out_dir, launch_worker=launch_worker)
+        super().__init__(
+            conf_cluster, out_dir, log_dir, launch_worker=launch_worker
+        )
         # retrieve parameters
         self.nb_workers = self.checked_conf_cluster["nb_workers"]
         self.task_timeout = self.checked_conf_cluster["task_timeout"]

cars/orchestrator/cluster/dask_jobqueue_utils.py

@@ -30,7 +30,7 @@ import warnings
 from datetime import timedelta
 
 
-def init_cluster_variables(
+def init_cluster_variables(  # pylint: disable=too-many-positional-arguments
     nb_workers,
     walltime,
     out_dir,

cars/orchestrator/cluster/log_wrapper.py

@@ -26,6 +26,7 @@ Contains functions for wrapper logs
 import copy
 import cProfile
 import datetime
+import functools
 import gc
 import io
 import logging
@@ -44,6 +45,8 @@ import numpy as np
 import pandas as pd
 import psutil
 from json_checker import Checker
+from matplotlib.backends.backend_pdf import PdfPages
+from PIL import Image
 
 from cars.core import cars_logging
 from cars.core.utils import safe_makedirs
@@ -496,11 +499,37 @@ def log_delta_memory(func, memory_start, memory_end):
     log_message(func, message)
 
 
+def exception_safe(func):
+    """
+    Decorator for consistent exception handling in profiling functions
+
+    :param func: function to wrap
+    :return: wrapped function
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        """
+        Catch error
+        """
+        try:
+            return func(*args, **kwargs)
+        except Exception as exc:
+            error_msg = (
+                f"Error in {func.__name__}: {type(exc).__name__}: {str(exc)}"
+            )
+            logging.error(error_msg)
+            cars_logging.add_profiling_message(f"ERROR - {error_msg}")
+            return None
+
+    return wrapper
+
+
+@exception_safe
 def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     """
     Generate Profiling summary
     """
-
     nb_workers = 1
     if "orchestrator" not in used_conf:
         first_key = next(iter(used_conf))
@@ -510,16 +539,15 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         if "nb_workers" in used_conf["orchestrator"]:
             nb_workers = used_conf["orchestrator"]["nb_workers"]
 
-    workers_log_dir = os.path.join(out_dir, "logs", "workers_log")
+    workers_log_dir = os.path.join(out_dir, "workers_log")
+    os.makedirs(workers_log_dir, exist_ok=True)
 
     log_file_main = os.path.join(
         workers_log_dir,
         "profiling.log",
     )
 
-    out_profiling_main = os.path.join(
-        out_dir, "logs", "profiling", "profiling.log"
-    )
+    out_profiling_main = os.path.join(out_dir, "profiling", "profiling.log")
 
     log_files = [log_file_main, out_profiling_main]
 
@@ -640,7 +668,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         cars_logging.add_profiling_message(message)
 
     # Generate png
-    _, axs = plt.subplots(4, 2, figsize=(15, 15), layout="tight")
+    _, axs = plt.subplots(3, 2, figsize=(20, 20), layout="tight")
     # Fill
 
     generate_boxplot(
@@ -689,6 +717,14 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         "calls",
     )
 
+    # file_name
+    profiling_plot = os.path.join(
+        out_dir,
+        "profiling",
+        "profiling_plots_histograms.png",
+    )
+    plt.savefig(profiling_plot)
+
     # Pie chart
 
     (name_task_workers, summary_workers) = filter_lists(
@@ -704,7 +740,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     )
 
     (_, [pipeline_time]) = filter_lists(
-        summary_names, summary_total_time, lambda name: "pipeline" in name
+        summary_names, summary_total_time, lambda name: "unit_pipeline" in name
     )
 
     (_, [multiprocessing_time]) = filter_lists(
@@ -717,8 +753,10 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
 
     total_time_workers = nb_workers * multiprocessing_time
 
+    _, axs2 = plt.subplots(2, 1, figsize=(40, 40), layout="tight")
+
     generate_pie_chart(
-        axs.flat[6],
+        axs2.flat[0],
         name_task_workers,
         100 * np.array(summary_workers) / total_time_workers,
         "Total time in parallel tasks ({} workers) : {}".format(
@@ -728,7 +766,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     )
 
     generate_pie_chart(
-        axs.flat[7],
+        axs2.flat[1],
         name_task_main,
         100 * np.array(summary_main) / sequential_time,
         "Total time in sequential tasks : {}".format(
@@ -736,19 +774,103 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         ),
     )
 
-    # file_name
-    profiling_plot = os.path.join(
+    profiling_plot2 = os.path.join(
         out_dir,
-        "logs",
         "profiling",
-        "profiling_plots.png",
+        "profiling_plots_pie_chart.png",
     )
-    plt.savefig(profiling_plot)
+    plt.savefig(profiling_plot2)
 
     if clean_worker_logs and os.path.exists(workers_log_dir):
         shutil.rmtree(workers_log_dir)
 
 
+def generate_pdf_profiling(log_dir):
+    """
+    Generate PDF profiling summary for all res
+    """
+
+    pages_data = {}
+    resolutions = []
+
+    for item in os.listdir(log_dir):
+        item_path = os.path.join(log_dir, item)
+        if os.path.isdir(item_path) and item.startswith("res"):
+            # Get resolution
+            res = int(item[4:])
+            resolutions.append(res)
+
+            # Add paths
+            pages_data[res] = {
+                "function_profiling_histo": os.path.join(
+                    item_path, "profiling", "profiling_plots_histo.png"
+                ),
+                "function_profiling_pie_chart": os.path.join(
+                    item_path, "profiling", "profiling_plots_pie_chart.png"
+                ),
+                "global_profiling": os.path.join(
+                    item_path, "profiling", "memory_profiling.png"
+                ),
+            }
+
+    # ordered resolutions
+    resolutions.sort(reverse=True)
+
+    # Build pdf
+    pdf_path = os.path.join(log_dir, "profiling_summary.pdf")
+
+    with PdfPages(pdf_path) as pdf:
+        for res in resolutions:
+            # function_profiling
+            if os.path.exists(pages_data[res]["function_profiling_histo"]):
+                img = Image.open(pages_data[res]["function_profiling_histo"])
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Function Profiling Histograms - "
+                    f"Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+            if os.path.exists(pages_data[res]["function_profiling_pie_chart"]):
+                img = Image.open(
+                    pages_data[res]["function_profiling_pie_chart"]
+                )
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Function Profiling Pie Chart - Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+            # global_profiling
+            if os.path.exists(pages_data[res]["global_profiling"]):
+                img = Image.open(pages_data[res]["global_profiling"])
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Global Profiling - Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+    logging.info("PDF profiling summary generated: {}".format(pdf_path))
+
+
 def filter_lists(names, data, cond):
     """
     Filter lists with condition on name
@@ -776,7 +898,7 @@ def generate_boxplot(axis, names, data_full, title, data_type):
     axis.set_title(title)
 
 
-def generate_histo(
+def generate_histo(  # pylint: disable=too-many-positional-arguments
     axis, names, data, title, data_type, data_min_err=None, data_max_err=None
 ):
     """
@@ -815,8 +937,14 @@ def generate_pie_chart(axis, names, data, title):
         data.append(others)
         names.append("other")
 
-    axis.pie(data, labels=names, autopct="%1.1f%%")
-    axis.set_title(title)
+    axis.pie(
+        data,
+        labels=names,
+        autopct="%1.1f%%",
+        labeldistance=1.1,
+        textprops={"fontsize": 30},
+    )
+    axis.set_title(title, fontsize=40)
 
 
 def cars_profile(name=None, interval=0.1):
@@ -926,15 +1054,14 @@ class CarsMemProf(Thread):
                 # Get memory
                 current_mem = self.process.memory_info().rss
 
-                if current_mem > max_mem:
-                    max_mem = current_mem
+                max_mem = max(max_mem, current_mem)
 
                 # Get cpu max
                 current_cpu = self.process.cpu_percent(
                     interval=self.cpu_interval
                 )
-                if current_cpu > max_cpu:
-                    max_cpu = current_cpu
+
+                max_cpu = max(max_cpu, current_cpu)
 
                 if stop:
                     break

cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py

@@ -83,8 +83,13 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
     # pylint: disable=too-many-instance-attributes
     @cars_profile(name="Multiprocessing orchestrator initialization")
-    def __init__(
-        self, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):
         """
         Init function of MultiprocessingCluster
@@ -102,10 +107,12 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
             logging.warning(message)
 
         self.out_dir = out_dir
+        self.log_dir = log_dir
         # call parent init
         super().__init__(
             conf_cluster,
             out_dir,
+            log_dir,
             launch_worker=launch_worker,
             data_to_propagate=data_to_propagate,
         )
@@ -202,7 +209,7 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
             self.profiler = MultiprocessingProfiler(
                 self.pool,
-                self.out_dir,
+                self.log_dir,
                 self.checked_conf_cluster["max_ram_per_worker"],
                 mp_dataframe=mp_dataframe,
                 timer=timer,
@@ -459,6 +466,7 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
         return object_future
 
+    # pylint: disable=too-many-positional-arguments
     @staticmethod  # noqa: C901
     def refresh_task_cache(  # noqa: C901
         pool,