cars 1.0.0a2__cp310-cp310-win_amd64.whl → 1.0.0a4__cp310-cp310-win_amd64.whl

cars/orchestrator/cluster/log_wrapper.py

@@ -21,11 +21,12 @@
 """
 Contains functions for wrapper logs
 """
-# pylint: disable=too-many-lines
+# pylint: disable=C0302
 
 import copy
 import cProfile
 import datetime
+import functools
 import gc
 import io
 import logging
@@ -44,6 +45,8 @@ import numpy as np
 import pandas as pd
 import psutil
 from json_checker import Checker
+from matplotlib.backends.backend_pdf import PdfPages
+from PIL import Image
 
 from cars.core import cars_logging
 from cars.core.utils import safe_makedirs
@@ -496,11 +499,37 @@ def log_delta_memory(func, memory_start, memory_end):
     log_message(func, message)
 
 
+def exception_safe(func):
+    """
+    Decorator for consistent exception handling in profiling functions
+
+    :param func: function to wrap
+    :return: wrapped function
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        """
+        Catch error
+        """
+        try:
+            return func(*args, **kwargs)
+        except Exception as exc:
+            error_msg = (
+                f"Error in {func.__name__}: {type(exc).__name__}: {str(exc)}"
+            )
+            logging.error(error_msg)
+            cars_logging.add_profiling_message(f"ERROR - {error_msg}")
+            return None
+
+    return wrapper
+
+
+@exception_safe
 def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     """
     Generate Profiling summary
     """
-
     nb_workers = 1
     if "orchestrator" not in used_conf:
         first_key = next(iter(used_conf))
@@ -510,16 +539,15 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         if "nb_workers" in used_conf["orchestrator"]:
             nb_workers = used_conf["orchestrator"]["nb_workers"]
 
-    workers_log_dir = os.path.join(out_dir, "logs", "workers_log")
+    workers_log_dir = os.path.join(out_dir, "workers_log")
+    os.makedirs(workers_log_dir, exist_ok=True)
 
     log_file_main = os.path.join(
         workers_log_dir,
         "profiling.log",
     )
 
-    out_profiling_main = os.path.join(
-        out_dir, "logs", "profiling", "profiling.log"
-    )
+    out_profiling_main = os.path.join(out_dir, "profiling", "profiling.log")
 
     log_files = [log_file_main, out_profiling_main]
 
@@ -640,7 +668,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         cars_logging.add_profiling_message(message)
 
     # Generate png
-    _, axs = plt.subplots(4, 2, figsize=(15, 15), layout="tight")
+    _, axs = plt.subplots(3, 2, figsize=(20, 20), layout="tight")
     # Fill
 
     generate_boxplot(
@@ -689,6 +717,14 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         "calls",
     )
 
+    # file_name
+    profiling_plot = os.path.join(
+        out_dir,
+        "profiling",
+        "profiling_plots_histograms.png",
+    )
+    plt.savefig(profiling_plot)
+
     # Pie chart
 
     (name_task_workers, summary_workers) = filter_lists(
@@ -704,7 +740,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     )
 
     (_, [pipeline_time]) = filter_lists(
-        summary_names, summary_total_time, lambda name: "pipeline" in name
+        summary_names, summary_total_time, lambda name: "unit_pipeline" in name
     )
 
     (_, [multiprocessing_time]) = filter_lists(
@@ -717,8 +753,10 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
 
     total_time_workers = nb_workers * multiprocessing_time
 
+    _, axs2 = plt.subplots(2, 1, figsize=(40, 40), layout="tight")
+
     generate_pie_chart(
-        axs.flat[6],
+        axs2.flat[0],
         name_task_workers,
         100 * np.array(summary_workers) / total_time_workers,
         "Total time in parallel tasks ({} workers) : {}".format(
@@ -728,7 +766,7 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
     )
 
     generate_pie_chart(
-        axs.flat[7],
+        axs2.flat[1],
         name_task_main,
         100 * np.array(summary_main) / sequential_time,
         "Total time in sequential tasks : {}".format(
@@ -736,19 +774,103 @@ def generate_summary(out_dir, used_conf, clean_worker_logs=False):
         ),
     )
 
-    # file_name
-    profiling_plot = os.path.join(
+    profiling_plot2 = os.path.join(
         out_dir,
-        "logs",
         "profiling",
-        "profiling_plots.png",
+        "profiling_plots_pie_chart.png",
     )
-    plt.savefig(profiling_plot)
+    plt.savefig(profiling_plot2)
 
     if clean_worker_logs and os.path.exists(workers_log_dir):
         shutil.rmtree(workers_log_dir)
 
 
+def generate_pdf_profiling(log_dir):
+    """
+    Generate PDF profiling summary for all res
+    """
+
+    pages_data = {}
+    resolutions = []
+
+    for item in os.listdir(log_dir):
+        item_path = os.path.join(log_dir, item)
+        if os.path.isdir(item_path) and item.startswith("res"):
+            # Get resolution
+            res = int(item[4:])
+            resolutions.append(res)
+
+            # Add paths
+            pages_data[res] = {
+                "function_profiling_histo": os.path.join(
+                    item_path, "profiling", "profiling_plots_histo.png"
+                ),
+                "function_profiling_pie_chart": os.path.join(
+                    item_path, "profiling", "profiling_plots_pie_chart.png"
+                ),
+                "global_profiling": os.path.join(
+                    item_path, "profiling", "memory_profiling.png"
+                ),
+            }
+
+    # ordered resolutions
+    resolutions.sort(reverse=True)
+
+    # Build pdf
+    pdf_path = os.path.join(log_dir, "profiling_summary.pdf")
+
+    with PdfPages(pdf_path) as pdf:
+        for res in resolutions:
+            # function_profiling
+            if os.path.exists(pages_data[res]["function_profiling_histo"]):
+                img = Image.open(pages_data[res]["function_profiling_histo"])
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Function Profiling Histograms - "
+                    f"Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+            if os.path.exists(pages_data[res]["function_profiling_pie_chart"]):
+                img = Image.open(
+                    pages_data[res]["function_profiling_pie_chart"]
+                )
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Function Profiling Pie Chart - Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+            # global_profiling
+            if os.path.exists(pages_data[res]["global_profiling"]):
+                img = Image.open(pages_data[res]["global_profiling"])
+                fig, axis = plt.subplots(1, 1, figsize=(11.69, 8.27), dpi=300)
+                axis.imshow(img, interpolation="none")
+                axis.set_title(
+                    f"Global Profiling - Epipolar Resolution {res}",
+                    fontsize=16,
+                    fontweight="bold",
+                )
+                axis.axis("off")
+                plt.subplots_adjust(left=0, right=1, top=0.95, bottom=0)
+                pdf.savefig(fig, bbox_inches="tight", dpi=300)
+                plt.close(fig)
+
+    logging.info("PDF profiling summary generated: {}".format(pdf_path))
+
+
 def filter_lists(names, data, cond):
     """
     Filter lists with condition on name
@@ -776,7 +898,7 @@ def generate_boxplot(axis, names, data_full, title, data_type):
     axis.set_title(title)
 
 
-def generate_histo(
+def generate_histo(  # pylint: disable=too-many-positional-arguments
     axis, names, data, title, data_type, data_min_err=None, data_max_err=None
 ):
     """
@@ -815,8 +937,14 @@ def generate_pie_chart(axis, names, data, title):
         data.append(others)
         names.append("other")
 
-    axis.pie(data, labels=names, autopct="%1.1f%%")
-    axis.set_title(title)
+    axis.pie(
+        data,
+        labels=names,
+        autopct="%1.1f%%",
+        labeldistance=1.1,
+        textprops={"fontsize": 30},
+    )
+    axis.set_title(title, fontsize=40)
 
 
 def cars_profile(name=None, interval=0.1):
@@ -926,15 +1054,14 @@ class CarsMemProf(Thread):
                 # Get memory
                 current_mem = self.process.memory_info().rss
 
-                if current_mem > max_mem:
-                    max_mem = current_mem
+                max_mem = max(max_mem, current_mem)
 
                 # Get cpu max
                 current_cpu = self.process.cpu_percent(
                     interval=self.cpu_interval
                 )
-                if current_cpu > max_cpu:
-                    max_cpu = current_cpu
+
+                max_cpu = max(max_cpu, current_cpu)
 
                 if stop:
                     break

cars/orchestrator/cluster/mp_cluster/multiprocessing_cluster.py

@@ -21,7 +21,7 @@
 """
 Contains abstract function for multiprocessing Cluster
 """
-# pylint: disable=too-many-lines
+# pylint: disable=C0302
 
 import copy
 import itertools
@@ -83,8 +83,13 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
     # pylint: disable=too-many-instance-attributes
     @cars_profile(name="Multiprocessing orchestrator initialization")
-    def __init__(
-        self, conf_cluster, out_dir, launch_worker=True, data_to_propagate=None
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self,
+        conf_cluster,
+        out_dir,
+        log_dir,
+        launch_worker=True,
+        data_to_propagate=None,
     ):
         """
         Init function of MultiprocessingCluster
@@ -102,10 +107,12 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
             logging.warning(message)
 
         self.out_dir = out_dir
+        self.log_dir = log_dir
         # call parent init
         super().__init__(
             conf_cluster,
             out_dir,
+            log_dir,
             launch_worker=launch_worker,
             data_to_propagate=data_to_propagate,
         )
@@ -202,7 +209,7 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
             self.profiler = MultiprocessingProfiler(
                 self.pool,
-                self.out_dir,
+                self.log_dir,
                 self.checked_conf_cluster["max_ram_per_worker"],
                 mp_dataframe=mp_dataframe,
                 timer=timer,
@@ -459,6 +466,7 @@ class MultiprocessingCluster(abstract_cluster.AbstractCluster):
 
         return object_future
 
+    # pylint: disable=too-many-positional-arguments
     @staticmethod  # noqa: C901
     def refresh_task_cache(  # noqa: C901
         pool,

cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py

@@ -73,7 +73,7 @@ class MultiprocessingProfiler:  # pylint: disable=too-few-public-methods
     Used to profile memory in processes
     """
 
-    def __init__(
+    def __init__(  # pylint: disable=too-many-positional-arguments
         self, pool, out_dir, max_ram_per_worker, mp_dataframe=None, timer=None
     ):
         """
@@ -88,7 +88,7 @@ class MultiprocessingProfiler:  # pylint: disable=too-few-public-methods
         self.pool = pool
         self.out_dir = out_dir
         self.file_plot = os.path.join(
-            self.out_dir, "logs", "profiling", "memory_profiling.png"
+            self.out_dir, "profiling", "memory_profiling.png"
         )
         os.makedirs(os.path.dirname(self.file_plot), exist_ok=True)
         self.max_ram_per_worker = max_ram_per_worker

cars/orchestrator/cluster/pbs_dask_cluster.py

@@ -108,7 +108,7 @@ class PbsDaskCluster(abstract_dask_cluster.AbstractDaskCluster):
         logging.info("Dask cluster closed")
 
 
-def start_cluster(
+def start_cluster(  # pylint: disable=too-many-positional-arguments
     nb_workers,
     walltime,
     out_dir,

cars/orchestrator/cluster/sequential_cluster.py

@@ -37,7 +37,7 @@ class SequentialCluster(abstract_cluster.AbstractCluster):
     SequentialCluster
     """
 
-    def __init__(self, conf_cluster, out_dir, launch_worker=True):
+    def __init__(self, conf_cluster, out_dir, log_dir, launch_worker=True):
         """
         Init function of SequentialCluster
 
@@ -45,7 +45,9 @@ class SequentialCluster(abstract_cluster.AbstractCluster):
 
         """
         # call parent init
-        super().__init__(conf_cluster, out_dir, launch_worker=launch_worker)
+        super().__init__(
+            conf_cluster, out_dir, log_dir, launch_worker=launch_worker
+        )
 
         # retrieve parameters
         self.profiling = self.checked_conf_cluster["profiling"]
@@ -134,5 +136,4 @@ class SequentialCluster(abstract_cluster.AbstractCluster):
         :param future_list: future_list list
         """
 
-        for future in future_list:
-            yield future
+        yield from future_list

cars/orchestrator/cluster/slurm_dask_cluster.py

@@ -120,7 +120,7 @@ class SlurmDaskCluster(abstract_dask_cluster.AbstractDaskCluster):
         logging.info("Dask cluster closed")
 
 
-def start_cluster(
+def start_cluster(  # pylint: disable=too-many-positional-arguments
     nb_workers,
     walltime,
     out_dir,

cars/orchestrator/orchestrator.py

@@ -22,7 +22,7 @@
 this module contains the orchestrator class
 """
 
-# pylint: disable=too-many-lines
+# pylint: disable=C0302
 
 import collections
 import logging
@@ -80,10 +80,11 @@ class Orchestrator:
     # pylint: disable=too-many-instance-attributes
 
     # flake8: noqa: C901
-    def __init__(
+    def __init__(  # pylint: disable=too-many-positional-arguments
         self,
         orchestrator_conf=None,
         out_dir=None,
+        log_dir=None,
         launch_worker=True,
         out_json_path=None,
     ):
@@ -105,6 +106,11 @@ class Orchestrator:
             self.add_to_clean(self.out_dir)
             logging.debug("No out_dir defined")
 
+        if log_dir is not None:
+            self.log_dir = log_dir
+        else:
+            self.log_dir = os.path.join(self.out_dir, "logs")
+
         self.launch_worker = launch_worker
 
         # overload orchestrator_conf
@@ -137,7 +143,10 @@ class Orchestrator:
 
         # init cluster
         self.cluster = AbstractCluster(  # pylint: disable=E0110
-            orchestrator_conf, self.out_dir, launch_worker=self.launch_worker
+            orchestrator_conf,
+            self.out_dir,
+            self.log_dir,
+            launch_worker=self.launch_worker,
         )
         self.conf = self.cluster.get_conf()
 
@@ -202,7 +211,8 @@ class Orchestrator:
 
         return self.conf
 
-    def add_to_save_lists(
+    @cars_profile(name="Add to save lists", interval=0.5)
+    def add_to_save_lists(  # pylint: disable=too-many-positional-arguments
         self,
         file_name,
         tag,
@@ -546,6 +556,7 @@ class Orchestrator:
         self.cluster = AbstractCluster(  # pylint: disable=E0110
             self.orchestrator_conf,
             self.out_dir,
+            self.log_dir,
             launch_worker=self.launch_worker,
             data_to_propagate=data_to_propagate,
         )

cars/orchestrator/registry/id_generator.py

@@ -72,6 +72,7 @@ class IdGenerator:
         # Check if dataset already registered
         registered = False
         allready_registered_id = None
+        returned_id = None
         for registry in self.registries:
             (
                 is_registered,

cars/orchestrator/registry/saver_registry.py

@@ -158,7 +158,7 @@ class CarsDatasetsRegistrySaver(AbstractCarsDatasetRegistry):
             else:
                 logging.debug("Future result tile is None -> not saved")
 
-    def add_file_to_save(
+    def add_file_to_save(  # pylint: disable=too-many-positional-arguments
         self,
         file_name,
         cars_ds,
@@ -247,7 +247,7 @@ class SingleCarsDatasetSaver:
         self.count = 0
         self.folder_name = None
 
-    def add_file(
+    def add_file(  # pylint: disable=too-many-positional-arguments
         self,
         file_name,
         tag=None,

cars/pipelines/conf_resolution/conf_final_resolution.json

@@ -1,7 +1,9 @@
 {
-    "dense_matching": {
-        "confidence_filtering": {
-            "activated": false
+    "applications" : {
+        "dense_matching": {
+            "confidence_filtering": {
+                "activated": false
+            }
         }
     }
 }