cars 1.0.0rc2__cp312-cp312-win_amd64.whl

cars/orchestrator/cluster/mp_cluster/multiprocessing_profiler.py

@@ -0,0 +1,399 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains multiprocessing_profiler class
+"""
+
+import logging
+import os
+import threading
+import time
+
+import matplotlib
+import matplotlib.pyplot as plt
+import pandas as pd
+import psutil
+
+# Agg backend for non interactive
+matplotlib.use("Agg")
+
+TERMINATE = 1
+RUN = 0
+RAM_PER_WORKER_CHECK_SLEEP_TIME = 2
+INTERVAL_CPU = 0.2
+SAVE_TIME = 120
+
+TIME = "time"
+MAIN_MEMORY = "main_memory"
+MAX_PROCESS_MEMORY = "max_process_memory"
+MAIN_AND_PROCESS = "main_and_processes"
+TOTAL_PROCESS_MEMORY = "total_process_memory"
+AVAILABLE_RAM = "available_ram"
+TOTAL_RAM = "total_ram"
+
+MAIN_CPU_USAGE = "main_cpu_usage"
+TOTAL_PROCESS_CPU_USAGE = "total Proces_cpu_usage"
+MAIN_AND_PROCESS_CPU = "main_and_process_cpu"
+
+
+class Timer:  # pylint: disable=too-few-public-methods
+    """
+    Start time
+    """
+
+    def __init__(self):
+        """
+        Init
+        """
+        self.timer = time.time()
+
+
+class MultiprocessingProfiler:  # pylint: disable=too-few-public-methods
+    """
+    MultiprocessingProfiler
+
+    Used to profile memory in processes
+    """
+
+    def __init__(  # pylint: disable=too-many-positional-arguments
+        self, pool, out_dir, max_ram_per_worker, mp_dataframe=None, timer=None
+    ):
+        """
+        Init function of MultiprocessingProfiler
+
+        :param pool: pool process to monitor
+        :param out_dir: out_dir to save graph
+        :param max_ram_per_worker: max ram per worker to use
+        """
+
+        self.main_pid = os.getpid()
+        self.pool = pool
+        self.out_dir = out_dir
+        self.file_plot = os.path.join(
+            self.out_dir, "profiling", "memory_profiling.png"
+        )
+        os.makedirs(os.path.dirname(self.file_plot), exist_ok=True)
+        self.max_ram_per_worker = max_ram_per_worker
+
+        if mp_dataframe is not None and timer is not None:
+            self.memory_data = mp_dataframe
+            self.timer = timer
+        else:
+            self.timer = Timer()
+            self.memory_data = pd.DataFrame(
+                columns=[
+                    TIME,
+                    MAIN_MEMORY,
+                    MAX_PROCESS_MEMORY,
+                    MAIN_AND_PROCESS,
+                    TOTAL_PROCESS_MEMORY,
+                    AVAILABLE_RAM,
+                    TOTAL_RAM,
+                    MAIN_CPU_USAGE,
+                    TOTAL_PROCESS_CPU_USAGE,
+                    MAIN_AND_PROCESS_CPU,
+                ]
+            )
+
+        # Memory usage of Pool
+        self.monitor_thread = threading.Thread(
+            target=check_pool_memory_usage,
+            args=(
+                self.main_pid,
+                self.pool,
+                self.max_ram_per_worker,
+                self.memory_data,
+                self.timer,
+            ),
+        )
+        self.monitor_thread.daemon = True
+        self.monitor_thread._state = RUN
+        self.monitor_thread.start()
+
+        self.saver_thread = threading.Thread(
+            target=save_figure_in_thread,
+            args=(self.memory_data, self.file_plot),
+        )
+        self.saver_thread.daemon = True
+        self.saver_thread._state = RUN
+        self.saver_thread.start()
+
+    def cleanup(self):
+        """
+        Cleanup
+        """
+
+        clean_thread(self.monitor_thread)
+        clean_thread(self.saver_thread)
+
+    def save_plot(self):
+        """
+        Save plots
+        """
+        logging.info("Save profiling plots ...")
+        try:
+            save_data(self.memory_data, self.file_plot)
+        except Exception as exc:
+            logging.warning("unable to save monitoring graph : {}".format(exc))
+
+
+def clean_thread(thread):
+    """
+    Clean thread
+
+    :param thread: thread to clean
+    """
+    # Terminate worker
+    thread._state = TERMINATE  # pylint: disable=W0212
+    while thread.is_alive():
+        time.sleep(0)
+
+
+def get_process_memory(process):
+    """
+    Get process current memory
+
+    :param process
+
+    :return: memory Mb
+    """
+    return process.memory_info().rss / (1024 * 1024)
+
+
+def get_cpu_usage(process):
+    """
+    Get cpu usage
+
+    :param process: Process to monitor
+    """
+
+    try:
+        cpu_usage = process.cpu_percent(interval=0.1)
+    except Exception:
+        cpu_usage = 0
+
+    return cpu_usage
+
+
+def save_figure_in_thread(to_fill_dataframe, file_path):
+    """
+    Save data during compute
+
+    :param to_fill_dataframe: dataframe to fill
+    :param file_path: path to save path
+    """
+
+    thread = threading.current_thread()
+
+    start_time = time.time()
+    while thread._state == RUN:  # pylint: disable=protected-access
+        time.sleep(RAM_PER_WORKER_CHECK_SLEEP_TIME)
+        if time.time() - start_time > SAVE_TIME:
+            start_time = time.time()
+            # Save file
+            try:
+                save_data(to_fill_dataframe, file_path)
+            except Exception as exc:
+                logging.warning(
+                    "unable to save monitoring graph : {}".format(exc)
+                )
+
+
+def check_pool_memory_usage(
+    main_process_id, pool, max_ram_per_worker, to_fill_dataframe, timer
+):
+    """
+    Check memory usage of each worker in pool
+
+    :param main_process_id: main process id
+    :param pool: pool of worker
+    :param max_ram_per_worker: max ram to use per worker
+    :param to_fill_dataframe: dataframe to fill
+    :param timer: timer
+    """
+
+    thread = threading.current_thread()
+
+    main_process = psutil.Process(main_process_id)
+    start_time = timer.timer
+
+    while thread._state == RUN:  # pylint: disable=protected-access
+        # Get time
+        current_time = time.time() - start_time
+        minutes = current_time / 60
+
+        # Get available memory
+        ram = psutil.virtual_memory()
+        total_ram = ram.total / (1024 * 1024)
+        available = ram.available / (1024 * 1024)
+
+        # Check main process
+        main_current_memory = get_process_memory(main_process)
+        main_process_cpu = get_cpu_usage(main_process)
+
+        # Check workers
+        main_and_processes_total = main_current_memory
+        total_memory = 0
+        max_process_ram = 0
+        processes_cpu = 0
+        total_cpu = main_process_cpu
+
+        size_pool = len(pool._pool)  # pylint: disable=protected-access
+        sleep_time = max(
+            0, RAM_PER_WORKER_CHECK_SLEEP_TIME - INTERVAL_CPU * size_pool
+        )
+        for worker in pool._pool:  # pylint: disable=protected-access
+            pid = worker.pid
+            try:
+                process = psutil.Process(pid)
+                memory_usage_mb = get_process_memory(process)
+
+                # Add to metrics
+                max_process_ram = max(max_process_ram, memory_usage_mb)
+                total_memory += memory_usage_mb
+                processes_cpu += get_cpu_usage(process)
+
+                # Check memory to inform user
+                if memory_usage_mb > max_ram_per_worker:
+                    logging.info(
+                        "Process {} is using {} Mb > "
+                        "max_ram_per_worker = {} Mb".format(
+                            pid, memory_usage_mb, max_ram_per_worker
+                        )
+                    )
+            except psutil.NoSuchProcess:
+                # Process no longer exists
+                pass
+
+        main_and_processes_total += total_memory
+        available_ram_mb = main_and_processes_total + available
+
+        total_cpu += processes_cpu
+
+        # Add to dataframe
+        to_fill_dataframe.loc[len(to_fill_dataframe)] = [
+            minutes,
+            main_current_memory,
+            max_process_ram,
+            main_and_processes_total,
+            total_memory,
+            available_ram_mb,
+            total_ram,
+            main_process_cpu,
+            processes_cpu,
+            total_cpu,
+        ]
+
+        time.sleep(sleep_time)
+
+
+def save_data(dataframe, file_path):
+    """
+    Save dataframe to disk
+
+    :param dataframe: file
+    :param file_path:
+    """
+
+    fig, axs = plt.subplots(5, 1, figsize=(10, 25))
+
+    axs[0].set_title("Total memory used by CARS  Mb")
+    axs[0].set_xlabel("Time (min)")
+    axs[0].set_ylabel("Memory (MB)")
+    dataframe.plot(
+        x=TIME,
+        y=MAIN_AND_PROCESS,
+        ax=axs[0],
+        label="Main + Processes memory",
+        color="blue",
+    )
+    dataframe.plot(
+        x=TIME, y=TOTAL_RAM, ax=axs[0], label="Machine max memory", color="red"
+    )
+    dataframe.plot(
+        x=TIME,
+        y=AVAILABLE_RAM,
+        ax=axs[0],
+        label=" total CARS + still Available memory",
+        color="green",
+    )
+
+    axs[1].set_title("Main CARS Process Memory Mb")
+    axs[1].set_xlabel("Time (min)")
+    axs[1].set_ylabel("Memory (MB)")
+    dataframe.plot(
+        x=TIME,
+        y=MAIN_MEMORY,
+        ax=axs[1],
+        label="CARS main process",
+        color="blue",
+    )
+    dataframe.plot(
+        x=TIME,
+        y=MAX_PROCESS_MEMORY,
+        ax=axs[1],
+        label="CARS max of workers ",
+        color="red",
+    )
+
+    axs[2].set_title("CARS workers  Process Memory Mb")
+    axs[2].set_xlabel("Time (min)")
+    axs[2].set_ylabel("Memory (MB)")
+    dataframe.plot(
+        x=TIME,
+        y=TOTAL_PROCESS_MEMORY,
+        ax=axs[2],
+        label="Total Process Memory",
+        color="blue",
+    )
+
+    axs[3].set_title("CARS CPU Usage")
+    axs[3].set_xlabel("Time (min)")
+    axs[3].set_ylabel("CPU (%")
+    dataframe.plot(
+        x=TIME,
+        y=TOTAL_PROCESS_CPU_USAGE,
+        ax=axs[3],
+        label="Total Process CPU ",
+        color="blue",
+    )
+    dataframe.plot(
+        x=TIME,
+        y=MAIN_AND_PROCESS_CPU,
+        ax=axs[3],
+        label="MAIN + Total Process CPU",
+        color="red",
+    )
+
+    axs[4].set_title("CARS CPU Usage of main process")
+    axs[4].set_xlabel("Time (min)")
+    axs[4].set_ylabel("CPU (%")
+    dataframe.plot(
+        x=TIME,
+        y=MAIN_CPU_USAGE,
+        ax=axs[4],
+        label="Main Process CPU ",
+        color="blue",
+    )
+
+    plt.savefig(file_path, format="png")
+
+    plt.close(fig)

cars/orchestrator/cluster/pbs_dask_cluster.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Contains abstract function for PBS dask Cluster
+"""
+
+# Standard imports
+import logging
+import os
+import warnings
+
+# Third party imports
+from dask.distributed import Client
+
+from cars.orchestrator.cluster.dask_cluster_tools import (
+    check_configuration,
+    create_checker_schema,
+)
+from cars.orchestrator.cluster.dask_jobqueue_utils import (
+    get_dashboard_link,
+    init_cluster_variables,
+    stop_cluster,
+)
+
+with warnings.catch_warnings():
+    # Ignore some internal dask_jobqueue warnings
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message=".*format_bytes is deprecated.*",
+    )
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message=".*parse_bytes is deprecated.*",
+    )
+    warnings.filterwarnings(
+        "ignore",
+        category=FutureWarning,
+        message=".*tmpfile is deprecated.*",
+    )
+    from dask_jobqueue import PBSCluster
+
+# CARS imports
+from cars.orchestrator.cluster import (  # pylint: disable=C0412
+    abstract_cluster,
+    abstract_dask_cluster,
+)
+
+
+@abstract_cluster.AbstractCluster.register_subclass("pbs_dask")
+class PbsDaskCluster(abstract_dask_cluster.AbstractDaskCluster):
+    """
+    PbsDaskCluster
+    """
+
+    def check_conf(self, conf):
+        """
+        Check configuration
+
+        :param conf: configuration to check
+        :type conf: dict
+
+        :return: overloaded configuration
+        :rtype: dict
+
+        """
+
+        return check_configuration(*create_checker_schema(conf))
+
+    def start_dask_cluster(self):
+        """
+        Start dask cluster
+        """
+
+        return start_cluster(
+            self.nb_workers,
+            self.walltime,
+            self.out_dir,
+            activate_dashboard=self.activate_dashboard,
+            python=self.python,
+        )
+
+    def cleanup(self):
+        """
+        Cleanup cluster
+
+        """
+        stop_cluster(self.cluster, self.client)
+        logging.info("Dask cluster closed")
+
+
+def start_cluster(  # pylint: disable=too-many-positional-arguments
+    nb_workers,
+    walltime,
+    out_dir,
+    timeout=600,
+    activate_dashboard=False,
+    python=None,
+):
+    """Create a Dask cluster.
+
+    Each worker will be spawned in an independent job with a single CPU
+    allocated to it, and will use a single process. This is done to maximize
+    CPU utilization and minimize scheduling delay.
+
+    The CARS_PBS_QUEUE environment variable, if defined, is used to specify the
+    queue in which worker jobs are scheduled.
+
+    :param nb_workers: Number of dask workers
+    :type nb_workers: int
+    :param walltime: Walltime for each dask worker
+    :type walltime: string
+    :param out_dir: Output directory
+    :type out_dir: string
+    :return: Dask cluster and dask client
+    :rtype: (dask_jobqueue.PBSCluster, dask.distributed.Client) tuple
+    """
+
+    (
+        python,
+        nb_workers_per_job,
+        memory,
+        nb_cpus,
+        stagger,
+        lifetime_with_margin,
+        scheduler_options,
+        envs,
+        log_directory,
+        local_directory,
+    ) = init_cluster_variables(
+        nb_workers,
+        walltime,
+        out_dir,
+        activate_dashboard,
+        python,
+        core_memory=5000,
+        cluster_name="PBS",
+    )
+
+    # Retrieve PBS queue
+    pbs_queue = os.environ.get("CARS_PBS_QUEUE")
+
+    with warnings.catch_warnings():
+        # Ignore some internal dask_jobqueue warnings
+        # TODO remove when Future warning do not exist anymore
+        warnings.filterwarnings(
+            "ignore",
+            category=FutureWarning,
+            message=".*extra has been renamed to worker_extra_args*",
+        )
+        warnings.filterwarnings(
+            "ignore",
+            category=FutureWarning,
+            message=".*job_extra has been renamed to job_extra_directives*",
+        )
+        warnings.filterwarnings(
+            "ignore",
+            category=FutureWarning,
+            message=".*env_extra has been renamed to job_script_prologue*",
+        )
+        cluster = PBSCluster(
+            processes=nb_workers_per_job,
+            cores=nb_workers_per_job,
+            memory="{}MiB".format(memory),
+            local_directory=local_directory,
+            account="dask-test",
+            walltime=walltime,
+            interface="ib0",
+            queue=pbs_queue,
+            job_script_prologue=envs,
+            log_directory=log_directory,
+            python=python,
+            worker_extra_args=[
+                "--lifetime",
+                f"{int(lifetime_with_margin.total_seconds())}s",
+                "--lifetime-stagger",
+                f"{int(stagger.total_seconds())}s",
+            ],
+            scheduler_options=scheduler_options,
+            resource_spec="select=1:ncpus={}:mem={}MB".format(nb_cpus, memory),
+        )
+        logging.info("Dask cluster started")
+        cluster.adapt(minimum=nb_workers, maximum=nb_workers)
+        client = Client(cluster, timeout=timeout)
+        logging.info(
+            "Dashboard started at {}".format(get_dashboard_link(cluster))
+        )
+    return cluster, client