triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/plots/detailed_plot.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from collections import defaultdict
+
+import matplotlib.pyplot as plt
+from matplotlib import patches as mpatches
+
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.record.metrics_manager import MetricsManager
+
+logging.getLogger("matplotlib").setLevel(logging.ERROR)
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class DetailedPlot:
+    """
+    A wrapper class around a matplotlib
+    plot that adapts with the kinds of
+    plots the model analyzer wants to generates
+
+    Detailed plots detail th
+    """
+
+    detailed_metrics = [
+        "perf_server_queue",
+        "perf_server_compute_input",
+        "perf_server_compute_infer",
+        "perf_server_compute_output",
+    ]
+
+    def __init__(self, name, title, bar_width=0.5):
+        """
+        Parameters
+        ----------
+        name: str
+            The name of the file that the plot
+            will be saved as
+        title : str
+            The title of this plot/figure
+        bar_width: float
+            width of the latency breakdown bars
+        """
+
+        self._name = name
+        self._title = title
+
+        self._fig, self._ax_latency = plt.subplots()
+        self._ax_latency.set_title(title)
+        self._ax_throughput = self._ax_latency.twinx()
+
+        latency_axis_label, throughput_axis_label = [
+            metric.header(aggregation_tag="")
+            for metric in MetricsManager.get_metric_types(
+                ["perf_latency_avg", "perf_throughput"]
+            )
+        ]
+
+        # Okabe-Ito guideline colors for colorblind
+        # https://jfly.uni-koeln.de/color/#select
+        self._bar_colors = {
+            "perf_server_queue": "#e69f00",
+            "perf_server_compute_input": "#56b4e9",
+            "perf_server_compute_infer": "#009e73",
+            "perf_server_compute_output": "#f0e442",
+            "perf_throughput": "#000000",
+        }
+
+        self._bar_width = bar_width
+        self._legend_x = 0.92
+        self._legend_y = 1.15
+        self._legend_font_size = 10
+        self._fig.set_figheight(8)
+        self._fig.set_figwidth(12)
+
+        self._ax_latency.set_xlabel("Concurrent Client Requests")
+        self._ax_latency.set_ylabel(latency_axis_label)
+        self._ax_throughput.set_ylabel(throughput_axis_label)
+
+        self._data = defaultdict(list)
+
+    def data(self):
+        """
+        Get the data in this plot
+
+        Returns
+        -------
+        dict
+            keys are line labels
+            and values are lists of floats
+        """
+
+        return self._data
+
+    def add_run_config_measurement(self, run_config_measurement):
+        """
+        Adds a measurement to this plot
+
+        Parameters
+        ----------
+        measurement : Measurement
+            The measurement containing the data to
+            be plotted.
+        """
+
+        # TODO-TMA-568: This needs to be updated because there will be multiple model configs
+        if (
+            "concurrency-range" in run_config_measurement.model_specific_pa_params()[0]
+            and run_config_measurement.model_specific_pa_params()[0][
+                "concurrency-range"
+            ]
+        ):
+            self._data["concurrency"].append(
+                run_config_measurement.model_specific_pa_params()[0][
+                    "concurrency-range"
+                ]
+            )
+
+        if (
+            "request-rate-range" in run_config_measurement.model_specific_pa_params()[0]
+            and run_config_measurement.model_specific_pa_params()[0][
+                "request-rate-range"
+            ]
+        ):
+            self._data["request_rate"].append(
+                run_config_measurement.model_specific_pa_params()[0][
+                    "request-rate-range"
+                ]
+            )
+
+        self._data["perf_throughput"].append(
+            run_config_measurement.get_non_gpu_metric_value(tag="perf_throughput")
+        )
+
+        for metric in self.detailed_metrics:
+            if MetricsManager.is_gpu_metric(tag=metric):
+                self._data[metric].append(
+                    run_config_measurement.get_gpu_metric_value(tag=metric)
+                )
+            else:
+                self._data[metric].append(
+                    run_config_measurement.get_non_gpu_metric_value(tag=metric)
+                )
+
+    def plot_data(self):
+        """
+        Calls plotting function
+        on this plot's Axes object
+        """
+
+        # Need to change the default x-axis plot title for request rates
+        if "request_rate" in self._data and self._data["request_rate"][0]:
+            self._ax_latency.set_xlabel("Client Request Rate")
+
+        # Sort the data by request rate or concurrency
+        if "request_rate" in self._data and self._data["request_rate"][0]:
+            sort_indices = list(
+                zip(*sorted(enumerate(self._data["request_rate"]), key=lambda x: x[1]))
+            )[0]
+        else:
+            sort_indices = list(
+                zip(*sorted(enumerate(self._data["concurrency"]), key=lambda x: x[1]))
+            )[0]
+
+        sorted_data = {
+            key: [data_list[i] for i in sort_indices]
+            for key, data_list in self._data.items()
+        }
+
+        # Plot latency breakdown bars
+        labels = dict(
+            zip(
+                self.detailed_metrics,
+                [
+                    metric.header()
+                    for metric in MetricsManager.get_metric_types(
+                        tags=self.detailed_metrics
+                    )
+                ],
+            )
+        )
+        bottoms = None
+
+        if "request_rate" in self._data:
+            sorted_data["indices"] = list(map(str, sorted_data["request_rate"]))
+        else:
+            sorted_data["indices"] = list(map(str, sorted_data["concurrency"]))
+
+        # Plot latency breakdown with concurrency casted as string to make uniform x
+        for metric, label in labels.items():
+            self._ax_latency.bar(
+                sorted_data["indices"],
+                sorted_data[metric],
+                width=self._bar_width,
+                label=label,
+                bottom=bottoms,
+                color=self._bar_colors[metric],
+            )
+            if not bottoms:
+                bottoms = sorted_data[metric]
+            else:
+                bottoms = list(map(lambda x, y: x + y, bottoms, sorted_data[metric]))
+
+        # Plot the inference line
+        inference_line = self._ax_throughput.plot(
+            sorted_data["indices"],
+            sorted_data["perf_throughput"],
+            label="Inferences/second",
+            marker="o",
+            color=self._bar_colors["perf_throughput"],
+        )
+
+        # Create legend handles
+        handles = [
+            mpatches.Patch(color=self._bar_colors[m], label=labels[m])
+            for m in self._bar_colors
+            if m != "perf_throughput"
+        ]
+        handles.append(inference_line[0])
+
+        self._ax_latency.legend(
+            handles=handles,
+            ncol=(len(self._bar_colors) // 2) + 1,
+            bbox_to_anchor=(self._legend_x, self._legend_y),
+            prop=dict(size=self._legend_font_size),
+        )
+        # Annotate inferences
+        for x, y in zip(sorted_data["indices"], sorted_data["perf_throughput"]):
+            self._ax_throughput.annotate(
+                str(round(y, 2)),
+                xy=(x, y),
+                textcoords="offset points",  # how to position the text
+                xytext=(0, 10),  # distance from text to points (x,y)
+                ha="center",
+            )
+
+        self._ax_latency.grid()
+        self._ax_latency.set_axisbelow(True)
+
+    def save(self, filepath):
+        """
+        Saves a .png of the plot to disk
+
+        Parameters
+        ----------
+        filepath : the path to the directory
+            this plot should be saved to
+        """
+
+        self._fig.savefig(os.path.join(filepath, self._name))

model_analyzer/plots/plot_manager.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from collections import defaultdict
+from typing import DefaultDict, Dict, Union
+
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.config_command_report import ConfigCommandReport
+from model_analyzer.config.input.config_defaults import DEFAULT_CPU_MEM_PLOT
+from model_analyzer.config.input.objects.config_plot import ConfigPlot
+from model_analyzer.constants import GLOBAL_CONSTRAINTS_KEY, TOP_MODELS_REPORT_KEY
+from model_analyzer.result.constraint_manager import ConstraintManager
+from model_analyzer.result.result_manager import ResultManager
+
+from .detailed_plot import DetailedPlot
+from .simple_plot import SimplePlot
+
+
+class PlotManager:
+    """
+    This class manages the construction and arrangement
+    of plots generated by model analyzer
+    """
+
+    def __init__(
+        self,
+        config: Union[ConfigCommandProfile, ConfigCommandReport],
+        result_manager: ResultManager,
+        constraint_manager: ConstraintManager,
+    ):
+        """
+        Parameters
+        ----------
+        config : ConfigCommandProfile or ConfigCommandReport
+            The model analyzer's config containing information
+            about the kind of plots to generate
+        result_manager : ResultManager
+            instance that manages the result tables and
+            adding results
+        constraint_manager: ConstraintManager
+            instance that manages constraints
+        """
+
+        self._config = config
+        self._result_manager = result_manager
+
+        # Constraints should be plotted as well
+        self._constraints = constraint_manager.get_constraints_for_all_models()
+
+        # Construct plot output directory
+        self._plot_export_directory = os.path.join(config.export_path, "plots")
+        os.makedirs(self._plot_export_directory, exist_ok=True)
+
+        # Dict of list of plots
+        self._simple_plots: DefaultDict[str, Dict[str, SimplePlot]] = defaultdict()
+        self._detailed_plots: Dict[str, DetailedPlot] = {}
+
+    def create_summary_plots(self):
+        """
+        Constructs simple plots based on config specs
+        """
+
+        model_names = self._result_manager._profile_model_names
+
+        for plots_key in model_names:
+            self._create_summary_plot_for_model(
+                plots_key=plots_key,
+                model_name=plots_key,
+                num_results=self._config.num_configs_per_model,
+            )
+
+        if self._config.num_top_model_configs:
+            self._create_summary_plot_for_model(
+                plots_key=TOP_MODELS_REPORT_KEY,
+                model_name=None,
+                num_results=self._config.num_top_model_configs,
+            )
+
+    def _create_summary_plot_for_model(self, model_name, plots_key, num_results):
+        """
+        helper function that creates the summary plots
+        for a given model
+        """
+
+        for plot_config in self._config.plots:
+            constraints = self._constraints[GLOBAL_CONSTRAINTS_KEY]
+            if plots_key in self._constraints:
+                constraints = self._constraints[plots_key]
+            for run_config_result in self._result_manager.top_n_results(
+                model_name=model_name, n=num_results, include_default=True
+            ):
+                if run_config_result.run_config().cpu_only():
+                    if plot_config.y_axis() == "gpu_used_memory":
+                        plot_name, plot_config_dict = list(
+                            DEFAULT_CPU_MEM_PLOT.items()
+                        )[0]
+                        plot_config = ConfigPlot(plot_name, **plot_config_dict)
+                self._create_update_simple_plot(
+                    plots_key=plots_key,
+                    plot_config=plot_config,
+                    run_config_measurements=run_config_result.run_config_measurements(),
+                    constraints=constraints,
+                )
+
+    def _create_update_simple_plot(
+        self, plots_key, plot_config, run_config_measurements, constraints
+    ):
+        """
+        Creates or updates a single simple plot, given a config name,
+        some measurements, and a key to put the plot into the simple plots
+        """
+
+        if plots_key not in self._simple_plots:
+            self._simple_plots[plots_key] = {}
+        if plot_config.name() not in self._simple_plots[plots_key]:
+            self._simple_plots[plots_key][plot_config.name()] = SimplePlot(
+                name=plot_config.name(),
+                title=plot_config.title(),
+                x_axis=plot_config.x_axis(),
+                y_axis=plot_config.y_axis(),
+                monotonic=plot_config.monotonic(),
+            )
+
+        for run_config_measurement in run_config_measurements:
+            self._simple_plots[plots_key][
+                plot_config.name()
+            ].add_run_config_measurement(
+                label=run_config_measurement.model_variants_name(),
+                run_config_measurement=run_config_measurement,
+            )
+
+        # In case this plot already had lines, we want to clear and replot
+        self._simple_plots[plots_key][plot_config.name()].clear()
+        self._simple_plots[plots_key][plot_config.name()].plot_data_and_constraints(
+            constraints=constraints
+        )
+
+    def create_detailed_plots(self):
+        """
+        Constructs detailed plots based on
+        requested config specs
+        """
+
+        # Create detailed plots
+        for model in self._config.report_model_configs:
+            model_config_name = model.model_config_name()
+            self._detailed_plots[model_config_name] = DetailedPlot(
+                f"latency_breakdown", "Online Performance"
+            )
+            (
+                model_config,
+                run_config_measurements,
+            ) = self._result_manager.get_model_configs_run_config_measurements(
+                model_config_name
+            )
+
+            # If model_config_name was present in results
+            if run_config_measurements:
+                for run_config_measurement in run_config_measurements:
+                    self._detailed_plots[model_config_name].add_run_config_measurement(
+                        run_config_measurement
+                    )
+                self._detailed_plots[model_config_name].plot_data()
+
+            # Create the simple plots for the detailed reports
+            for plot_config in model.plots():
+                if model_config.cpu_only() and (
+                    plot_config.y_axis().startswith("gpu_")
+                    or plot_config.x_axis().startswith("gpu_")
+                ):
+                    continue
+                self._create_update_simple_plot(
+                    plots_key=model_config_name,
+                    plot_config=plot_config,
+                    run_config_measurements=run_config_measurements,
+                    constraints=None,
+                )
+
+    def export_summary_plots(self):
+        """
+        write the plots to disk
+        """
+
+        simple_plot_dir = os.path.join(self._plot_export_directory, "simple")
+        for plots_key, plot_dicts in self._simple_plots.items():
+            model_plot_dir = os.path.join(simple_plot_dir, plots_key)
+            os.makedirs(model_plot_dir, exist_ok=True)
+            for plot in plot_dicts.values():
+                plot.save(model_plot_dir)
+
+    def export_detailed_plots(self):
+        """
+        Write detailed plots to disk
+        """
+
+        detailed_plot_dir = os.path.join(self._plot_export_directory, "detailed")
+        simple_plot_dir = os.path.join(self._plot_export_directory, "simple")
+        for model_config_name, detailed_plot in self._detailed_plots.items():
+            detailed_model_config_plot_dir = os.path.join(
+                detailed_plot_dir, model_config_name
+            )
+            os.makedirs(detailed_model_config_plot_dir, exist_ok=True)
+            detailed_plot.save(detailed_model_config_plot_dir)
+
+            simple_model_config_plot_dir = os.path.join(
+                simple_plot_dir, model_config_name
+            )
+            os.makedirs(simple_model_config_plot_dir, exist_ok=True)
+            for simple_plot in self._simple_plots[model_config_name].values():
+                simple_plot.save(simple_model_config_plot_dir)

model_analyzer/plots/simple_plot.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from collections import defaultdict
+
+import matplotlib.pyplot as plt
+
+from model_analyzer.perf_analyzer.perf_config import PerfAnalyzerConfig
+from model_analyzer.record.metrics_manager import MetricsManager
+from model_analyzer.reports.report_utils import truncate_model_config_name
+
+
+class SimplePlot:
+    """
+    A wrapper class around a matplotlib
+    plot that adapts with the kinds of
+    plots the model analyzer wants to generates
+
+    A singe plot holds data for multiple
+    model configs, but only holds one
+    type of plot
+    """
+
+    def __init__(self, name, title, x_axis, y_axis, monotonic=False):
+        """
+        Parameters
+        ----------
+        name: str
+            The name of the file that the plot
+            will be saved as
+        title : str
+            The title of this plot/figure
+        x_axis : str
+            The metric tag for the x-axis of this plot
+        y_axis : str
+            The metric tag for the y-axis of this plot
+        monotonic: bool
+            Whether or not to prune decreasing points in this
+            plot
+        """
+
+        self._name = name
+        self._title = title
+        self._x_axis = x_axis
+        self._y_axis = y_axis
+        self._monotonic = monotonic
+
+        self._fig, self._ax = plt.subplots()
+
+        self._data = {}
+
+    def add_run_config_measurement(self, label, run_config_measurement):
+        """
+        Adds a measurement to this plot
+
+        Parameters
+        ----------
+        label : str
+            The name of the config(s) this measurement
+            is taken from.
+        run_config_measurement : RunConfigMeasurement
+            The measurement containing the data to
+            be plotted.
+        """
+
+        if label not in self._data:
+            self._data[label] = defaultdict(list)
+
+        if self._x_axis.replace("_", "-") in PerfAnalyzerConfig.allowed_keys():
+            self._data[label]["x_data"].append(
+                run_config_measurement.model_specific_pa_params()[0][
+                    self._x_axis.replace("_", "-")
+                ]
+            )
+        else:
+            if MetricsManager.is_gpu_metric(tag=self._x_axis):
+                self._data[label]["x_data"].append(
+                    run_config_measurement.get_gpu_metric_value(tag=self._x_axis)
+                )
+            else:
+                self._data[label]["x_data"].append(
+                    run_config_measurement.get_non_gpu_metric_value(tag=self._x_axis)
+                )
+
+        if self._y_axis.replace("_", "-") in PerfAnalyzerConfig.allowed_keys():
+            self._data[label]["y_data"].append(
+                run_config_measurement.model_specific_pa_params()[0][
+                    self._y_axis.replace("_", "-")
+                ]
+            )
+        else:
+            if MetricsManager.is_gpu_metric(tag=self._y_axis):
+                self._data[label]["y_data"].append(
+                    run_config_measurement.get_gpu_metric_value(tag=self._y_axis)
+                )
+            else:
+                self._data[label]["y_data"].append(
+                    run_config_measurement.get_non_gpu_metric_value(tag=self._y_axis)
+                )
+
+    def clear(self):
+        """
+        Clear the contents of the current Axes object
+        """
+
+        self._ax.clear()
+
+    def plot_data_and_constraints(self, constraints):
+        """
+        Calls plotting function
+        on this plot's Axes object
+
+        Parameters
+        ----------
+        constraints: ModelConstraints object
+            The keys are metric tags and values are dicts whose
+            keys are constraint types (min, max) and values are their
+            values
+        """
+
+        self._ax.set_title(self._title)
+
+        if self._x_axis.replace("_", "-") in PerfAnalyzerConfig.allowed_keys():
+            self._x_header = self._x_axis.replace("_", " ").title()
+        else:
+            self._x_header = MetricsManager.get_metric_types([self._x_axis])[0].header(
+                aggregation_tag=""
+            )
+
+        if self._y_axis.replace("_", "-") in PerfAnalyzerConfig.allowed_keys():
+            self._y_header = self._y_axis.replace("_", " ").title()
+        else:
+            self._y_header = MetricsManager.get_metric_types([self._y_axis])[0].header(
+                aggregation_tag=""
+            )
+
+        self._ax.set_xlabel(self._x_header)
+        self._ax.set_ylabel(self._y_header)
+
+        for model_config_name, data in self._data.items():
+            # Sort the data by x-axis
+            x_data, y_data = (
+                list(t) for t in zip(*sorted(zip(data["x_data"], data["y_data"])))
+            )
+
+            model_config_name = truncate_model_config_name(model_config_name)
+
+            if self._monotonic:
+                filtered_x, filtered_y = [x_data[0]], [y_data[0]]
+                for i in range(1, len(x_data)):
+                    if y_data[i] > filtered_y[-1]:
+                        filtered_x.append(x_data[i])
+                        filtered_y.append(y_data[i])
+                x_data, y_data = filtered_x, filtered_y
+
+            self._ax.plot(x_data, y_data, marker="o", label=model_config_name)
+
+        # Plot constraints
+        if constraints:
+            if constraints.has_metric(self._x_axis):
+                for _, constraint_val in constraints[self._x_axis].items():
+                    constraint_label = f"Target {self._x_header.rsplit(' ',1)[0]}"
+                    self._ax.axvline(
+                        x=constraint_val, linestyle="--", label=constraint_label
+                    )
+            if constraints.has_metric(self._y_axis):
+                for _, constraint_val in constraints[self._y_axis].items():
+                    constraint_label = f"Target {self._y_header.rsplit(' ', 1)[0]}"
+                    self._ax.axhline(
+                        y=constraint_val, linestyle="--", label=constraint_label
+                    )
+            # plot h lines
+        self._ax.legend()
+        self._ax.grid()
+
+    def data(self):
+        """
+        Get the data in this plot
+
+        Returns
+        -------
+        dict
+            keys are line labels
+            and values are lists of floats
+        """
+
+        return self._data
+
+    def save(self, filepath):
+        """
+        Saves a .png of the plot to disk
+
+        Parameters
+        ----------
+        filepath : the path to the directory
+            this plot should be saved to
+        """
+
+        self._fig.savefig(os.path.join(filepath, self._name))