triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/config/input/config_command_report.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from model_analyzer.config.input.config_utils import (
+    file_path_validator,
+    parent_path_validator,
+)
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+from .config_command import ConfigCommand
+from .config_defaults import (
+    DEFAULT_ALWAYS_REPORT_GPU_METRICS,
+    DEFAULT_CHECKPOINT_DIRECTORY,
+    DEFAULT_EXPORT_PATH,
+    DEFAULT_OFFLINE_REPORT_PLOTS,
+    DEFAULT_ONLINE_REPORT_PLOTS,
+    DEFAULT_REPORT_FORMAT,
+)
+from .config_enum import ConfigEnum
+from .config_field import ConfigField
+from .config_list_generic import ConfigListGeneric
+from .config_list_string import ConfigListString
+from .config_object import ConfigObject
+from .config_primitive import ConfigPrimitive
+from .config_union import ConfigUnion
+from .objects.config_model_report_spec import ConfigModelReportSpec
+from .objects.config_plot import ConfigPlot
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class ConfigCommandReport(ConfigCommand):
+    """
+    Model Analyzer config object.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._fill_config()
+
+    def _fill_config(self):
+        """
+        Builder function makes calls to add config to
+        fill the config with options
+        """
+
+        self._add_config(
+            ConfigField(
+                "config_file",
+                flags=["-f", "--config-file"],
+                field_type=ConfigPrimitive(str),
+                description="Path to Config File for subcommand 'report'.",
+            )
+        )
+        self._add_config(
+            ConfigField(
+                "checkpoint_directory",
+                flags=["-s", "--checkpoint-directory"],
+                default_value=DEFAULT_CHECKPOINT_DIRECTORY,
+                field_type=ConfigPrimitive(str, validator=parent_path_validator),
+                description="Full path to directory to which to read and write checkpoints and profile data.",
+            )
+        )
+        self._add_config(
+            ConfigField(
+                "export_path",
+                flags=["-e", "--export-path"],
+                default_value=DEFAULT_EXPORT_PATH,
+                field_type=ConfigPrimitive(str, validator=file_path_validator),
+                description="Full path to directory in which to store the results",
+            )
+        )
+        plots_scheme = ConfigObject(
+            schema={
+                "*": ConfigObject(
+                    schema={
+                        "title": ConfigPrimitive(type_=str),
+                        "x_axis": ConfigPrimitive(type_=str),
+                        "y_axis": ConfigPrimitive(type_=str),
+                        "monotonic": ConfigPrimitive(type_=bool),
+                    }
+                )
+            },
+            output_mapper=ConfigPlot.from_object,
+        )
+        self._add_config(
+            ConfigField(
+                "plots",
+                field_type=ConfigUnion(
+                    [
+                        plots_scheme,
+                        ConfigListGeneric(
+                            type_=plots_scheme, output_mapper=ConfigPlot.from_list
+                        ),
+                    ]
+                ),
+                default_value=DEFAULT_ONLINE_REPORT_PLOTS,
+                description="Model analyzer uses the information in this section to construct plots of the results.",
+            )
+        )
+
+        report_model_scheme = ConfigObject(
+            required=True,
+            schema={
+                # Any key is allowed, but the keys must follow the pattern
+                # below
+                "*": ConfigObject(schema={"plots": plots_scheme})
+            },
+            output_mapper=ConfigModelReportSpec.model_object_to_config_model_report_spec,
+        )
+        self._add_config(
+            ConfigField(
+                "report_model_configs",
+                flags=["-n", "--report-model-configs"],
+                field_type=ConfigUnion(
+                    [
+                        report_model_scheme,
+                        ConfigListGeneric(
+                            ConfigUnion(
+                                [
+                                    report_model_scheme,
+                                    ConfigPrimitive(
+                                        str,
+                                        output_mapper=ConfigModelReportSpec.model_str_to_config_model_report_spec,
+                                    ),
+                                ]
+                            ),
+                            required=True,
+                            output_mapper=ConfigModelReportSpec.model_mixed_to_config_model_report_spec,
+                        ),
+                        ConfigListString(
+                            output_mapper=ConfigModelReportSpec.model_list_to_config_model_report_spec
+                        ),
+                    ],
+                    required=True,
+                ),
+                description=(
+                    "Comma delimited list of the names of model configs"
+                    " for which to generate detailed reports."
+                ),
+            )
+        )
+        self._add_config(
+            ConfigField(
+                "output_formats",
+                flags=["-o", "--output-formats"],
+                default_value=DEFAULT_REPORT_FORMAT,
+                field_type=ConfigUnion(
+                    [
+                        ConfigListGeneric(
+                            type_=ConfigEnum(choices=["pdf", "csv", "png"])
+                        ),
+                        ConfigListString(),
+                    ]
+                ),
+                description="Output file format for detailed report.",
+            )
+        )
+        self._add_config(
+            ConfigField(
+                "always_report_gpu_metrics",
+                flags=["--always_report-gpu-metrics"],
+                field_type=ConfigPrimitive(bool),
+                parser_args={"action": "store_true"},
+                default_value=DEFAULT_ALWAYS_REPORT_GPU_METRICS,
+                description="Report GPU metrics, even when the model is `cpu_only`.",
+            )
+        )
+
+    def set_config_values(self, args):
+        """
+        Set the config values. This function sets all the values for the
+        config. CLI arguments have the highest priority, then YAML config
+        values and then default values.
+
+        Parameters
+        ----------
+        args : argparse.Namespace
+            Parsed arguments from the CLI
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If the required fields are not specified, it will raise
+            this exception
+        """
+
+        if args.mode == "online":
+            self._fields["plots"].set_default_value(DEFAULT_ONLINE_REPORT_PLOTS)
+        elif args.mode == "offline":
+            self._fields["plots"].set_default_value(DEFAULT_OFFLINE_REPORT_PLOTS)
+
+        super().set_config_values(args)
+
+    def _preprocess_and_verify_arguments(self):
+        """
+        Enforces some rules on the config.
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If there is a problem with arguments or config.
+        """
+
+        if not self.export_path:
+            logger.warning(
+                f"--export-path not specified. Using {self._fields['export_path'].default_value()}"
+            )
+        elif self.export_path and not os.path.isdir(self.export_path):
+            raise TritonModelAnalyzerException(
+                f"Export path {self.export_path} is not a directory."
+            )
+
+    def _autofill_values(self):
+        """
+        Fill in the implied or default
+        config values.
+        """
+
+        new_report_model_configs = {}
+        for model in self.report_model_configs:
+            new_report_model_config = {}
+
+            # Plots
+            if not model.plots():
+                new_report_model_config["plots"] = {
+                    plot.name(): {
+                        "title": plot.title(),
+                        "x_axis": plot.x_axis(),
+                        "y_axis": plot.y_axis(),
+                        "monotonic": plot.monotonic(),
+                    }
+                    for plot in self.plots
+                }
+            else:
+                new_report_model_config["plots"] = {
+                    plot.name(): {
+                        "title": plot.title(),
+                        "x_axis": plot.x_axis(),
+                        "y_axis": plot.y_axis(),
+                        "monotonic": plot.monotonic(),
+                    }
+                    for plot in model.plots()
+                }
+
+            new_report_model_configs[
+                model.model_config_name()
+            ] = new_report_model_config
+
+        self._fields["report_model_configs"].set_value(new_report_model_configs)

model_analyzer/config/input/config_defaults.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+#
+# Common defaults
+#
+
+DEFAULT_CHECKPOINT_DIRECTORY = os.path.join(os.getcwd(), "checkpoints")
+DEFAULT_ONLINE_OBJECTIVES = {"perf_throughput": 10}
+DEFAULT_OFFLINE_OBJECTIVES = {"perf_throughput": 10}
+DEFAULT_MODEL_WEIGHTING = 1
+
+#
+# Profile Config defaults
+#
+
+DEFAULT_MONITORING_INTERVAL = 1.0
+DEFAULT_DURATION_SECONDS = 3
+DEFAULT_COLLECT_CPU_METRICS = False
+DEFAULT_LOG_LEVEL = "INFO"
+DEFAULT_GPUS = "all"
+DEFAULT_ALWAYS_REPORT_GPU_METRICS = False
+DEFAULT_SKIP_SUMMARY_REPORTS = False
+DEFAULT_SKIP_DETAILED_REPORTS = False
+DEFAULT_OUTPUT_MODEL_REPOSITORY = os.path.join(os.getcwd(), "output_model_repository")
+DEFAULT_OVERRIDE_OUTPUT_REPOSITORY_FLAG = False
+DEFAULT_BATCH_SIZES = 1
+DEFAULT_MAX_RETRIES = 50
+DEFAULT_CLIENT_PROTOCOL = "grpc"
+DEFAULT_RUN_CONFIG_MAX_CONCURRENCY = 1024
+DEFAULT_RUN_CONFIG_MIN_CONCURRENCY = 1
+DEFAULT_RUN_CONFIG_MAX_REQUEST_RATE = 8192
+DEFAULT_RUN_CONFIG_MIN_REQUEST_RATE = 16
+DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT = 5
+DEFAULT_RUN_CONFIG_MIN_INSTANCE_COUNT = 1
+DEFAULT_RUN_CONFIG_MIN_MODEL_BATCH_SIZE = 1
+DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE = 128
+DEFAULT_RUN_CONFIG_MAX_BINARY_SEARCH_STEPS = 5
+DEFAULT_RUN_CONFIG_SEARCH_DISABLE = False
+DEFAULT_RUN_CONFIG_SEARCH_MODE = "brute"
+DEFAULT_RUN_CONFIG_PROFILE_MODELS_CONCURRENTLY_ENABLE = False
+DEFAULT_OPTUNA_MIN_PERCENTAGE_OF_SEARCH_SPACE = 5
+DEFAULT_OPTUNA_MAX_PERCENTAGE_OF_SEARCH_SPACE = 10
+DEFAULT_OPTUNA_MIN_TRIALS = 20
+DEFAULT_OPTUNA_MAX_TRIALS = 200
+DEFAULT_OPTUNA_EARLY_EXIT_THRESHOLD = 10
+DEFAULT_USE_CONCURRENCY_FORMULA = False
+DEFAULT_REQUEST_RATE_SEARCH_ENABLE = False
+DEFAULT_CONCURRENCY_SWEEP_DISABLE = False
+DEFAULT_DCGM_DISABLE = False
+DEFAULT_TRITON_LAUNCH_MODE = "local"
+DEFAULT_TRITON_DOCKER_IMAGE = "nvcr.io/nvidia/tritonserver:25.11-py3"
+DEFAULT_TRITON_HTTP_ENDPOINT = "localhost:8000"
+DEFAULT_TRITON_GRPC_ENDPOINT = "localhost:8001"
+DEFAULT_TRITON_METRICS_URL = "http://localhost:8002/metrics"
+DEFAULT_TRITON_SERVER_PATH = "tritonserver"
+DEFAULT_TRITON_INSTALL_PATH = "/opt/tritonserver"
+DEFAULT_PERF_ANALYZER_TIMEOUT = 600
+DEFAULT_PERF_ANALYZER_CPU_UTIL = 80.0
+DEFAULT_PERF_ANALYZER_PATH = "perf_analyzer"
+DEFAULT_PERF_OUTPUT_FLAG = False
+DEFAULT_PERF_MAX_AUTO_ADJUSTS = 10
+DEFAULT_MEASUREMENT_MODE = "count_windows"
+DEFAULT_MODEL_TYPE = "generic"
+
+DEFAULT_ONLINE_PLOTS = {
+    "throughput_v_latency": {
+        "title": "Throughput vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "perf_throughput",
+        "monotonic": True,
+    },
+    "gpu_mem_v_latency": {
+        "title": "GPU Memory vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "gpu_used_memory",
+        "monotonic": False,
+    },
+}
+
+DEFAULT_OFFLINE_PLOTS = {
+    "through_v_batch_size": {
+        "title": "Throughput vs. Batch Size",
+        "x_axis": "batch_size",
+        "y_axis": "perf_throughput",
+        "monotonic": False,
+    }
+}
+
+DEFAULT_CPU_MEM_PLOT = {
+    "cpu_mem_v_latency": {
+        "title": "CPU Memory vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "cpu_used_ram",
+        "monotonic": False,
+    }
+}
+
+DEFAULT_EXPORT_PATH = os.getcwd()
+DEFAULT_FILENAME_MODEL_INFERENCE = "metrics-model-inference.csv"
+DEFAULT_FILENAME_MODEL_GPU = "metrics-model-gpu.csv"
+DEFAULT_FILENAME_SERVER_ONLY = "metrics-server-only.csv"
+
+DEFAULT_INFERENCE_OUTPUT_FIELDS = [
+    "model_name",
+    "batch_size",
+    "concurrency",
+    "model_config_path",
+    "instance_group",
+    "max_batch_size",
+    "satisfies_constraints",
+    "perf_throughput",
+    "perf_latency_p99",
+]
+DEFAULT_LLM_INFERENCE_OUTPUT_FIELDS = [
+    "model_name",
+    "batch_size",
+    "concurrency",
+    "model_config_path",
+    "instance_group",
+    "max_batch_size",
+    "satisfies_constraints",
+    "perf_throughput",
+    "perf_latency_p99",
+    "inter_token_latency_p99",
+    "time_to_first_token_p99",
+    "output_token_throughput",
+]
+DEFAULT_REQUEST_RATE_INFERENCE_OUTPUT_FIELDS = [
+    "model_name",
+    "batch_size",
+    "request_rate",
+    "model_config_path",
+    "instance_group",
+    "max_batch_size",
+    "satisfies_constraints",
+    "perf_throughput",
+    "perf_latency_p99",
+]
+DEFAULT_GPU_OUTPUT_FIELDS = [
+    "model_name",
+    "gpu_uuid",
+    "batch_size",
+    "concurrency",
+    "model_config_path",
+    "instance_group",
+    "satisfies_constraints",
+    "gpu_used_memory",
+    "gpu_utilization",
+    "gpu_power_usage",
+]
+DEFAULT_REQUEST_RATE_GPU_OUTPUT_FIELDS = [
+    "model_name",
+    "gpu_uuid",
+    "batch_size",
+    "request_rate",
+    "model_config_path",
+    "instance_group",
+    "satisfies_constraints",
+    "gpu_used_memory",
+    "gpu_utilization",
+    "gpu_power_usage",
+]
+DEFAULT_SERVER_OUTPUT_FIELDS = [
+    "model_name",
+    "gpu_uuid",
+    "gpu_used_memory",
+    "gpu_utilization",
+    "gpu_power_usage",
+]
+
+DEFAULT_NUM_CONFIGS_PER_MODEL = 3
+DEFAULT_NUM_TOP_MODEL_CONFIGS = 0
+
+#
+# Report Config defaults
+#
+
+DEFAULT_REPORT_FORMAT = "pdf"
+
+DEFAULT_ONLINE_REPORT_PLOTS = {
+    "gpu_mem_v_latency": {
+        "title": "GPU Memory vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "gpu_used_memory",
+        "monotonic": False,
+    },
+    "gpu_util_v_latency": {
+        "title": "GPU Utilization vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "gpu_utilization",
+        "monotonic": False,
+    },
+    "cpu_mem_v_latency": {
+        "title": "RAM Usage vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "cpu_used_ram",
+        "monotonic": False,
+    },
+    "gpu_power_v_latency": {
+        "title": "GPU Power vs. Latency",
+        "x_axis": "perf_latency_p99",
+        "y_axis": "gpu_power_usage",
+        "monotonic": False,
+    },
+}
+
+DEFAULT_OFFLINE_REPORT_PLOTS = {
+    "throughput_v_batch_size": {
+        "title": "Throughput vs. Batch Size",
+        "x_axis": "batch_size",
+        "y_axis": "perf_throughput",
+        "monotonic": False,
+    },
+    "latency_v_batch_size": {
+        "title": "p99 Latency vs. Batch Size",
+        "x_axis": "batch_size",
+        "y_axis": "perf_latency_p99",
+        "monotonic": False,
+    },
+}

model_analyzer/config/input/config_enum.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.constants import CONFIG_PARSER_FAILURE
+
+from .config_status import ConfigStatus
+from .config_value import ConfigValue
+
+
+class ConfigEnum(ConfigValue):
+    """
+    Enum type support for config.
+    """
+
+    def __init__(
+        self,
+        choices,
+        preprocess=None,
+        required=False,
+        validator=None,
+        output_mapper=None,
+        name=None,
+    ):
+        """
+        Create a new enum config field.
+
+        Parameters
+        ----------
+        choices : A list of allowed choices
+            The type of elements in the list
+        preprocess : callable
+            Function be called before setting new values.
+        required : bool
+            Whether a given config is required or not.
+        validator : callable or None
+            A validator for the final value of the field.
+        output_mapper: callable
+            This callable unifies the output value of this field.
+        name : str
+            Fully qualified name for this field.
+        """
+
+        self._choices = choices
+        self._type = self
+        super().__init__(preprocess, required, validator, output_mapper, name)
+
+    def set_value(self, value):
+        choices = self._choices
+
+        if value not in choices:
+            return ConfigStatus(
+                CONFIG_PARSER_FAILURE,
+                f'Value "{value}" for field "{self.name()}" is not acceptable.'
+                f' Value should be one of the following values: "{choices}".',
+                self,
+            )
+
+        return super().set_value(value)
+
+    def cli_type(self):
+        """
+        Get the type of this field for CLI.
+
+        Returns
+        -------
+        type
+            str
+        """
+
+        return str