triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/config/run/model_run_config.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List, Optional
+
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.perf_analyzer.perf_config import PerfAnalyzerConfig
+from model_analyzer.triton.model.model_config import ModelConfig
+from model_analyzer.triton.model.model_config_variant import ModelConfigVariant
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class ModelRunConfig:
+    """
+    Encapsulates all the information (ModelConfigVariant + PerfConfig) needed to run
+    a model in Perf Analyzer
+    """
+
+    DEFAULT_MAX_BATCH_SIZE = 1
+    DEFAULT_PERF_BATCH_SIZE = 1
+
+    def __init__(
+        self,
+        model_name: str,
+        model_config_variant: ModelConfigVariant,
+        perf_config: PerfAnalyzerConfig,
+    ) -> None:
+        """
+        Parameters
+        ----------
+        model_name: str
+            The name of the model
+        model_config_variant : ModelConfigVariant
+            model config variant corresponding to this run
+        perf_config : PerfAnalyzerConfig
+            List of possible run parameters to pass
+            to Perf Analyzer
+        """
+
+        self._model_name = model_name
+        self._model_config_variant = model_config_variant
+        self._perf_config = perf_config
+        self._composing_config_variants: List[ModelConfigVariant] = []
+
+    def model_name(self) -> str:
+        """
+        Get the original model name for this run config.
+
+        Returns
+        -------
+        str
+            Original model name
+        """
+
+        return self._model_name
+
+    def model_variant_name(self) -> str:
+        """
+        Get the model config variant name for this config.
+
+        Returns
+        -------
+        str
+            Model variant name
+        """
+        return (
+            self._model_config_variant.variant_name
+            if self._model_config_variant
+            else ""
+        )
+
+    def model_config_variant(self) -> ModelConfigVariant:
+        """
+        Returns
+        -------
+        ModelConfigVariant
+            The ModelConfigVariant corresponding to this run
+        """
+
+        return self._model_config_variant
+
+    def model_config(self) -> Optional[ModelConfig]:
+        """
+        Returns
+        -------
+        ModelConfig
+            The ModelConfig corresponding to this run
+        """
+
+        return (
+            self._model_config_variant.model_config
+            if self._model_config_variant
+            else None
+        )
+
+    def perf_config(self) -> PerfAnalyzerConfig:
+        """
+        Returns
+        -------
+        PerfAnalyzerConfig
+            run parameters corresponding to this run of
+            the perf analyzer
+        """
+
+        return self._perf_config
+
+    def composing_config_variants(self) -> List[ModelConfigVariant]:
+        """
+        Returns the list of composing model config variants
+        """
+
+        return self._composing_config_variants
+
+    def composing_configs(self) -> List[ModelConfig]:
+        """
+        Returns the list of composing model configs
+        """
+
+        if self._composing_config_variants:
+            composing_configs = [
+                composing_config_variant.model_config
+                for composing_config_variant in self._composing_config_variants
+            ]
+            return composing_configs
+        else:
+            return []
+
+    def representation(self) -> str:
+        """
+        Returns a representation string for the ModelRunConfig that can be used
+        as a key to uniquely identify it
+        """
+        repr = self.model_variant_name()
+        repr += " " + self.perf_config().representation()
+
+        if self._composing_config_variants:
+            repr += " " + (",").join(self.get_composing_config_names())  # type: ignore
+
+        return repr
+
+    def _check_for_client_vs_model_batch_size(self) -> bool:
+        """
+        Returns false if client batch size is greater than model batch size. Else true
+        """
+        model_config = self._model_config_variant.model_config.get_config()
+
+        max_batch_size = (
+            model_config["max_batch_size"]
+            if "max_batch_size" in model_config
+            else self.DEFAULT_MAX_BATCH_SIZE
+        )
+        perf_batch_size = (
+            self._perf_config["batch-size"]
+            if "batch-size" in self._perf_config
+            else self.DEFAULT_PERF_BATCH_SIZE
+        )
+
+        legal = max_batch_size >= perf_batch_size
+        if not legal:
+            logger.debug(
+                f"Illegal model run config because client batch size {perf_batch_size} is greater than model max batch size {max_batch_size}"
+            )
+
+        return legal
+
+    def _check_for_preferred_vs_model_batch_size(self) -> bool:
+        """
+        Returns false if maximum of preferred batch size is greater than model batch size. Else true
+        """
+        legal = True
+
+        model_configs = self._create_model_config_dicts()
+
+        for model_config in model_configs:
+            max_batch_size = (
+                model_config["max_batch_size"]
+                if "max_batch_size" in model_config
+                else self.DEFAULT_MAX_BATCH_SIZE
+            )
+
+            if (
+                "dynamic_batching" in model_config
+                and "preferred_batch_size" in model_config["dynamic_batching"]
+            ):
+                max_preferred_batch_size = max(
+                    model_config["dynamic_batching"]["preferred_batch_size"]
+                )
+                legal = max_batch_size >= max_preferred_batch_size
+
+                if not legal:
+                    logger.debug(
+                        f"Illegal model run config because maximum of {model_config['name']}'s preferred batch size {max_preferred_batch_size} is greater than model max batch size {max_batch_size}"
+                    )
+                    return legal
+
+        return legal
+
+    def _create_model_config_dicts(self) -> List[Dict]:
+        """
+        Create a list of model config dictionaries for
+        the given model + composing models
+        """
+        model_configs = (
+            []
+            if self.is_ensemble_model()
+            else [self._model_config_variant.model_config.get_config()]
+        )
+
+        model_configs.extend(
+            [
+                composing_config_variant.model_config.get_config()
+                for composing_config_variant in self._composing_config_variants
+            ]
+        )
+
+        return model_configs
+
+    def is_legal_combination(self):
+        """
+        Returns true if the run_config is valid and should be run. Else false
+        """
+        legal = (
+            self._check_for_client_vs_model_batch_size()
+            and self._check_for_preferred_vs_model_batch_size()
+        )
+
+        return legal
+
+    def is_ensemble_model(self) -> bool:
+        """
+        Returns true if the model config is an ensemble model
+        """
+        return self._model_config_variant.model_config.is_ensemble()
+
+    def is_bls_model(self) -> bool:
+        """
+        Returns true if the model config is a BLS model
+        """
+        # If composing configs are present and it's not an ensemble it must be a BLS
+        # Note: this will need to change if we allow ensembles to contain BLS models
+        return (
+            not self._model_config_variant.model_config.is_ensemble()
+            and len(self._composing_config_variants) > 0
+        )
+
+    def get_composing_config_names(self) -> Optional[List[str]]:
+        """
+        Returns list of composing config names
+        """
+        return [
+            composing_config_variant.variant_name
+            for composing_config_variant in self._composing_config_variants
+        ]
+
+    def add_composing_model_config_variants(
+        self, composing_model_config_variants: List[ModelConfigVariant]
+    ) -> None:
+        """
+        Adds a list of composing model config variants
+        """
+        for composing_model_config_variant in composing_model_config_variants:
+            self._composing_config_variants.append(composing_model_config_variant)
+
+    @classmethod
+    def from_dict(cls, model_run_config_dict):
+        model_run_config = ModelRunConfig(None, None, None)
+        model_run_config._model_name = model_run_config_dict["_model_name"]
+
+        if "_model_config_variant" in model_run_config_dict:
+            model_config = ModelConfig.from_dict(
+                model_run_config_dict["_model_config_variant"]["model_config"]
+            )
+            variant_name = model_run_config_dict["_model_config_variant"][
+                "variant_name"
+            ]
+
+            model_run_config._model_config_variant = ModelConfigVariant(
+                model_config, variant_name
+            )
+
+        model_run_config._perf_config = PerfAnalyzerConfig.from_dict(
+            model_run_config_dict["_perf_config"]
+        )
+
+        if "_composing_config_variants" in model_run_config_dict:
+            model_run_config._composing_config_variants = [
+                ModelConfigVariant(
+                    ModelConfig.from_dict(
+                        composing_config_variant_dict["model_config"]
+                    ),
+                    composing_config_variant_dict["variant_name"],
+                )
+                for composing_config_variant_dict in model_run_config_dict[
+                    "_composing_config_variants"
+                ]
+            ]
+
+        return model_run_config

model_analyzer/config/run/run_config.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from model_analyzer.config.run.model_run_config import ModelRunConfig
+from model_analyzer.perf_analyzer.genai_perf_config import GenaiPerfConfig
+
+
+class RunConfig:
+    """
+    Encapsulates all the information needed to run one or more models
+    at the same time in Perf Analyzer
+    """
+
+    def __init__(self, triton_env, genai_perf_flags=None):
+        """
+        Parameters
+        ----------
+        triton_env : dict
+            A dictionary of environment variables to set
+            when launching tritonserver
+
+        genai_perf_flags: dict
+            The set of flags used when calling genai_perf for LLM models
+        """
+
+        self._triton_env = triton_env
+        self._genai_perf_config = GenaiPerfConfig()
+        self._genai_perf_config.update_config(genai_perf_flags)
+        self._model_run_configs: List[ModelRunConfig] = []
+
+    def add_model_run_config(self, model_run_config):
+        """
+        Add a ModelRunConfig to this RunConfig
+        """
+        self._model_run_configs.append(model_run_config)
+
+    def model_run_configs(self) -> List[ModelRunConfig]:
+        """
+        Returns the list of ModelRunConfigs to run concurrently
+        """
+        return self._model_run_configs
+
+    def representation(self):
+        """
+        Returns a representation string for the RunConfig that can be used
+        as a key to uniquely identify it
+        """
+        return "".join([mrc.representation() for mrc in self.model_run_configs()])
+
+    def is_legal_combination(self):
+        """
+        Returns true if all model_run_configs are valid
+        """
+        return all(
+            [
+                model_run_config.is_legal_combination()
+                for model_run_config in self._model_run_configs
+            ]
+        )
+
+    def is_ensemble_model(self) -> bool:
+        """
+        Returns true if the first model config is an ensemble
+        (an ensemble cannot be part of a multi-model)
+        """
+        return self._model_run_configs[0].is_ensemble_model()
+
+    def is_bls_model(self) -> bool:
+        """
+        Returns true if the first model config is a BLS model
+        (a BLS cannot be part of a multi-model)
+        """
+        return self._model_run_configs[0].is_bls_model()
+
+    def cpu_only(self):
+        """
+        Returns true if all model_run_configs only operate on the CPU
+        """
+        return all(
+            [
+                model_run_config.model_config_variant().cpu_only
+                for model_run_config in self._model_run_configs
+            ]
+        )
+
+    def triton_environment(self):
+        """
+        Returns
+        -------
+        dict
+            The environment that tritonserver
+            was run with for this RunConfig
+        """
+
+        return self._triton_env
+
+    def genai_perf_config(self):
+        return self._genai_perf_config
+
+    def models_name(self):
+        """Returns a single comma-joined name of the original model names"""
+        return ",".join([mrc.model_name() for mrc in self.model_run_configs()])
+
+    def model_variants_name(self):
+        """Returns a single comma-joined name of the model variant names"""
+        return ",".join([mrc.model_variant_name() for mrc in self.model_run_configs()])
+
+    def composing_config_variants(self):
+        """
+        Returns a list of composing model config variants from the first model run config
+        (an ensemble/BLS cannot be part of a multi-model profile)
+        """
+        return self._model_run_configs[0].composing_config_variants()
+
+    def composing_model_variants_name(self):
+        """
+        Returns a single comma-joined name of the composing model variant names
+        (an ensemble/BLS cannot be part of a multi-model profile)
+        """
+        return ",".join(
+            [
+                cvc.variant_name
+                for cvc in self.model_run_configs()[0].composing_config_variants()
+            ]
+        )
+
+    def composing_configs(self):
+        """
+        Returns a list of composing model configs from the first model run config
+        (an ensemble/BLS cannot be part of a multi-model profile)
+        """
+        return self._model_run_configs[0].composing_configs()
+
+    def combined_model_variants_name(self):
+        """
+        Combines the model + composing model's variant names (joined with a '::')
+        """
+        if self.composing_model_variants_name():
+            return (
+                f"{self.model_variants_name()}::{self.composing_model_variants_name()}"
+            )
+        else:
+            return self.model_variants_name()
+
+    @classmethod
+    def from_dict(cls, run_config_dict):
+        run_config = RunConfig({})
+
+        run_config._triton_env = run_config_dict["_triton_env"]
+        for mrc_dict in run_config_dict["_model_run_configs"]:
+            run_config._model_run_configs.append(ModelRunConfig.from_dict(mrc_dict))
+
+        return run_config

model_analyzer/constants.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict
+
+# Config constants
+CONFIG_PARSER_SUCCESS = 1
+CONFIG_PARSER_FAILURE = 0
+
+# Result Table constants
+RESULT_TABLE_COLUMN_PADDING = 2
+
+# Result Comparator Constants
+COMPARISON_SCORE_THRESHOLD = 0
+
+# Dict of parameters to apply on top of the default
+# config to result in the default config (empty dict)
+DEFAULT_CONFIG_PARAMS: Dict[str, Any] = {}
+
+# Run Search
+THROUGHPUT_MINIMUM_GAIN = 0.05
+THROUGHPUT_MINIMUM_CONSECUTIVE_PARAMETER_TRIES = 4
+THROUGHPUT_MINIMUM_CONSECUTIVE_BATCH_SIZE_TRIES = 4
+
+# Quick search algorithm constants
+RADIUS = 3
+MIN_INITIALIZED = 3
+
+# Reports
+TOP_MODELS_REPORT_KEY = "Best Configs Across All Models"
+
+# State Management
+MAX_NUMBER_OF_INTERRUPTS = 3
+
+# Perf Analyzer
+MEASUREMENT_WINDOW_STEP = 1000
+MEASUREMENT_REQUEST_COUNT_STEP = 50
+INTERVAL_SLEEP_TIME = 1
+PERF_ANALYZER_MEASUREMENT_WINDOW = 5000
+PERF_ANALYZER_MINIMUM_REQUEST_COUNT = 50
+SECONDS_TO_MILLISECONDS_MULTIPLIER = 1000
+
+# Triton Server
+SERVER_OUTPUT_TIMEOUT_SECS = 5
+
+# Logging
+LOGGER_NAME = "model_analyzer_logger"
+
+# PA Error Log Filename
+PA_ERROR_LOG_FILENAME = "perf_analyzer_error.log"
+
+# Constraints
+GLOBAL_CONSTRAINTS_KEY = "__default__"
+
+# Measurement constants
+INVALID_MEASUREMENT_THRESHOLD = 2
+
+# Model analyzer package name
+PACKAGE_NAME = "triton-model-analyzer"
+
+# GENAI-PERF
+GENAI_PERF_CSV = "profile_export_genai_perf.csv"
+GENAI_PERF_COLLATERAL = ["llm_inputs.json", "profile_export.json"]

model_analyzer/device/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/device/device.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Device:
+    """
+    Generic device class representing devices being monitored
+    """
+
+    def __init__(self):
+        pass

model_analyzer/device/gpu_device.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.device.device import Device
+
+
+class GPUDevice(Device):
+    """
+    Representing a GPU device
+    """
+
+    def __init__(self, device_name, device_id, pci_bus_id, device_uuid):
+        """
+        Parameters
+        ----------
+            device_name: str
+                Human readable name of the device
+            device_id : int
+                Device id according to the `nvidia-smi` output
+            pci_bus_id : str
+                PCI bus id
+            device_uuid : str
+                Device UUID
+        """
+
+        assert type(device_name) is str
+        assert type(device_id) is int
+        assert type(pci_bus_id) is str
+        assert type(device_uuid) is str
+
+        self._device_name = device_name
+        self._device_id = device_id
+        self._pci_bus_id = pci_bus_id
+        self._device_uuid = device_uuid
+
+    def device_name(self):
+        """
+        Returns
+        -------
+        str
+            device name
+        """
+
+        return self._device_name
+
+    def device_id(self):
+        """
+        Returns
+        -------
+        int
+            device id of this GPU
+        """
+
+        return self._device_id
+
+    def pci_bus_id(self):
+        """
+        Returns
+        -------
+        bytes
+            PCI bus id of this GPU
+        """
+
+        return self._pci_bus_id
+
+    def device_uuid(self):
+        """
+        Returns
+        -------
+        str
+            UUID of this GPU
+        """
+
+        return self._device_uuid