triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/config/generate/manual_model_config_generator.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List
+
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.constants import DEFAULT_CONFIG_PARAMS, LOGGER_NAME
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.triton.model.model_config_variant import ModelConfigVariant
+
+from .base_model_config_generator import BaseModelConfigGenerator
+from .generator_utils import GeneratorUtils
+from .model_profile_spec import ModelProfileSpec
+
+logger = logging.getLogger(LOGGER_NAME)
+from copy import deepcopy
+
+
+class ManualModelConfigGenerator(BaseModelConfigGenerator):
+    """Given a model, generates model configs in manual search mode"""
+
+    _log_first_run = False
+
+    def __init__(
+        self,
+        config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        model: ModelProfileSpec,
+        client: TritonClient,
+        model_variant_name_manager: ModelVariantNameManager,
+        default_only: bool,
+        early_exit_enable: bool,
+    ) -> None:
+        """
+        Parameters
+        ----------
+        config: ModelAnalyzerConfig
+        gpus: List of GPUDevices
+        model: The model to generate ModelConfigs for
+        client: TritonClient
+        model_variant_name_manager: ModelVariantNameManager
+        default_only: Bool
+            If true, only the default config will be generated
+            If false, the default config will NOT be generated
+        early_exit_enable: Bool
+            If true, the generator can early exit if throughput plateaus
+        """
+        super().__init__(
+            config,
+            gpus,
+            model,
+            client,
+            model_variant_name_manager,
+            default_only,
+            early_exit_enable,
+        )
+
+        if not ManualModelConfigGenerator._log_first_run:
+            logger.info("")
+            logger.info("Starting manual brute search")
+            logger.info("")
+            ManualModelConfigGenerator._log_first_run = True
+
+        self._search_disabled = config.run_config_search_disable
+        self._curr_config_index = 0
+        self._curr_max_batch_size_index = 0
+
+        self._max_batch_sizes = None
+        self._non_max_batch_size_param_combos: List[Dict] = []
+        self._determine_max_batch_sizes_and_param_combos()
+
+        # All configs are pregenerated in _configs[][]
+        # Indexed as follows:
+        #    _configs[_curr_config_index][_curr_max_batch_size_index]
+        #
+        self._configs = self._generate_model_config_variants()
+
+    def _done_walking(self) -> bool:
+        return len(self._configs) == self._curr_config_index
+
+    def _done_walking_max_batch_size(self) -> bool:
+        if (
+            self._max_batch_sizes is None
+            or len(self._max_batch_sizes) == self._curr_max_batch_size_index
+        ):
+            return True
+
+        if self._early_exit_enable and self._last_results_erroneous():
+            return True
+
+        if self._early_exit_enable and not self._last_results_increased_throughput():
+            self._print_max_batch_size_plateau_warning()
+            return True
+        return False
+
+    def _step(self) -> None:
+        self._step_max_batch_size()
+
+        if self._done_walking_max_batch_size():
+            self._reset_max_batch_size()
+            self._step_config()
+
+    def _reset_max_batch_size(self) -> None:
+        super()._reset_max_batch_size()
+        self._curr_max_batch_size_index = 0
+
+    def _step_config(self) -> None:
+        self._curr_config_index += 1
+
+    def _step_max_batch_size(self) -> None:
+        self._curr_max_batch_size_index += 1
+
+        last_max_throughput = self._get_last_results_max_throughput()
+        if last_max_throughput:
+            self._curr_max_batch_size_throughputs.append(last_max_throughput)
+
+    def _get_next_model_config_variant(self) -> ModelConfigVariant:
+        return self._configs[self._curr_config_index][self._curr_max_batch_size_index]
+
+    def _generate_model_config_variants(self) -> List[List[ModelConfigVariant]]:
+        """Generate all model config combinations"""
+
+        model_config_variants = []
+        for param_combo in self._non_max_batch_size_param_combos:
+            configs_with_max_batch_size = []
+            if self._max_batch_sizes:
+                for mbs in self._max_batch_sizes:
+                    param_combo["max_batch_size"] = mbs
+                    model_config_variant = self._make_direct_mode_model_config_variant(
+                        param_combo
+                    )
+                    configs_with_max_batch_size.append(model_config_variant)
+            else:
+                model_config_variant = self._make_direct_mode_model_config_variant(
+                    param_combo
+                )
+                configs_with_max_batch_size.append(model_config_variant)
+
+            model_config_variants.append(configs_with_max_batch_size)
+
+        return model_config_variants
+
+    def _determine_max_batch_sizes_and_param_combos(self) -> None:
+        """
+        Determine self._max_batch_sizes and self._non_max_batch_size_param_combos
+        """
+        if self._default_only:
+            self._non_max_batch_size_param_combos = [DEFAULT_CONFIG_PARAMS]
+        else:
+            model_config_params = deepcopy(self._base_model.model_config_parameters())
+            if model_config_params:
+                self._max_batch_sizes = model_config_params.pop("max_batch_size", None)
+                self._non_max_batch_size_param_combos = (
+                    GeneratorUtils.generate_combinations(model_config_params)
+                )
+            else:
+                if self._search_disabled:
+                    self._non_max_batch_size_param_combos = (
+                        self._generate_search_disabled_param_combos()
+                    )
+                else:
+                    raise TritonModelAnalyzerException(
+                        f"Automatic search not supported in ManualModelConfigGenerator"
+                    )
+
+    def _generate_search_disabled_param_combos(self) -> List[Dict]:
+        """Return the configs when we want to search but searching is disabled"""
+        return [DEFAULT_CONFIG_PARAMS]

model_analyzer/config/generate/model_config_generator_factory.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.triton.client.client import TritonClient
+
+from .automatic_model_config_generator import AutomaticModelConfigGenerator
+from .config_generator_interface import ConfigGeneratorInterface
+from .manual_model_config_generator import ManualModelConfigGenerator
+from .model_profile_spec import ModelProfileSpec
+
+
+class ModelConfigGeneratorFactory:
+    """
+    Factory that creates the correct Config Generators
+    """
+
+    @staticmethod
+    def create_model_config_generator(
+        config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        model: ModelProfileSpec,
+        client: TritonClient,
+        model_variant_name_manager: ModelVariantNameManager,
+        default_only: bool,
+        early_exit_enable: bool,
+    ) -> ConfigGeneratorInterface:
+        """
+        Parameters
+        ----------
+        config: ConfigCommandProfile
+            The Model Analyzer config file for the profile step
+        gpus: List of GPUDevices
+        model: ConfigModelProfileSpec
+            The model to generate ModelRunConfigs for
+        client: TritonClient
+            The client handle used to send requests to Triton
+        model_variant_name_manager: ModelVariantNameManager
+            Used to manage the model variant names
+        default_only: Bool
+            If true, only the default config will be generated by the created generator
+            If false, the default config will NOT be generated by the created generator
+        early_exit_enable: Bool
+            If true, the created generator can early exit if throughput plateaus
+
+        Returns
+        -------
+        A generator that implements ConfigGeneratorInterface and creates ModelConfigs
+        """
+
+        search_disabled = config.run_config_search_disable
+        model_config_params = model.model_config_parameters()
+
+        if search_disabled or model_config_params:
+            return ManualModelConfigGenerator(
+                config,
+                gpus,
+                model,
+                client,
+                model_variant_name_manager,
+                default_only,
+                early_exit_enable,
+            )
+        else:
+            return AutomaticModelConfigGenerator(
+                config,
+                gpus,
+                model,
+                client,
+                model_variant_name_manager,
+                default_only,
+                early_exit_enable,
+            )

model_analyzer/config/generate/model_profile_spec.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import deepcopy
+from typing import List
+
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.objects.config_model_profile_spec import (
+    ConfigModelProfileSpec,
+)
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.triton.model.model_config import ModelConfig
+
+
+class ModelProfileSpec(ConfigModelProfileSpec):
+    """
+    The profile configuration and default model config for a single model to be profiled
+    """
+
+    def __init__(
+        self,
+        spec: ConfigModelProfileSpec,
+        config: ConfigCommandProfile,
+        client: TritonClient,
+        gpus: List[GPUDevice],
+    ):
+        super().__init__(spec.model_name())
+        self.__dict__ = deepcopy(spec.__dict__)
+
+        self._default_model_config = ModelConfig.create_model_config_dict(
+            config, client, gpus, config.model_repository, spec.model_name()
+        )
+
+        if spec.model_name() in config.cpu_only_composing_models:
+            self._cpu_only = True
+
+    def get_default_config(self) -> dict:
+        """Returns the default configuration for this model"""
+        return deepcopy(self._default_model_config)
+
+    def supports_batching(self) -> bool:
+        """Returns True if this model supports batching. Else False"""
+        if (
+            "max_batch_size" not in self._default_model_config
+            or self._default_model_config["max_batch_size"] == 0
+        ):
+            return False
+        return True
+
+    def supports_dynamic_batching(self) -> bool:
+        """Returns True if this model supports dynamic batching. Else False"""
+        supports_dynamic_batching = self.supports_batching()
+
+        if "sequence_batching" in self._default_model_config:
+            supports_dynamic_batching = False
+        return supports_dynamic_batching
+
+    def is_ensemble(self) -> bool:
+        """Returns true if the model is an ensemble"""
+        return "ensemble_scheduling" in self._default_model_config

model_analyzer/config/generate/model_run_config_generator.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Generator, List, Optional
+
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.run.model_run_config import ModelRunConfig
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.perf_analyzer.perf_config import PerfAnalyzerConfig
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.triton.model.model_config_variant import ModelConfigVariant
+
+from .config_generator_interface import ConfigGeneratorInterface
+from .model_config_generator_factory import ModelConfigGeneratorFactory
+from .model_profile_spec import ModelProfileSpec
+from .perf_analyzer_config_generator import PerfAnalyzerConfigGenerator
+
+
+class ModelRunConfigGenerator(ConfigGeneratorInterface):
+    """
+    Given a model, generates all ModelRunConfigs (combination of
+    ModelConfig and PerfConfig)
+    """
+
+    def __init__(
+        self,
+        config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        model: ModelProfileSpec,
+        client: TritonClient,
+        model_variant_name_manager: ModelVariantNameManager,
+        default_only: bool,
+    ) -> None:
+        """
+        Parameters
+        ----------
+        config: ModelAnalyzerConfig
+
+        gpus: List of GPUDevices
+
+        model: ConfigModelProfileSpec
+            The model to generate ModelRunConfigs for
+
+        client: TritonClient
+
+        model_variant_name_manager: ModelVariantNameManager
+
+        default_only: Bool
+        """
+        self._config = config
+        self._gpus = gpus
+        self._model = model
+        self._client = client
+        self._model_variant_name_manager = model_variant_name_manager
+
+        self._model_name = model.model_name()
+
+        self._model_pa_flags = model.perf_analyzer_flags()
+        self._model_parameters = model.parameters()
+        self._triton_server_env = model.triton_server_environment()
+
+        self._determine_early_exit_enables(config, model)
+
+        self._mcg = ModelConfigGeneratorFactory.create_model_config_generator(
+            self._config,
+            self._gpus,
+            model,
+            self._client,
+            self._model_variant_name_manager,
+            default_only,
+            self._mcg_early_exit_enable,
+        )
+
+        self._curr_mc_measurements: List[Optional[RunConfigMeasurement]] = []
+
+    def get_configs(self) -> Generator[ModelRunConfig, None, None]:
+        """
+        Returns
+        -------
+        ModelRunConfig
+            The next ModelRunConfig generated by this class
+        """
+        for model_config_variant in self._mcg.get_configs():
+            self._pacg = PerfAnalyzerConfigGenerator(
+                self._config,
+                model_config_variant.model_config.get_field("name"),
+                self._model_pa_flags,
+                self._model_parameters,
+                self._pacg_early_exit_enable,
+            )
+
+            for perf_analyzer_config in self._pacg.get_configs():
+                run_config = self._generate_model_run_config(
+                    model_config_variant, perf_analyzer_config
+                )
+                yield run_config
+
+            self._set_last_results_model_config_generator()
+
+    def set_last_results(
+        self, measurements: List[Optional[RunConfigMeasurement]]
+    ) -> None:
+        """
+        Given the results from the last ModelRunConfig, make decisions
+        about future configurations to generate
+
+        Parameters
+        ----------
+        measurements: List of Measurements from the last run(s)
+        """
+        self._pacg.set_last_results(measurements)
+        self._curr_mc_measurements.extend(measurements)
+
+    def _set_last_results_model_config_generator(self) -> None:
+        self._mcg.set_last_results(self._curr_mc_measurements)
+        self._curr_mc_measurements = []
+
+    def _generate_model_run_config(
+        self,
+        model_config_variant: ModelConfigVariant,
+        perf_analyzer_config: PerfAnalyzerConfig,
+    ) -> ModelRunConfig:
+        run_config = ModelRunConfig(
+            self._model_name, model_config_variant, perf_analyzer_config
+        )
+
+        return run_config
+
+    def _determine_early_exit_enables(
+        self, config: ConfigCommandProfile, model: ModelProfileSpec
+    ) -> None:
+        early_exit_enable = config.early_exit_enable
+        concurrency_specified = model.parameters()["concurrency"]
+        config_parameters_exist = model.model_config_parameters()
+
+        self._pacg_early_exit_enable = early_exit_enable or not concurrency_specified
+        self._mcg_early_exit_enable = early_exit_enable or not config_parameters_exist

model_analyzer/config/generate/model_variant_name_manager.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import deepcopy
+from typing import Dict, List, Tuple
+
+from model_analyzer.constants import DEFAULT_CONFIG_PARAMS
+from model_analyzer.triton.model.model_config_variant import ModelConfigVariant
+
+
+class ModelVariantNameManager:
+    def __init__(self) -> None:
+        # Dict of {model_config_name: model_config_dict}
+        self._model_config_dicts: Dict[str, Dict] = {}
+
+        # Dict of {base_model_name: current_count_integer}
+        self._model_name_index: Dict[str, int] = {}
+
+    @classmethod
+    def from_dict(
+        cls, model_variant_name_manager_dict: Dict
+    ) -> "ModelVariantNameManager":
+        model_variant_name_manager = ModelVariantNameManager()
+
+        model_variant_name_manager._model_config_dicts = (
+            model_variant_name_manager_dict["_model_config_dicts"]
+        )
+        model_variant_name_manager._model_name_index = model_variant_name_manager_dict[
+            "_model_name_index"
+        ]
+
+        return model_variant_name_manager
+
+    @staticmethod
+    def make_ensemble_composing_model_key(
+        ensemble_model_config_variants: List[ModelConfigVariant],
+    ) -> Dict[str, str]:
+        ensemble_names = [emcv.variant_name for emcv in ensemble_model_config_variants]
+        ensemble_key = ",".join(ensemble_names)
+
+        return {"key": ensemble_key}
+
+    def get_model_variant_name(
+        self, model_name: str, model_config_dict: Dict, param_combo: Dict
+    ) -> Tuple[bool, str]:
+        """
+        Given a base model name and a dict of parameters to be applied
+        to the base model config, return if the variant already existed
+        and the name of the model variant
+
+        If the same input values are provided to this function multiple times,
+        the same value will be returned
+        """
+        return self._get_variant_name(
+            model_name, model_config_dict, is_ensemble=False, param_combo=param_combo
+        )
+
+    def get_ensemble_model_variant_name(
+        self, model_name: str, ensemble_dict: Dict
+    ) -> Tuple[bool, str]:
+        """
+        Given a base ensemble model name and a dict of ensemble composing configs,
+        return if the variant already existed and the name of the model variant
+
+        If the same input values are provided to this function multiple times,
+        the same value will be returned
+        """
+        return self._get_variant_name(model_name, ensemble_dict, is_ensemble=True)
+
+    def _get_variant_name(
+        self,
+        model_name: str,
+        config_dict: Dict,
+        is_ensemble: bool,
+        param_combo: Dict = {},
+    ) -> Tuple[bool, str]:
+        model_config_dict = self._copy_and_restore_model_config_dict_name(
+            model_name, config_dict
+        )
+
+        variant_found, model_variant_name = self._find_existing_variant(
+            model_config_dict
+        )
+
+        if is_ensemble:
+            if self._is_ensemble_default_config(config_dict):
+                return (False, model_name + "_config_default")
+        else:
+            if self._is_default_config(param_combo):
+                return (False, model_name + "_config_default")
+
+        if variant_found:
+            return (True, model_variant_name)
+
+        model_variant_name = self._create_new_model_variant(
+            model_name, model_config_dict
+        )
+
+        return (False, model_variant_name)
+
+    def _copy_and_restore_model_config_dict_name(
+        self, model_name: str, model_config_dict: Dict
+    ) -> Dict:
+        model_config_dict_copy = deepcopy(model_config_dict)
+        model_config_dict_copy["name"] = model_name
+
+        return model_config_dict_copy
+
+    def _find_existing_variant(self, model_config_dict: Dict) -> Tuple[bool, str]:
+        for (
+            model_config_name,
+            model_config_variant_dict,
+        ) in self._model_config_dicts.items():
+            if model_config_dict == model_config_variant_dict:
+                return (True, model_config_name)
+
+        return (False, "")
+
+    def _is_default_config(self, param_combo: Dict) -> bool:
+        return param_combo == DEFAULT_CONFIG_PARAMS
+
+    def _is_ensemble_default_config(self, ensemble_dict: Dict) -> bool:
+        return "_config_default" in ensemble_dict["key"]
+
+    def _create_new_model_variant(
+        self, model_name: str, model_config_dict: Dict
+    ) -> str:
+        if model_name not in self._model_name_index:
+            new_index = 0
+        else:
+            new_index = self._model_name_index[model_name] + 1
+
+        self._model_name_index[model_name] = new_index
+        model_config_name = model_name + "_config_" + str(new_index)
+        self._model_config_dicts[model_config_name] = model_config_dict
+
+        return model_config_name