triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/config/generate/run_config_generator_factory.py

@@ -0,0 +1,329 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List
+
+from model_analyzer.config.generate.model_profile_spec import ModelProfileSpec
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.generate.search_parameters import SearchParameters
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.objects.config_model_profile_spec import (
+    ConfigModelProfileSpec,
+)
+from model_analyzer.constants import MIN_INITIALIZED, RADIUS
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+from model_analyzer.result.result_manager import ResultManager
+from model_analyzer.state.analyzer_state_manager import AnalyzerStateManager
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.triton.model.model_config import ModelConfig
+
+from .brute_plus_binary_parameter_search_run_config_generator import (
+    BrutePlusBinaryParameterSearchRunConfigGenerator,
+)
+from .config_generator_interface import ConfigGeneratorInterface
+from .optuna_plus_concurrency_sweep_run_config_generator import (
+    OptunaPlusConcurrencySweepRunConfigGenerator,
+)
+from .quick_plus_concurrency_sweep_run_config_generator import (
+    QuickPlusConcurrencySweepRunConfigGenerator,
+)
+from .search_config import SearchConfig
+from .search_dimension import SearchDimension
+from .search_dimensions import SearchDimensions
+
+
+class RunConfigGeneratorFactory:
+    """
+    Factory that creates the correct RunConfig Generators
+    """
+
+    @staticmethod
+    def create_run_config_generator(
+        command_config: ConfigCommandProfile,
+        state_manager: AnalyzerStateManager,
+        gpus: List[GPUDevice],
+        models: List[ConfigModelProfileSpec],
+        client: TritonClient,
+        result_manager: ResultManager,
+        model_variant_name_manager: ModelVariantNameManager,
+        search_parameters: Dict[str, SearchParameters],
+        composing_search_parameters: Dict[str, SearchParameters],
+    ) -> ConfigGeneratorInterface:
+        """
+        Parameters
+        ----------
+        command_config: ConfigCommandProfile
+            The Model Analyzer config file for the profile step
+        state_manager: AnalyzerStateManager
+            The object that allows control and update of checkpoint state
+        gpus: List of GPUDevices
+        models: list of ConfigModelProfileSpec
+            The models to generate RunConfigs for
+        client: TritonClient
+            The client handle used to send requests to Triton
+        result_manager: ResultManager
+            The object that handles storing and sorting the results from the perf analyzer
+        model_variant_name_manager: ModelVariantNameManager
+            Maps model variants to config names
+        search_parameters: SearchParameters
+            The object that handles the users configuration search parameters
+        composing_search_parameters: SearchParameters
+            The object that handles the users configuration search parameters for composing models
+
+        Returns
+        -------
+        A generator that implements ConfigGeneratorInterface and creates RunConfigs
+        """
+
+        new_models = []
+        for model in models:
+            new_models.append(ModelProfileSpec(model, command_config, client, gpus))
+
+        composing_models = RunConfigGeneratorFactory._create_composing_models(
+            new_models, command_config, client, gpus
+        )
+
+        for composing_model in composing_models:
+            composing_search_parameters[
+                composing_model.model_name()
+            ] = SearchParameters(
+                config=command_config,
+                model=composing_model,
+                is_composing_model=True,
+            )
+
+        if command_config.run_config_search_mode == "optuna":
+            return RunConfigGeneratorFactory._create_optuna_plus_concurrency_sweep_run_config_generator(
+                command_config=command_config,
+                state_manager=state_manager,
+                gpu_count=len(gpus),
+                models=new_models,
+                composing_models=composing_models,
+                result_manager=result_manager,
+                search_parameters=search_parameters,
+                composing_search_parameters=composing_search_parameters,
+                model_variant_name_manager=model_variant_name_manager,
+            )
+        elif command_config.run_config_search_mode == "quick" or composing_models:
+            return RunConfigGeneratorFactory._create_quick_plus_concurrency_sweep_run_config_generator(
+                command_config=command_config,
+                gpu_count=len(gpus),
+                models=new_models,
+                composing_models=composing_models,
+                result_manager=result_manager,
+                model_variant_name_manager=model_variant_name_manager,
+            )
+        elif command_config.run_config_search_mode == "brute":
+            return RunConfigGeneratorFactory._create_brute_plus_binary_parameter_search_run_config_generator(
+                command_config=command_config,
+                gpus=gpus,
+                models=new_models,
+                client=client,
+                result_manager=result_manager,
+                model_variant_name_manager=model_variant_name_manager,
+            )
+        else:
+            raise TritonModelAnalyzerException(
+                f"Unexpected search mode {command_config.run_config_search_mode}"
+            )
+
+    @staticmethod
+    def _create_brute_plus_binary_parameter_search_run_config_generator(
+        command_config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        models: List[ModelProfileSpec],
+        client: TritonClient,
+        result_manager: ResultManager,
+        model_variant_name_manager: ModelVariantNameManager,
+    ) -> ConfigGeneratorInterface:
+        return BrutePlusBinaryParameterSearchRunConfigGenerator(
+            config=command_config,
+            gpus=gpus,
+            models=models,
+            client=client,
+            result_manager=result_manager,
+            model_variant_name_manager=model_variant_name_manager,
+        )
+
+    @staticmethod
+    def _create_optuna_plus_concurrency_sweep_run_config_generator(
+        command_config: ConfigCommandProfile,
+        state_manager: AnalyzerStateManager,
+        gpu_count: int,
+        models: List[ModelProfileSpec],
+        composing_models: List[ModelProfileSpec],
+        result_manager: ResultManager,
+        model_variant_name_manager: ModelVariantNameManager,
+        search_parameters: Dict[str, SearchParameters],
+        composing_search_parameters: Dict[str, SearchParameters],
+    ) -> ConfigGeneratorInterface:
+        return OptunaPlusConcurrencySweepRunConfigGenerator(
+            config=command_config,
+            state_manager=state_manager,
+            gpu_count=gpu_count,
+            composing_models=composing_models,
+            models=models,
+            result_manager=result_manager,
+            model_variant_name_manager=model_variant_name_manager,
+            search_parameters=search_parameters,
+            composing_search_parameters=composing_search_parameters,
+        )
+
+    @staticmethod
+    def _create_quick_plus_concurrency_sweep_run_config_generator(
+        command_config: ConfigCommandProfile,
+        gpu_count: int,
+        models: List[ModelProfileSpec],
+        composing_models: List[ModelProfileSpec],
+        result_manager: ResultManager,
+        model_variant_name_manager: ModelVariantNameManager,
+    ) -> ConfigGeneratorInterface:
+        search_config = RunConfigGeneratorFactory._create_search_config(
+            models, composing_models
+        )
+        return QuickPlusConcurrencySweepRunConfigGenerator(
+            search_config=search_config,
+            config=command_config,
+            gpu_count=gpu_count,
+            models=models,
+            composing_models=composing_models,
+            result_manager=result_manager,
+            model_variant_name_manager=model_variant_name_manager,
+        )
+
+    @staticmethod
+    def _create_search_config(
+        models: List[ModelProfileSpec], composing_models: List[ModelProfileSpec]
+    ) -> SearchConfig:
+        dimensions = SearchDimensions()
+
+        index = 0
+        all_models = models + composing_models
+
+        for model in all_models:
+            # Top level ensemble models don't have any dimensions
+            if model.is_ensemble():
+                continue
+
+            dims = RunConfigGeneratorFactory._get_dimensions_for_model(
+                model.supports_batching()
+            )
+            dimensions.add_dimensions(index, dims)
+            index += 1
+
+        search_config = SearchConfig(
+            dimensions=dimensions, radius=RADIUS, min_initialized=MIN_INITIALIZED
+        )
+
+        return search_config
+
+    @staticmethod
+    def _get_dimensions_for_model(is_batching_supported: bool) -> List[SearchDimension]:
+        if is_batching_supported:
+            return RunConfigGeneratorFactory._get_batching_supported_dimensions()
+        else:
+            return RunConfigGeneratorFactory._get_batching_not_supported_dimensions()
+
+    @staticmethod
+    def _get_batching_supported_dimensions() -> List[SearchDimension]:
+        return [
+            SearchDimension(
+                f"max_batch_size", SearchDimension.DIMENSION_TYPE_EXPONENTIAL
+            ),
+            SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_LINEAR),
+        ]
+
+    @staticmethod
+    def _get_batching_not_supported_dimensions() -> List[SearchDimension]:
+        return [
+            SearchDimension(f"instance_count", SearchDimension.DIMENSION_TYPE_LINEAR)
+        ]
+
+    @staticmethod
+    def _create_composing_models(
+        models: List[ModelProfileSpec],
+        config: ConfigCommandProfile,
+        client: TritonClient,
+        gpus: List[GPUDevice],
+    ) -> List[ModelProfileSpec]:
+        """
+        Given a list of models create a list of all the composing models (BLS + Ensemble)
+        """
+        composing_models = RunConfigGeneratorFactory._create_bls_composing_models(
+            config, client, gpus
+        )
+
+        for model in models:
+            composing_models.extend(
+                RunConfigGeneratorFactory._create_ensemble_composing_models(
+                    model, config, client, gpus
+                )
+            )
+
+        for composing_model in composing_models:
+            if composing_model.is_ensemble():
+                raise TritonModelAnalyzerException(
+                    f"Model Analyzer does not support ensembles as a composing model type: {composing_model.model_name()}"
+                )
+
+        return composing_models
+
+    @staticmethod
+    def _create_bls_composing_models(
+        config: ConfigCommandProfile, client: TritonClient, gpus: List[GPUDevice]
+    ) -> List[ModelProfileSpec]:
+        """
+        Creates a list of BLS composing model configs based on the profile command config
+        """
+        bls_composing_model_configs = [
+            ModelProfileSpec(bls_composing_model_spec, config, client, gpus)
+            for bls_composing_model_spec in config.bls_composing_models
+        ]
+
+        return bls_composing_model_configs
+
+    @staticmethod
+    def _create_ensemble_composing_models(
+        model: ModelProfileSpec,
+        config: ConfigCommandProfile,
+        client: TritonClient,
+        gpus: List[GPUDevice],
+    ) -> List[ModelProfileSpec]:
+        """
+        Creates a list of Ensemble composing model configs based on the model
+        """
+        model_config = ModelConfig.create_from_profile_spec(model, config, client, gpus)
+
+        if not model_config.is_ensemble():
+            return []
+
+        ensemble_composing_model_names = model_config.get_ensemble_composing_models()
+
+        ensemble_composing_model_specs = (
+            ConfigModelProfileSpec.model_list_to_config_model_profile_spec(
+                ensemble_composing_model_names
+            )
+        )
+
+        ensemble_composing_model_configs = [
+            ModelProfileSpec(ensemble_composing_model_spec, config, client, gpus)
+            for ensemble_composing_model_spec in ensemble_composing_model_specs
+        ]
+
+        return ensemble_composing_model_configs

model_analyzer/config/generate/search_config.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional
+
+from model_analyzer.config.generate.search_dimension import SearchDimension
+
+from .search_dimensions import SearchDimensions
+
+
+class NeighborhoodConfig:
+    """
+    Defines the configuration for a Neighborhood object
+    """
+
+    def __init__(self, dimensions: SearchDimensions, radius: int, min_initialized: int):
+        """
+        Parameters
+        ----------
+        dimensions: SearchDimensions
+        radius: int
+            All points within distance=radius from a location will be in
+            its neighborhood
+        min_initialized: int
+            Minimum number of initialized values in a neighborhood
+            before a step can be taken
+        """
+        self._dimensions = dimensions
+        self._radius = radius
+        self._min_initialized = min_initialized
+
+    def get_num_dimensions(self) -> int:
+        """Returns the number of dimensions in this search"""
+        return len(self._dimensions)
+
+    def get_dimensions(self) -> SearchDimensions:
+        """Returns the SearchDimensions"""
+        return self._dimensions
+
+    def get_dimension(self, idx: int) -> SearchDimension:
+        """Returns the SearchDimension at the given index"""
+        return self._dimensions[idx]
+
+    def get_min_indexes(self) -> List[int]:
+        """
+        Returns a list corresponding to the minimum index of all SearchDimensions
+        """
+        min_indexes = []
+        for dimension in self._dimensions:
+            min_indexes.append(dimension.get_min_idx())
+        return min_indexes
+
+    def get_min_initialized(self) -> int:
+        """
+        Returns the minimum number of initialized coordinates needed
+        in a neighborhood before a step can be taken
+        """
+        return self._min_initialized
+
+    def get_radius(self) -> int:
+        """Returns the base radius of a neighborhood"""
+        return self._radius
+
+
+class SearchConfig(NeighborhoodConfig):
+    """
+    Defines all dimensions to search
+    """
+
+    def __init__(self, dimensions: SearchDimensions, radius: int, min_initialized: int):
+        """
+        Parameters
+        ----------
+        dimensions: SearchDimensions
+        radius: int
+            All points within distance=radius from a location will be in
+            each neighborhood
+        min_initialized: int
+            Minimum number of initialized values in a neighborhood
+            before a step can be taken
+
+        """
+        super().__init__(
+            dimensions=dimensions, radius=radius, min_initialized=min_initialized
+        )
+
+    def get_neighborhood_config(
+        self, radius: Optional[int] = None
+    ) -> NeighborhoodConfig:
+        """
+        Return a NeighborhoodConfig with an optional override to the radius
+        """
+        radius_to_use = radius if radius is not None else self.get_radius()
+
+        return NeighborhoodConfig(
+            dimensions=self._dimensions,
+            radius=radius_to_use,
+            min_initialized=self._min_initialized,
+        )

model_analyzer/config/generate/search_dimension.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import sys
+
+
+class SearchDimension:
+    """
+    Defines a single dimension to search, and how the values
+    of that dimension grow
+    """
+
+    DIMENSION_TYPE_LINEAR = 0
+    DIMENSION_TYPE_EXPONENTIAL = 1
+    DIMENSION_NO_MAX = sys.maxsize
+
+    def __init__(self, name: str, type: int, min: int = 0, max: int = DIMENSION_NO_MAX):
+        """
+        Parameters
+        ----------
+        name: str
+        type: enum
+            Enum indicating how the values of this dimension grow
+        min: int
+            The minimum index for this search dimension. If unspecified, min is 0
+        max: int
+            The maximum index for this search dimension. If unspecified, then there is no max
+
+        """
+        self._name = name
+        self._type = type
+        self._min = min
+        self._max = max
+
+    def get_min_idx(self) -> int:
+        """Return the minimum index for this dimension"""
+        return self._min
+
+    def get_max_idx(self) -> int:
+        """Return the maximum index for this dimension"""
+        return self._max
+
+    def get_name(self) -> str:
+        """Return the name for this dimension"""
+        return self._name
+
+    def get_value_at_idx(self, idx: int) -> int:
+        """Return the value of the dimension at the given index"""
+        if idx < self._min or idx > self._max:
+            raise IndexError(
+                f"Index {idx} is out of range for search dimension {self._name}"
+            )
+
+        if self._type == SearchDimension.DIMENSION_TYPE_LINEAR:
+            return idx + 1
+        elif self._type == SearchDimension.DIMENSION_TYPE_EXPONENTIAL:
+            return int(math.pow(2, idx))
+        else:
+            raise Exception(f"Unknown type {self._type}")

model_analyzer/config/generate/search_dimensions.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Iterator, List
+
+from .coordinate import Coordinate
+from .search_dimension import SearchDimension
+
+
+class SearchDimensions:
+    """
+    Data class that holds one or more dimensions and associates each one to a key
+    """
+
+    def __init__(self) -> None:
+        self._dimensions: List[SearchDimension] = []
+        self._dimension_keys: List[Any] = []
+
+    def add_dimensions(self, key: Any, dimensions: List[SearchDimension]) -> None:
+        """
+        Add dimensions and associate them all with the given key
+
+        Parameters
+        ----------
+        Key: int
+            The key to associate the dimensions with
+
+        Dimensions: List of SearchDimension
+            Dimensions to add and associate with the key
+        """
+        for dim in dimensions:
+            self._dimensions.append(dim)
+            self._dimension_keys.append(key)
+
+    def get_values_for_coordinate(
+        self, coordinate: Coordinate
+    ) -> Dict[Any, Dict[str, Any]]:
+        """
+        Given a Coordinate, return all dimension_name:dimension_value pairs associated with
+        that coordinate, organized by the dimension's key
+
+        Parameters
+        ----------
+
+        coordinate: Coordinate
+            The coordinate to get values for
+
+        Returns: Dict of Dicts
+            ret[key][SearchDimension name] = value
+
+        """
+        vals: Dict[Any, Dict[str, Any]] = {}
+        for i, v in enumerate(coordinate):
+            key = self._dimension_keys[i]
+            if key not in vals:
+                vals[key] = {}
+
+            dim = self._dimensions[i]
+            name = dim.get_name()
+            val = dim.get_value_at_idx(v)
+            vals[key][name] = val
+
+        return vals
+
+    def __iter__(self) -> Iterator:
+        return iter(self._dimensions)
+
+    def __len__(self) -> int:
+        return len(self._dimensions)
+
+    def __getitem__(self, index: int) -> SearchDimension:
+        return self._dimensions[index]

model_analyzer/config/generate/search_parameter.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any, List, Optional
+
+
+class ParameterUsage(Enum):
+    MODEL = auto()
+    RUNTIME = auto()
+    BUILD = auto()
+
+
+class ParameterCategory(Enum):
+    INTEGER = auto()
+    EXPONENTIAL = auto()
+    STR_LIST = auto()
+    INT_LIST = auto()
+
+
+@dataclass
+class SearchParameter:
+    """
+    A dataclass that holds information about a configuration's search parameter
+    """
+
+    usage: ParameterUsage
+    category: ParameterCategory
+
+    # This is only applicable to the LIST categories
+    enumerated_list: Optional[List[Any]] = None
+
+    # These are only applicable to INTEGER and EXPONENTIAL categories
+    min_range: Optional[int] = None
+    max_range: Optional[int] = None