triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/config/generate/brute_run_config_generator.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Generator, List, Optional
+
+from model_analyzer.config.generate.model_profile_spec import ModelProfileSpec
+from model_analyzer.config.generate.model_run_config_generator import (
+    ModelRunConfigGenerator,
+)
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.run.model_run_config import ModelRunConfig
+from model_analyzer.config.run.run_config import RunConfig
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+from model_analyzer.triton.client.client import TritonClient
+
+from .config_generator_interface import ConfigGeneratorInterface
+
+
+class BruteRunConfigGenerator(ConfigGeneratorInterface):
+    """
+    Generates all RunConfigs to execute via brute force given a list of models
+    """
+
+    def __init__(
+        self,
+        config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        models: List[ModelProfileSpec],
+        client: TritonClient,
+        model_variant_name_manager: ModelVariantNameManager,
+        skip_default_config: bool = False,
+    ):
+        """
+        Parameters
+        ----------
+        config: ModelAnalyzerConfig
+
+        gpus: List of GPUDevices
+
+        models: List of ConfigModelProfileSpec
+            The models to generate ModelRunConfigs for
+
+        client: TritonClient
+
+        model_variant_name_manager: ModelVariantNameManager
+
+        skip_default_config: bool
+        """
+        self._config = config
+        self._gpus = gpus
+        self._models = models
+        self._client = client
+        self._model_variant_name_manager = model_variant_name_manager
+
+        self._triton_env = BruteRunConfigGenerator.determine_triton_server_env(models)
+
+        self._num_models = len(models)
+
+        self._curr_model_run_configs: List[Optional[ModelRunConfig]] = [
+            None for n in range(self._num_models)
+        ]
+        self._curr_results: List = [[] for n in range(self._num_models)]
+        self._curr_generators: Dict[int, ConfigGeneratorInterface] = {}
+
+        self._skip_default_config = skip_default_config
+
+    def set_last_results(
+        self, measurements: List[Optional[RunConfigMeasurement]]
+    ) -> None:
+        for index in range(self._num_models):
+            self._curr_results[index].extend(measurements)
+
+    def get_configs(self) -> Generator[RunConfig, None, None]:
+        """
+        Returns
+        -------
+        RunConfig
+            The next RunConfig generated by this class
+        """
+
+        yield from self._get_next_config()
+
+    def _get_next_config(self) -> Generator[RunConfig, None, None]:
+        if not self._skip_default_config:
+            yield from self._generate_subset(0, default_only=True)
+
+        yield from self._generate_subset(0, default_only=False)
+
+    def _generate_subset(
+        self, index: int, default_only: bool
+    ) -> Generator[RunConfig, None, None]:
+        mrcg = ModelRunConfigGenerator(
+            self._config,
+            self._gpus,
+            self._models[index],
+            self._client,
+            self._model_variant_name_manager,
+            default_only,
+        )
+
+        self._curr_generators[index] = mrcg
+
+        for model_run_config in mrcg.get_configs():
+            self._curr_model_run_configs[index] = model_run_config
+
+            if index == (len(self._models) - 1):
+                yield (self._make_run_config())
+            else:
+                yield from self._generate_subset(index + 1, default_only)
+
+            self._send_results_to_generator(index)
+
+    def _make_run_config(self) -> RunConfig:
+        run_config = RunConfig(self._triton_env, self._models[0].genai_perf_flags())
+        for index in range(len(self._models)):
+            run_config.add_model_run_config(self._curr_model_run_configs[index])
+        return run_config
+
+    def _send_results_to_generator(self, index: int) -> None:
+        self._curr_generators[index].set_last_results(self._curr_results[index])
+        self._curr_results[index] = []
+
+    @classmethod
+    def determine_triton_server_env(cls, models: List[ModelProfileSpec]) -> Dict:
+        """
+        Given a list of models, return the triton environment
+        """
+        triton_env = models[0].triton_server_environment()
+
+        for model in models:
+            if model.triton_server_environment() != triton_env:
+                raise TritonModelAnalyzerException(
+                    f"Mismatching triton server environments. The triton server environment must be the same for all models when run concurrently"
+                )
+
+        return triton_env

model_analyzer/config/generate/concurrency_sweeper.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from copy import deepcopy
+from typing import Generator, List, Optional
+
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.run.run_config import RunConfig
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.result.parameter_search import ParameterSearch
+from model_analyzer.result.result_manager import ResultManager
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class ConcurrencySweeper:
+    """
+    Sweeps concurrency for the top-N model configs
+    """
+
+    def __init__(
+        self,
+        config: ConfigCommandProfile,
+        result_manager: ResultManager,
+    ):
+        self._config = config
+        self._result_manager = result_manager
+        self._last_measurement: Optional[RunConfigMeasurement] = None
+
+    def set_last_results(
+        self, measurements: List[Optional[RunConfigMeasurement]]
+    ) -> None:
+        self._last_measurement = measurements[-1]
+
+    def get_configs(self) -> Generator[RunConfig, None, None]:
+        """
+        A generator which creates RunConfigs based on sweeping
+        concurrency over the top-N models
+        """
+        for model_name in self._result_manager.get_model_names():
+            top_results = self._result_manager.top_n_results(
+                model_name=model_name,
+                n=self._config.num_configs_per_model,
+                include_default=True,
+            )
+
+            for result in top_results:
+                run_config = deepcopy(result.run_config())
+                parameter_search = ParameterSearch(self._config)
+                for concurrency in parameter_search.search_parameters():
+                    run_config = self._create_run_config(run_config, concurrency)
+                    yield run_config
+                    parameter_search.add_run_config_measurement(self._last_measurement)
+
+    def _create_run_config(self, run_config: RunConfig, concurrency: int) -> RunConfig:
+        for model_run_config in run_config.model_run_configs():
+            perf_config = model_run_config.perf_config()
+            perf_config.update_config({"concurrency-range": concurrency})
+
+        return run_config

model_analyzer/config/generate/config_generator_interface.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+from typing import Any, Generator, List, Optional
+
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+
+
+class ConfigGeneratorInterface(abc.ABC):
+    """
+    An interface class for config generators
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass: Any) -> bool:
+        return (
+            hasattr(subclass, "__init__")
+            and callable(subclass.__init__)
+            and hasattr(subclass, "get_configs")
+            and callable(subclass.get_configs)
+            and hasattr(subclass, "set_last_results")
+            and callable(subclass.set_last_results)
+            or NotImplemented
+        )
+
+    @abc.abstractmethod
+    def __init__(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_configs(self) -> Generator[Any, None, None]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def set_last_results(
+        self, measurements: List[Optional[RunConfigMeasurement]]
+    ) -> None:
+        raise NotImplementedError

model_analyzer/config/generate/coordinate.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from copy import deepcopy
+from functools import total_ordering
+from typing import Any, Iterator, List, Union
+
+
+@total_ordering
+class Coordinate:
+    """
+    Class to define a coordinate in n-dimension space
+    """
+
+    def __init__(self, val: Union["Coordinate", List[int]]):
+        """
+        val: list
+            List of floats or integers corresponding to the location in space
+        """
+        if isinstance(val, Coordinate):
+            val = val._values
+
+        self._values: List[int] = deepcopy(val)
+
+    def __getitem__(self, idx: int) -> int:
+        return self._values[idx]
+
+    def __setitem__(self, idx: int, item: int) -> None:
+        self._values[idx] = item
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    def __add__(self, other: Any) -> "Coordinate":
+        if type(other) == Coordinate:
+            return self._add_coordinate(other)
+        elif type(other) == int or type(other) == float:
+            return self._add_number(other)
+        else:
+            raise Exception("Unhandled addition type")
+
+    def __sub__(self, other: Any) -> "Coordinate":
+        if type(other) == Coordinate:
+            return self._sub_coordinate(other)
+        elif type(other) == int or type(other) == float:
+            return self._sub_number(other)
+        else:
+            raise Exception("Unhandled subtraction type")
+
+    def __truediv__(self, other: Any) -> "Coordinate":
+        if type(other) == int or type(other) == float:
+            return self._div_number(other)
+        else:
+            raise Exception("Unhandled division type")
+
+    def __mul__(self, other: Any) -> "Coordinate":
+        if type(other) == int or type(other) == float:
+            return self._mul_number(other)
+        else:
+            raise Exception("Unhandled mul type")
+
+    def __eq__(self, other: Any) -> bool:
+        for i, v in enumerate(self._values):
+            if v != other[i]:
+                return False
+        return True
+
+    def __lt__(self, other: Any) -> bool:
+        for i, v in enumerate(self._values):
+            if v != other[i]:
+                return v < other[i]
+        return False
+
+    def round(self) -> None:
+        """Rounds the coordinate in-place"""
+        for i, _ in enumerate(self._values):
+            self._values[i] = round(self._values[i])
+
+    def _add_coordinate(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v + other[i]
+        return ret
+
+    def _add_number(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v + other
+        return ret
+
+    def _sub_coordinate(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v - other[i]
+        return ret
+
+    def _sub_number(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v - other
+        return ret
+
+    def _mul_number(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v * other
+        return ret
+
+    def _div_number(self, other: Any) -> "Coordinate":
+        ret = Coordinate(self._values)
+        for i, v in enumerate(self._values):
+            ret[i] = v / other
+        return ret
+
+    def __iter__(self) -> Iterator:
+        self._idx = 0
+        return self
+
+    def __next__(self) -> int:
+        if self._idx < len(self._values):
+            val = self._values[self._idx]
+            self._idx += 1
+            return val
+        raise StopIteration
+
+    def __str__(self) -> str:
+        return str(self._values)
+
+    def __repr__(self) -> str:
+        return repr(f"Coordinate({self._values})")

model_analyzer/config/generate/coordinate_data.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Optional, Tuple
+
+from model_analyzer.config.generate.coordinate import Coordinate
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+
+CoordinateKey = Tuple[Coordinate, ...]
+
+
+class CoordinateData:
+    """
+    A class that tracks the measurement data in the current neighborhood
+    and the visit counts of all the coordinates in the coordinate space.
+    """
+
+    def __init__(self) -> None:
+        self._measurements: Dict[CoordinateKey, Optional[RunConfigMeasurement]] = {}
+        self._visit_counts: Dict[CoordinateKey, int] = {}
+        self._is_measured: Dict[CoordinateKey, bool] = {}
+
+    def get_measurement(self, coordinate: Coordinate) -> Optional[RunConfigMeasurement]:
+        """
+        Return the measurement data of the given coordinate.
+        """
+        key: CoordinateKey = tuple(coordinate)
+        return self._measurements.get(key, None)
+
+    def set_measurement(
+        self, coordinate: Coordinate, measurement: Optional[RunConfigMeasurement]
+    ) -> None:
+        """
+        Set the measurement for the given coordinate.
+        """
+        key: CoordinateKey = tuple(coordinate)
+        self._measurements[key] = measurement
+        self._is_measured[key] = True
+
+    def is_measured(self, coordinate: Coordinate) -> bool:
+        """
+        Returns true if a measurement has been set for the given Coordinate
+        """
+        key: CoordinateKey = tuple(coordinate)
+        return self._is_measured.get(key, False)
+
+    def has_valid_measurement(self, coordinate: Coordinate) -> bool:
+        """
+        Returns true if there is a valid measurement for the given Coordinate
+        """
+        return self.get_measurement(coordinate) is not None
+
+    def reset_measurements(self) -> None:
+        """
+        Resets the collection of measurements.
+        """
+        self._measurements = {}
+
+    def get_visit_count(self, coordinate: Coordinate) -> int:
+        """
+        Get the visit count for the given coordinate.
+        Returns 0 if the coordinate hasn't been visited yet
+        """
+        key: CoordinateKey = tuple(coordinate)
+        return self._visit_counts.get(key, 0)
+
+    def increment_visit_count(self, coordinate: Coordinate) -> None:
+        """
+        Increase the visit count for the given coordinate by 1
+        """
+        key: CoordinateKey = tuple(coordinate)
+        new_count = self.get_visit_count(coordinate) + 1
+        self._visit_counts[key] = new_count

model_analyzer/config/generate/generator_utils.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from itertools import product
+from typing import Dict, List
+
+
+class GeneratorUtils:
+    """Class for utility functions for Generators"""
+
+    @staticmethod
+    def generate_combinations(value: object) -> List:
+        """
+        Generates all the alternative fields for
+        a given value.
+
+        Parameters
+        ----------
+        value : object
+            The value to be used for sweeping.
+
+        Returns
+        -------
+        list
+            A list of all the alternatives for the parameters.
+        """
+
+        if type(value) is dict:
+            sweeped_dict = {}
+            for key, sweep_choices in value.items():
+                sweep_parameter_list = []
+
+                # This is the list of sweep parameters. When parsing a
+                # config every sweepable parameter will be converted
+                # to a list of values to make the parameter sweeping easier in
+                # here.
+                for sweep_choice in sweep_choices:
+                    sweep_parameter_list += GeneratorUtils.generate_combinations(
+                        sweep_choice
+                    )
+
+                sweeped_dict[key] = sweep_parameter_list
+
+            # Generate parameter combinations for this field.
+            return GeneratorUtils.generate_parameter_combinations(sweeped_dict)
+
+        # When this line of code is executed the value for this field is
+        # a list. This list does NOT represent possible sweep values.
+        # Because of this we need to ensure that in every sweep configuration,
+        # one item from every list item exists.
+        elif type(value) is list:
+            # This list contains a set of lists. The return value from this
+            # branch of the code is a list of lists where in each inner list
+            # there is one item from every list item.
+            sweep_parameter_list = []
+            for item in value:
+                sweep_parameter_list_item = GeneratorUtils.generate_combinations(item)
+                sweep_parameter_list.append(sweep_parameter_list_item)
+
+            # Cartesian product of all the elements in the sweep_parameter_list
+            return [list(x) for x in list(product(*sweep_parameter_list))]
+
+        # In the default case return a list of the value. This function should
+        # always return a list.
+        return [value]
+
+    @staticmethod
+    def generate_parameter_combinations(params: Dict) -> List[Dict]:
+        """
+        Generate a list of all possible subdictionaries
+        from given dictionary. The subdictionaries will
+        have all the same keys, but only one value from
+        each key.
+
+        Parameters
+        ----------
+        params : dict
+            keys are strings and the values must be lists
+        """
+
+        param_combinations = list(product(*tuple(params.values())))
+        return [dict(zip(params.keys(), vals)) for vals in param_combinations]
+
+    @staticmethod
+    def generate_doubled_list(min_value: int, max_value: int) -> List[int]:
+        """
+        Generates a list of values from min_value -> max_value doubling
+        min_value for each entry
+
+        Parameters
+        ----------
+        min_value: int
+            The minimum value for the generated list
+        max_value : int
+            The value that the generated list will not exceed
+        """
+
+        list = []
+        val = 1 if min_value == 0 else min_value
+        while val <= max_value:
+            list.append(val)
+            val *= 2
+        return list