triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/result/run_config_result.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from bisect import insort
+from functools import total_ordering
+from typing import List
+
+from model_analyzer.config.run.run_config import RunConfig
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.result.constraint_manager import ConstraintManager
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+from model_analyzer.result.run_config_result_comparator import RunConfigResultComparator
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+@total_ordering
+class RunConfigResult:
+    """
+    A class that represents the group of measurements (result) from
+    a single RunConfig. This RunConfigResult belongs
+    to a particular ResultTable
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        run_config: RunConfig,
+        comparator: RunConfigResultComparator,
+        constraint_manager: ConstraintManager,
+    ):
+        """
+        Parameters
+        ----------
+        model_name: str
+            The name of the model
+        run_config : RunConfig
+        comparator : RunConfigResultComparator
+            An object whose compare function receives two
+            RunConfigResults and returns 1 if the first is better than
+            the second, 0 if they are equal and -1
+            otherwise
+        constraint_manager: ConstraintManager
+            The object that handles processing and applying
+            constraints on a given measurements
+        """
+
+        self._model_name = model_name
+        self._run_config = run_config
+        self._comparator = comparator
+        self._constraint_manager = constraint_manager
+
+        # Heaps
+        self._measurements: List[RunConfigMeasurement] = []
+        self._passing_measurements: List[RunConfigMeasurement] = []
+        self._failing_measurements: List[RunConfigMeasurement] = []
+
+    def model_name(self):
+        """
+        Returns
+        -------
+        str
+            Returns the name of the model corresponding to this RunConfigResult
+        """
+
+        return self._model_name
+
+    def run_config(self):
+        """
+        Returns
+        -------
+        RunConfig
+            associated with this result
+        """
+        return self._run_config
+
+    def failing(self):
+        """
+        Returns
+        -------
+        bool
+            Returns true if there are no passing RunConfigMeasurements
+        """
+
+        if not self._passing_measurements:
+            return True
+        return False
+
+    def add_run_config_measurement(self, run_config_measurement):
+        """
+        This function checks whether a RunConfigMeasurement
+        passes the constraints and adds the measurements to
+        the corresponding heap
+
+        Parameters
+        ----------
+        run_config_measurement : RunConfigMeasurement
+            The profiled RunConfigMeasurement
+        """
+
+        insort(self._measurements, run_config_measurement)
+
+        if self._constraint_manager.satisfies_constraints(run_config_measurement):
+            insort(self._passing_measurements, run_config_measurement)
+        else:
+            insort(self._failing_measurements, run_config_measurement)
+
+    def run_config_measurements(self):
+        """
+        Returns
+        -------
+        list
+            of RunConfigMeasurements in this RunConfigResult
+        """
+        return [measurement for measurement in reversed(self._measurements)]
+
+    def passing_measurements(self):
+        """
+        Returns
+        -------
+        list
+            of passing measurements in this RunConfigResult
+        """
+
+        return [
+            passing_measurement
+            for passing_measurement in reversed(self._passing_measurements)
+        ]
+
+    def failing_measurements(self):
+        """
+        Returns
+        -------
+        list
+            of failing measurements in this RunConfigResult
+        """
+
+        return [
+            failing_measurement
+            for failing_measurement in reversed(self._failing_measurements)
+        ]
+
+    def top_n_measurements(self, n):
+        """
+        Parameters
+        ----------
+        n : int
+            The number of top RunConfigMeasurements
+            to retrieve
+
+        Returns
+        -------
+        list of RunConfigMeasurements
+            The top n RunConfigMeasurements
+        """
+
+        if len(self._passing_measurements) == 0:
+            logger.warning(
+                f"Requested top {n} RunConfigMeasurements, but none satisfied constraints. "
+                "Showing available constraint failing measurements for this config."
+            )
+
+            if n > len(self._failing_measurements):
+                logger.warning(
+                    f"Requested top {n} failing RunConfigMeasurements, "
+                    f"but found only {len(self._failing_measurements)}. "
+                    "Showing all available constraint failing measurements for this config."
+                )
+
+            return [
+                failing_measurement
+                for failing_measurement in reversed(self._failing_measurements[-n:])
+            ]
+
+        if n > len(self._passing_measurements):
+            logger.warning(
+                f"Requested top {n} RunConfigMeasurements, but "
+                f"found only {len(self._passing_measurements)}. "
+                "Showing all available measurements for this config."
+            )
+
+        return [
+            passing_measurement
+            for passing_measurement in reversed(self._passing_measurements[-n:])
+        ]
+
+    def __lt__(self, other):
+        """
+        Checks whether this RunConfigResult is better
+        than other
+
+        If True, this means this RunConfigResult is better
+        than the other.
+        """
+
+        return self._comparator.is_better_than(self, other)

model_analyzer/result/run_config_result_comparator.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List
+
+
+class RunConfigResultComparator:
+    """
+    Stores information needed to compare two RunConfigResults.
+    """
+
+    def __init__(
+        self, metric_objectives_list: List[Dict[str, int]], model_weights: List[int]
+    ):
+        """
+        Parameters
+        ----------
+        List of
+            metric_objectives : dict of RecordTypes
+                keys are the metric types, and values are The relative importance
+                of the keys with respect to other. If the values are 0,
+        """
+
+        # Normalize metric weights
+        self._metric_weights = []
+        self._model_weights = []
+        for i, metric_objectives in enumerate(metric_objectives_list):
+            self._metric_weights.append(
+                {
+                    key: (val / sum(metric_objectives.values()))
+                    for key, val in metric_objectives.items()
+                }
+            )
+
+            self._model_weights.append(model_weights[i])
+
+    def get_metric_weights(self):
+        return self._metric_weights
+
+    def get_model_weights(self):
+        return self._model_weights
+
+    def is_better_than(self, run_config_result1, run_config_result2):
+        """
+        Aggregates and compares the score for two RunConfigResults
+
+        Parameters
+        ----------
+        run_config_result1 : RunConfigResult
+            first result to be compared
+        run_config_result2 : RunConfigResult
+            second result to be compared
+
+        Returns
+        -------
+        bool
+           True: if result1 is better than result2
+        """
+
+        agg_run_config_measurement1 = self._aggregate_run_config_measurements(
+            run_config_result1, aggregation_func=max
+        )
+        agg_run_config_measurement2 = self._aggregate_run_config_measurements(
+            run_config_result2, aggregation_func=max
+        )
+
+        return agg_run_config_measurement1.is_better_than(agg_run_config_measurement2)
+
+    def _aggregate_run_config_measurements(self, run_config_result, aggregation_func):
+        """
+        Returns
+        -------
+        (list, list)
+            A 2-tuple of average RunConfigMeasurements,
+            The first is across non-gpu specific metrics
+            The second is across gpu-specific measurements
+        """
+
+        # For the gpu_measurements we have a list of dicts of lists
+        # Assumption here is that its okay to average over all GPUs over all perf runs
+        # This is done within the measurement itself
+
+        if run_config_result.passing_measurements():
+            aggregated_run_config_measurement = aggregation_func(
+                run_config_result.passing_measurements()
+            )
+        else:
+            aggregated_run_config_measurement = aggregation_func(
+                run_config_result.run_config_measurements()
+            )
+
+        aggregated_run_config_measurement.set_model_config_weighting(
+            self._model_weights
+        )
+        aggregated_run_config_measurement.set_metric_weightings(self._metric_weights)
+
+        return aggregated_run_config_measurement

model_analyzer/result/sorted_results.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from copy import deepcopy
+from typing import List, Optional
+
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.result.run_config_result import RunConfigResult
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class SortedResults:
+    """
+    A data structure used by the result manager
+    to store and sort results
+    """
+
+    GET_ALL_RESULTS = -1
+
+    def __init__(self) -> None:
+        self._run_config_results: List[RunConfigResult] = []
+
+    def results(self) -> List[RunConfigResult]:
+        """
+        Returns
+        -------
+        All the results
+        """
+
+        self._run_config_results.sort()
+        return self._run_config_results
+
+    def add_result(self, run_config_result: RunConfigResult) -> None:
+        """
+        Adds a run_config_result to the result lists
+        This can either be a new result or new measurements added
+        to an existing result
+
+        Parameters
+        ----------
+        result: ModelResult
+            The result to be added
+        """
+
+        existing_run_config_result = self._find_existing_run_config_result(
+            run_config_result
+        )
+
+        if existing_run_config_result:
+            self._add_measurements_to_existing_run_config_result(
+                existing_run_config_result, run_config_result
+            )
+        else:
+            self._add_new_run_config_result(run_config_result)
+
+    def top_n_results(self, n: int) -> List[RunConfigResult]:
+        """
+        Parameters
+        ----------
+        n : int
+            The number of  top results
+            to retrieve
+
+        Returns
+        -------
+        list of RunConfigResults
+            The n best results for this model,
+            must all be passing results
+        """
+        passing_results, failing_results = self._create_passing_and_failing_lists()
+
+        if len(passing_results) == 0:
+            logger.warning(
+                f"Requested top {n} configs, but none satisfied constraints. "
+                "Showing available constraint failing configs for this model."
+            )
+
+            return self._get_top_n_results(failing_results, n)
+        else:
+            return self._get_top_n_results(passing_results, n)
+
+    def _find_existing_run_config_result(
+        self, run_config_result: RunConfigResult
+    ) -> Optional[RunConfigResult]:
+        if not run_config_result.run_config():
+            return None
+
+        for rcr in self._run_config_results:
+            if (
+                run_config_result.run_config().model_variants_name()
+                == rcr.run_config().model_variants_name()
+            ):
+                return rcr
+
+        return None
+
+    def _add_measurements_to_existing_run_config_result(
+        self,
+        existing_run_config_result: RunConfigResult,
+        new_run_config_result: RunConfigResult,
+    ) -> None:
+        for rcm in new_run_config_result.run_config_measurements():
+            existing_run_config_result.add_run_config_measurement(rcm)
+
+    def _add_new_run_config_result(self, run_config_result: RunConfigResult) -> None:
+        new_run_config_result = deepcopy(run_config_result)
+
+        self._run_config_results.append(new_run_config_result)
+
+    def _create_passing_and_failing_lists(self):
+        self._run_config_results.sort()
+
+        passing = []
+        failing = []
+        for rcr in self._run_config_results:
+            if rcr.failing():
+                failing.append(rcr)
+            else:
+                passing.append(rcr)
+
+        return passing, failing
+
+    def _get_top_n_results(
+        self, results: List[RunConfigResult], n: int
+    ) -> List[RunConfigResult]:
+        if n == SortedResults.GET_ALL_RESULTS:
+            return results
+        if n > len(results):
+            logger.warning(
+                f"Requested top {n} configs, "
+                f"but found only {len(results)}. "
+                "Showing all available configs for this model."
+            )
+
+        result_len = min(n, len(results))
+        return results[0:result_len]

model_analyzer/state/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/state/analyzer_state.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.record.record import RecordType
+from model_analyzer.result.results import Results
+
+
+class AnalyzerState:
+    """
+    All the state information needed by
+    Model Analyzer in one place
+    """
+
+    def __init__(self):
+        self._state_dict = {}
+
+    def to_dict(self):
+        return self._state_dict
+
+    @classmethod
+    def from_dict(cls, state_dict):
+        state = AnalyzerState()
+
+        # Model Variant mapping
+        state._state_dict["ModelManager.model_variant_name_manager"] = state_dict[
+            "ModelManager.model_variant_name_manager"
+        ]
+
+        # Fill results
+        state._state_dict["ResultManager.results"] = Results.from_dict(
+            state_dict["ResultManager.results"]
+        )
+
+        # Server data
+        state._state_dict["ResultManager.server_only_data"] = {}
+        for gpu_uuid, gpu_data_list in state_dict[
+            "ResultManager.server_only_data"
+        ].items():
+            metric_list = []
+            for [tag, record_dict] in gpu_data_list:
+                record_type = RecordType.get(tag)
+                record = record_type.from_dict(record_dict)
+                metric_list.append(record)
+            state._state_dict["ResultManager.server_only_data"][gpu_uuid] = metric_list
+
+        # GPU data
+        state._state_dict["MetricsManager.gpus"] = state_dict["MetricsManager.gpus"]
+
+        # Optuna Seed
+        state._state_dict["OptunaRunConfigGenerator.seed"] = state_dict.get(
+            "OptunaRunConfigGenerator.seed", 0
+        )
+
+        return state
+
+    def get(self, name):
+        if name in self._state_dict:
+            return self._state_dict[name]
+        else:
+            return None
+
+    def set(self, name, value):
+        self._state_dict[name] = value