triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/model_manager.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Dict, List, Optional
+
+from model_analyzer.config.generate.model_variant_name_manager import (
+    ModelVariantNameManager,
+)
+from model_analyzer.config.generate.run_config_generator_factory import (
+    RunConfigGeneratorFactory,
+)
+from model_analyzer.config.generate.search_parameters import SearchParameters
+from model_analyzer.config.input.config_command_profile import ConfigCommandProfile
+from model_analyzer.config.input.objects.config_model_profile_spec import (
+    ConfigModelProfileSpec,
+)
+from model_analyzer.constants import INVALID_MEASUREMENT_THRESHOLD, LOGGER_NAME
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.record.metrics_manager import MetricsManager
+from model_analyzer.result.constraint_manager import ConstraintManager
+from model_analyzer.result.result_manager import ResultManager
+from model_analyzer.result.run_config_measurement import RunConfigMeasurement
+from model_analyzer.state.analyzer_state_manager import AnalyzerStateManager
+from model_analyzer.triton.client.client import TritonClient
+from model_analyzer.triton.model.model_config import ModelConfig
+from model_analyzer.triton.server.server import TritonServer
+
+from .model_analyzer_exceptions import TritonModelAnalyzerException
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class ModelManager:
+    """
+    This class handles the search for, creation of, and execution of run configs.
+    It also records the best results for each model.
+    """
+
+    def __init__(
+        self,
+        config: ConfigCommandProfile,
+        gpus: List[GPUDevice],
+        client: TritonClient,
+        server: TritonServer,
+        metrics_manager: MetricsManager,
+        result_manager: ResultManager,
+        state_manager: AnalyzerStateManager,
+        constraint_manager: ConstraintManager,
+        search_parameters: Dict[str, SearchParameters],
+        composing_search_parameters: Dict[str, SearchParameters],
+    ):
+        """
+        Parameters
+        ----------
+        config:ConfigCommandProfile
+            The config for the model analyzer
+        gpus: List of GPUDevice
+        client: TritonClient
+            The client handle used to send requests to Triton
+        server: TritonServer
+            The server handle used to start and stop Triton instances
+        metrics_manager: MetricsManager
+            The object that handles launching perf analyzer instances and profiling.
+        result_manager: ResultManager
+            The object that handles storing and sorting the results from the perf analyzer
+        state_manager: AnalyzerStateManager
+            The object that handles serializing the state of the analyzer and saving.
+        constraint_manager: ConstraintManager
+            The object that handles processing and applying
+            constraints on a given measurements
+        search_parameters: SearchParameters
+            The object that handles the users configuration search parameters
+        composing_search_parameters: SearchParameters
+            The object that handles the users configuration search parameters for composing models
+        """
+
+        self._config = config
+        self._gpus = gpus
+        self._client = client
+        self._server = server
+        self._metrics_manager = metrics_manager
+        self._result_manager = result_manager
+        self._state_manager = state_manager
+        self._constraint_manager = constraint_manager
+        self._search_parameters = search_parameters
+        self._composing_search_parameters = composing_search_parameters
+
+        if state_manager.starting_fresh_run():
+            self._init_state()
+
+        self._failed_measurement_attempts = 0
+        self._received_measurement_values_from_pa = False
+
+        self._model_variant_name_manager = ModelVariantNameManager.from_dict(
+            self._state_manager.get_state_variable(
+                "ModelManager.model_variant_name_manager"
+            )
+        )
+
+    def run_models(self, models: List[ConfigModelProfileSpec]) -> None:
+        """
+        Generates configs, runs inferences, gets
+        measurements for a list of models
+
+        Parameters
+        ----------
+        models : List of ConfigModelProfileSpec
+            The models to run
+        """
+
+        # Note: this is not done in config_command, because there isn't a ModelConfig yet,
+        # so we cannot determine if the model is an ensemble
+        self._check_for_ensemble_model_incompatibility(models)
+
+        self._metrics_manager.start_new_model()
+
+        # Save the global server config and update the server's config for this model run
+        server_config_copy = self._server.config().copy()
+
+        triton_server_flags = self._get_triton_server_flags(models)
+        self._server.update_config(params=triton_server_flags)
+
+        rcg = RunConfigGeneratorFactory.create_run_config_generator(
+            command_config=self._config,
+            state_manager=self._state_manager,
+            gpus=self._gpus,
+            models=models,
+            client=self._client,
+            result_manager=self._result_manager,
+            search_parameters=self._search_parameters,
+            composing_search_parameters=self._composing_search_parameters,
+            model_variant_name_manager=self._model_variant_name_manager,
+        )
+
+        for run_config in rcg.get_configs():
+            if self._state_manager.exiting():
+                break
+
+            if run_config.is_legal_combination():
+                measurement = self._metrics_manager.execute_run_config(run_config)
+
+                self._check_for_valid_measurement(measurement)
+                self._stop_ma_if_no_valid_measurement_threshold_reached()
+            else:
+                logger.info("Skipping illegal run configuration")
+                measurement = None
+
+            if measurement:
+                objectives = [model.objectives() for model in models]
+                weightings = [model.weighting() for model in models]
+
+                measurement.set_metric_weightings(metric_objectives=objectives)
+                measurement.set_constraint_manager(
+                    constraint_manager=self._constraint_manager
+                )
+                measurement.set_model_config_weighting(model_config_weights=weightings)
+
+            rcg.set_last_results([measurement])
+            self._state_manager.save_checkpoint()
+
+        self._metrics_manager.finalize()
+
+        # Reset the server args to global config
+        self._server.update_config(params=server_config_copy.server_args())
+
+        model_variant_name_manager_dict = self._state_manager.default_encode(
+            self._model_variant_name_manager
+        )
+
+        self._state_manager.set_state_variable(
+            "ModelManager.model_variant_name_manager", model_variant_name_manager_dict
+        )
+
+    def _get_triton_server_flags(self, models):
+        triton_server_flags = models[0].triton_server_flags()
+
+        for model in models:
+            if model.triton_server_flags() != triton_server_flags:
+                raise TritonModelAnalyzerException(
+                    f"Triton server flags must be the same for all models to run concurrently"
+                )
+
+    def _check_for_ensemble_model_incompatibility(
+        self, models: List[ConfigModelProfileSpec]
+    ) -> None:
+        for model in models:
+            model_config = ModelConfig.create_from_profile_spec(
+                model, self._config, self._client, self._gpus
+            )
+
+            if model_config.is_ensemble():
+                if len(models) > 1:
+                    raise TritonModelAnalyzerException(
+                        f"\nProfiling of multiple models is not supported for ensemble models"
+                    )
+
+                if self._config.run_config_search_mode == "brute":
+                    if self._config.get_config()[
+                        "run_config_search_mode"
+                    ].is_set_by_user():
+                        raise TritonModelAnalyzerException(
+                            f"\nBrute search mode is not supported for ensemble models"
+                            "\nPlease use quick search mode (--run-config-search-mode quick)"
+                        )
+                    else:
+                        self._config.run_config_search_mode = "quick"
+            elif not self._config.bls_composing_models:
+                if len(self._config.cpu_only_composing_models) > 0:
+                    raise TritonModelAnalyzerException(
+                        f"\nCan only specify --cpu-only-composing-models for ensemble or BLS models."
+                    )
+
+    def _init_state(self):
+        """
+        Sets ModelManager object managed
+        state variables in AnalyzerState
+        """
+
+        self._state_manager.set_state_variable(
+            "ModelManager.model_variant_name_manager",
+            self._state_manager.default_encode(ModelVariantNameManager()),
+        )
+
+    def _check_for_valid_measurement(
+        self, measurement: Optional[RunConfigMeasurement]
+    ) -> None:
+        if measurement:
+            self._received_measurement_values_from_pa = True
+        else:
+            self._failed_measurement_attempts += 1
+
+    def _stop_ma_if_no_valid_measurement_threshold_reached(self) -> None:
+        if self._received_measurement_values_from_pa:
+            return
+
+        if self._failed_measurement_attempts >= INVALID_MEASUREMENT_THRESHOLD:
+            raise TritonModelAnalyzerException(
+                f"The first {INVALID_MEASUREMENT_THRESHOLD} attempts to acquire measurements "
+                "have failed. Please examine the Tritonserver/PA error logs "
+                "to determine what has gone wrong."
+            )

model_analyzer/monitor/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/monitor/cpu_monitor.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.record.types.cpu_available_ram import CPUAvailableRAM
+from model_analyzer.record.types.cpu_used_ram import CPUUsedRAM
+
+from .monitor import Monitor
+
+
+class CPUMonitor(Monitor):
+    """
+    A monitor for measuring the CPU usage of tritonserver during inference
+    """
+
+    cpu_metrics = {CPUAvailableRAM, CPUUsedRAM}
+
+    def __init__(self, server, frequency, metrics):
+        """
+        Parameters
+        ----------
+        server : TritonServer
+            A handle to the TritonServer
+        frequency : float
+            How often the metrics should be monitored.
+        metrics : list
+            A list of Record objects that will be monitored.
+        """
+
+        super().__init__(frequency, metrics)
+        self._cpu_memory_records = []
+        self._server = server
+
+    def is_monitoring_connected(self) -> bool:
+        return True
+
+    def _monitoring_iteration(self):
+        """
+        Get memory info of process and
+        append
+        """
+        if (CPUUsedRAM in self._metrics) or (CPUAvailableRAM in self._metrics):
+            used_mem, free_mem = self._server.cpu_stats()
+            if CPUUsedRAM in self._metrics:
+                self._cpu_memory_records.append(CPUUsedRAM(value=used_mem))
+            if CPUAvailableRAM in self._metrics:
+                self._cpu_memory_records.append(CPUAvailableRAM(value=free_mem))
+
+    def _collect_records(self):
+        """
+        Returns
+        -------
+        List of Records
+            the records metrics
+        """
+
+        return self._cpu_memory_records

model_analyzer/monitor/dcgm/DcgmDiag.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs
+import model_analyzer.monitor.dcgm.dcgm_agent as dcgm_agent
+
+
+class DcgmDiag:
+
+    # Maps version codes to simple version values for range comparisons
+    _versionMap = {dcgm_structs.dcgmRunDiag_version: 5}
+
+    def __init__(self,
+                 gpuIds=None,
+                 testNamesStr='',
+                 paramsStr='',
+                 verbose=True,
+                 version=dcgm_structs.dcgmRunDiag_version):
+        # Make sure version is valid
+        if version not in DcgmDiag._versionMap:
+            raise ValueError("'%s' is not a valid version for dcgmRunDiag." %
+                             version)
+        self.version = version
+
+        if self.version == dcgm_structs.dcgmRunDiag_version7:
+            self.runDiagInfo = dcgm_structs.c_dcgmRunDiag_v7()
+        else:
+            self.runDiagInfo = dcgm_structs.c_dcgmRunDiag_t()
+
+        self.numTests = 0
+        self.numParams = 0
+        self.SetVerbose(verbose)
+        if testNamesStr == '':
+            # default to a level 1 test
+            self.runDiagInfo.validate = 1
+        elif testNamesStr == '1':
+            self.runDiagInfo.validate = 1
+        elif testNamesStr == '2':
+            self.runDiagInfo.validate = 2
+        elif testNamesStr == '3':
+            self.runDiagInfo.validate = 3
+        elif testNamesStr == '4':
+            self.runDiagInfo.validate = 4
+        else:
+            # Make sure no number other that 1-4 were submitted
+            if testNamesStr.isdigit():
+                raise ValueError("'%s' is not a valid test name." %
+                                 testNamesStr)
+
+            # Copy to the testNames portion of the object
+            names = testNamesStr.split(',')
+            if len(names) > dcgm_structs.DCGM_MAX_TEST_NAMES:
+                err = 'DcgmDiag cannot initialize: %d test names were specified exceeding the limit of %d.' %\
+                      (len(names), dcgm_structs.DCGM_MAX_TEST_NAMES)
+                raise ValueError(err)
+
+            for testName in names:
+                self.AddTest(testName)
+
+        if paramsStr != '':
+            params = paramsStr.split(';')
+            if len(params) >= dcgm_structs.DCGM_MAX_TEST_PARMS:
+                err = 'DcgmDiag cannot initialize: %d parameters were specified, exceeding the limit of %d.' %\
+                      (len(params), dcgm_structs.DCGM_MAX_TEST_PARMS)
+                raise ValueError(err)
+
+            for param in params:
+                self.AddParameter(param)
+
+        if gpuIds:
+            first = True
+            for gpu in gpuIds:
+                if first:
+                    self.runDiagInfo.gpuList = str(gpu)
+                    first = False
+                else:
+                    self.runDiagInfo.gpuList = "%s,%s" % (
+                        self.runDiagInfo.gpuList, str(gpu))
+
+    def SetVerbose(self, val):
+        if val == True:
+            self.runDiagInfo.flags |= dcgm_structs.DCGM_RUN_FLAGS_VERBOSE
+        else:
+            self.runDiagInfo.flags &= ~dcgm_structs.DCGM_RUN_FLAGS_VERBOSE
+
+    def UseFakeGpus(self):
+        self.runDiagInfo.fakeGpuList = self.runDiagInfo.gpuList
+
+    def GetStruct(self):
+        return self.runDiagInfo
+
+    def AddParameter(self, parameterStr):
+        if len(parameterStr) >= dcgm_structs.DCGM_MAX_TEST_PARMS_LEN:
+            err = 'DcgmDiag cannot add parameter \'%s\' because it exceeds max length %d.' % \
+                  (parameterStr, dcgm_structs.DCGM_MAX_TEST_PARMS_LEN)
+            raise ValueError(err)
+
+        index = 0
+        for c in parameterStr:
+            self.runDiagInfo.testParms[self.numParams][index] = ord(c)
+            index += 1
+
+        self.numParams += 1
+
+    def AddTest(self, testNameStr):
+        if len(testNameStr) >= dcgm_structs.DCGM_MAX_TEST_NAMES_LEN:
+            err = 'DcgmDiag cannot add test name \'%s\' because it exceeds max length %d.' % \
+                  (testNameStr, dcgm_structs.DCGM_MAX_TEST_NAMES_LEN)
+            raise ValueError(err)
+
+        index = 0
+        for c in testNameStr:
+            self.runDiagInfo.testNames[self.numTests][index] = ord(c)
+            index += 1
+
+        self.numTests += 1
+
+    def SetStatsOnFail(self, val):
+        if val == True:
+            self.runDiagInfo.flags |= dcgm_structs.DCGM_RUN_FLAGS_STATSONFAIL
+
+    def SetThrottleMask(self, value):
+        if DcgmDiag._versionMap[self.version] < 3:
+            raise ValueError(
+                "Throttle mask requires minimum version 3 for dcgmRunDiag.")
+        if isinstance(
+                value,
+                str) and len(value) >= dcgm_structs.DCGM_THROTTLE_MASK_LEN:
+            raise ValueError("Throttle mask value '%s' exceeds max length %d." %
+                             (value, dcgm_structs.DCGM_THROTTLE_MASK_LEN - 1))
+
+        self.runDiagInfo.throttleMask = str(value)
+
+    def SetFailEarly(self, enable=True, checkInterval=5):
+        if DcgmDiag._versionMap[self.version] < 5:
+            raise ValueError(
+                "Fail early requires minimum version 5 for dcgmRunDiag.")
+        if not isinstance(checkInterval, int):
+            raise ValueError("Invalid checkInterval value: %s" % checkInterval)
+
+        if enable:
+            self.runDiagInfo.flags |= dcgm_structs.DCGM_RUN_FLAGS_FAIL_EARLY
+            self.runDiagInfo.failCheckInterval = checkInterval
+        else:
+            self.runDiagInfo.flags &= ~dcgm_structs.DCGM_RUN_FLAGS_FAIL_EARLY
+
+    def Execute(self, handle):
+        return dcgm_agent.dcgmActionValidate_v2(handle, self.runDiagInfo,
+                                                self.version)
+
+    def SetStatsPath(self, statsPath):
+        if len(statsPath) >= dcgm_structs.DCGM_PATH_LEN:
+            err = "DcgmDiag cannot set statsPath '%s' because it exceeds max length %d." % \
+                   (statsPath, dcgm_structs.DCGM_PATH_LEN)
+            raise ValueError(err)
+
+        self.runDiagInfo.statsPath = statsPath
+
+    def SetConfigFileContents(self, configFileContents):
+        if len(configFileContents) >= dcgm_structs.DCGM_MAX_CONFIG_FILE_LEN:
+            err = "Dcgm Diag cannot set config file contents to '%s' because it exceeds max length %d." \
+                  % (configFileContents, dcgm_structs.DCGM_MAX_CONFIG_FILE_LEN)
+            raise ValueError(err)
+
+        self.runDiagInfo.configFileContents = configFileContents
+
+    def SetDebugLogFile(self, logFileName):
+        if len(logFileName) >= dcgm_structs.DCGM_FILE_LEN:
+            raise ValueError("Cannot set debug file to '%s' because it exceeds max length %d."\
+                % (logFileName, dcgm_structs.DCGM_FILE_LEN))
+
+        self.runDiagInfo.debugLogFile = logFileName
+
+    def SetDebugLevel(self, debugLevel):
+        if debugLevel < 0 or debugLevel > 5:
+            raise ValueError(
+                "Cannot set debug level to %d. Debug Level must be a value from 0-5 inclusive."
+            )
+
+        self.runDiagInfo.debugLevel = debugLevel

model_analyzer/monitor/dcgm/DcgmFieldGroup.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import model_analyzer.monitor.dcgm.dcgm_agent as dcgm_agent
+import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs
+'''
+Class for managing a group of field IDs in the host engine.
+'''
+
+
+class DcgmFieldGroup:
+    '''
+    Constructor
+
+    dcgmHandle - DcgmHandle() instance to use for communicating with the host engine
+    name - Name of the field group to use within DCGM. This must be unique
+    fieldIds - Fields that are part of this group
+    fieldGroupId - If provided, this is used to initialize the object from an existing field group ID
+    '''
+
+    def __init__(self, dcgmHandle, name="", fieldIds=None, fieldGroupId=None):
+        fieldIds = fieldIds or []
+        self.name = name
+        self.fieldIds = fieldIds
+        self._dcgmHandle = dcgmHandle
+        self.wasCreated = False
+
+        #If the user passed in an ID, the field group already exists. Fetch live info
+        if fieldGroupId is not None:
+            self.fieldGroupId = fieldGroupId
+            fieldGroupInfo = dcgm_agent.dcgmFieldGroupGetInfo(
+                self._dcgmHandle.handle, self.fieldGroupId)
+            self.name = fieldGroupInfo.fieldGroupName
+            self.fieldIds = fieldGroupInfo.fieldIds
+        else:
+            self.fieldGroupId = None  #Assign here so the destructor doesn't fail if the call below fails
+            self.fieldGroupId = dcgm_agent.dcgmFieldGroupCreate(
+                self._dcgmHandle.handle, fieldIds, name)
+            self.wasCreated = True
+
+    '''
+    Remove this field group from DCGM. This object can no longer be passed to other APIs after this call.
+    '''
+
+    def Delete(self):
+        if self.wasCreated and self.fieldGroupId is not None:
+            try:
+                try:
+                    dcgm_agent.dcgmFieldGroupDestroy(self._dcgmHandle.handle,
+                                                     self.fieldGroupId)
+                except dcgm_structs.dcgmExceptionClass(
+                        dcgm_structs.DCGM_ST_NO_DATA):
+                    # someone may have deleted the group under us. That's ok.
+                    pass
+                except dcgm_structs.dcgmExceptionClass(
+                        dcgm_structs.DCGM_ST_CONNECTION_NOT_VALID):
+                    # We lost our connection, but we're destructing this object anyway.
+                    pass
+            except AttributeError as ae:
+                # When we're cleaning up at the end, dcgm_agent and dcgm_structs have been unloaded and we'll
+                # get an AttributeError: "'NoneType' object has no 'dcgmExceptionClass'" Ignore this
+                pass
+            except TypeError as te:
+                # When we're cleaning up at the end, dcgm_agent and dcgm_structs have been unloaded and we might
+                # get a TypeError: "'NoneType' object is not callable'" Ignore this
+                pass
+            self.fieldGroupId = None
+            self._dcgmHandle = None
+
+    #Destructor
+    def __del__(self):
+        self.Delete()