triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/device/gpu_device_factory.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import numba.cuda
+
+import model_analyzer.monitor.dcgm.dcgm_agent as dcgm_agent
+import model_analyzer.monitor.dcgm.dcgm_structs as structs
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.device.gpu_device import GPUDevice
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class GPUDeviceFactory:
+    """
+    Factory class for creating GPUDevices
+    """
+
+    def __init__(self):
+        self._devices = []
+        self._devices_by_bus_id = {}
+        self._devices_by_uuid = {}
+        self.init_all_devices()
+
+    def init_all_devices(self, dcgmPath=None):
+        """
+        Create GPUDevice objects for all DCGM visible
+        devices.
+
+        Parameters
+        ----------
+        dcgmPath : str
+            Absolute path to dcgm shared library
+        """
+
+        if numba.cuda.is_available():
+            logger.info("Initializing GPUDevice handles")
+            structs._dcgmInit(dcgmPath)
+            dcgm_agent.dcgmInit()
+
+            # Start DCGM in the embedded mode to use the shared library
+            dcgm_handle = dcgm_agent.dcgmStartEmbedded(
+                structs.DCGM_OPERATION_MODE_MANUAL
+            )
+
+            # Create a GPU device for every supported DCGM device
+            dcgm_device_ids = dcgm_agent.dcgmGetAllSupportedDevices(dcgm_handle)
+
+            for device_id in dcgm_device_ids:
+                device_atrributes = dcgm_agent.dcgmGetDeviceAttributes(
+                    dcgm_handle, device_id
+                ).identifiers
+                pci_bus_id = device_atrributes.pciBusId
+                device_uuid = device_atrributes.uuid
+                device_name = device_atrributes.deviceName
+
+                gpu_device = GPUDevice(device_name, device_id, pci_bus_id, device_uuid)
+
+                self._devices.append(gpu_device)
+                self._devices_by_bus_id[pci_bus_id] = gpu_device
+                self._devices_by_uuid[device_uuid] = gpu_device
+
+            dcgm_agent.dcgmShutdown()
+
+    def get_device_by_bus_id(self, bus_id, dcgmPath=None):
+        """
+        Get a GPU device by using its bus ID.
+
+        Parameters
+        ----------
+        bus_id : bytes
+            Bus id corresponding to the GPU. The bus id should be created by
+            converting the colon separated hex notation into a bytes type
+            using ascii encoding. The bus id before conversion to bytes
+            should look like "00:65:00".
+
+        Returns
+        -------
+        Device
+            The device associated with this bus id.
+        """
+
+        if bus_id in self._devices_by_bus_id:
+            return self._devices_by_bus_id[bus_id]
+        else:
+            raise TritonModelAnalyzerException(
+                f"GPU with {bus_id} bus id is either not supported by DCGM or not present."
+            )
+
+    def get_device_by_cuda_index(self, index):
+        """
+        Get a GPU device using the CUDA index. This includes the index
+        provided by CUDA visible devices.
+
+        Parameters
+        ----------
+        index : int
+            index of the device in the list of visible CUDA devices.
+
+        Returns
+        -------
+        Device
+            The device associated with the index provided.
+
+        Raises
+        ------
+        IndexError
+            If the index is out of bound.
+        """
+
+        devices = numba.cuda.list_devices()
+        if index > len(devices) - 1:
+            raise IndexError
+
+        cuda_device = devices[index]
+        device_identity = cuda_device.get_device_identity()
+        pci_domain_id = device_identity["pci_domain_id"]
+        pci_device_id = device_identity["pci_device_id"]
+        pci_bus_id = device_identity["pci_bus_id"]
+        device_bus_id = f"{pci_domain_id:08X}:{pci_bus_id:02X}:{pci_device_id:02X}.0"
+
+        return self.get_device_by_bus_id(device_bus_id)
+
+    def get_device_by_uuid(self, uuid, dcgmPath=None):
+        """
+        Get a GPU device using the GPU uuid.
+
+        Parameters
+        ----------
+        uuid : str
+            index of the device in the list of visible CUDA devices.
+
+        Returns
+        -------
+        Device
+            The device associated with the uuid.
+
+        Raises
+        ------
+        TritonModelAnalyzerExcpetion
+            If the uuid does not exist this exception will be raised.
+        """
+
+        if uuid in self._devices_by_uuid:
+            return self._devices_by_uuid[uuid]
+        else:
+            raise TritonModelAnalyzerException(f"GPU UUID {uuid} was not found.")
+
+    def verify_requested_gpus(self, requested_gpus):
+        """
+        Creates a list of GPU UUIDs corresponding to the GPUs visible to
+        numba.cuda among the requested gpus
+
+        Parameters
+        ----------
+        requested_gpus : list of str or list of ints
+            Can either be GPU UUIDs or GPU device ids
+
+        Returns
+        -------
+        List of GPUDevices
+            list of GPUDevices corresponding to visible GPUs among requested
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+        """
+
+        cuda_visible_gpus = self.get_cuda_visible_gpus()
+
+        if len(requested_gpus) == 1:
+            if requested_gpus[0] == "all":
+                self._log_gpus_used(cuda_visible_gpus)
+                return cuda_visible_gpus
+            elif requested_gpus[0] == "[]":
+                logger.info("No GPUs requested")
+                return []
+
+        try:
+            # Check if each string in the list can be parsed as an int
+            requested_cuda_indices = list(map(int, requested_gpus))
+            requested_gpus = []
+
+            for idx in requested_cuda_indices:
+                try:
+                    requested_gpus.append(self.get_device_by_cuda_index(idx))
+                except TritonModelAnalyzerException:
+                    raise TritonModelAnalyzerException(
+                        f"Requested GPU with device id : {idx}. This GPU is not supported by DCGM."
+                    )
+        except ValueError:
+            # requested_gpus are assumed to be UUIDs
+            requested_gpus = [self.get_device_by_uuid(uuid) for uuid in requested_gpus]
+
+        # Return the intersection of CUDA visible UUIDs and requested/supported UUIDs.
+        available_gpus = list(set(cuda_visible_gpus) & set(requested_gpus))
+        self._log_gpus_used(available_gpus)
+
+        return available_gpus
+
+    def get_cuda_visible_gpus(self):
+        """
+        Returns
+        -------
+        list of GPUDevice
+            UUIDs of the DCGM supported devices visible to CUDA
+        """
+
+        cuda_visible_gpus = []
+        if numba.cuda.is_available():
+            for cuda_device in numba.cuda.list_devices():
+                try:
+                    cuda_visible_gpus.append(
+                        self.get_device_by_cuda_index(cuda_device.id)
+                    )
+                except TritonModelAnalyzerException:
+                    # Device not supported by DCGM, log warning
+                    logger.warning(
+                        f"Device '{str(cuda_device.name, encoding='ascii')}' with "
+                        f"cuda device id {cuda_device.id} is not supported by DCGM."
+                    )
+        return cuda_visible_gpus
+
+    def _log_gpus_used(self, gpus):
+        """
+        Log the info for the GPUDevices in use
+        """
+
+        for gpu in gpus:
+            logger.info(
+                f"Using GPU {gpu.device_id()} {gpu.device_name()} with UUID {gpu.device_uuid()}"
+            )

model_analyzer/entrypoint.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import shutil
+import sys
+from pprint import pformat
+
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.device.gpu_device_factory import GPUDeviceFactory
+
+from .analyzer import Analyzer
+from .cli.cli import CLI
+from .config.input.config_command_profile import ConfigCommandProfile
+from .config.input.config_command_report import ConfigCommandReport
+from .log_formatter import setup_logging
+from .model_analyzer_exceptions import TritonModelAnalyzerException
+from .state.analyzer_state_manager import AnalyzerStateManager
+from .triton.client.client_factory import TritonClientFactory
+from .triton.server.server_factory import TritonServerFactory
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+def get_client_handle(config):
+    """
+    Creates and returns a TritonClient
+    with specified arguments
+
+    Parameters
+    ----------
+    config : namespace
+        Arguments parsed from the CLI
+    """
+
+    if config.client_protocol == "http":
+        http_ssl_options = get_http_ssl_options(config)
+        client = TritonClientFactory.create_http_client(
+            server_url=config.triton_http_endpoint, ssl_options=http_ssl_options
+        )
+    elif config.client_protocol == "grpc":
+        grpc_ssl_options = get_grpc_ssl_options(config)
+        client = TritonClientFactory.create_grpc_client(
+            server_url=config.triton_grpc_endpoint, ssl_options=grpc_ssl_options
+        )
+    else:
+        raise TritonModelAnalyzerException(
+            f"Unrecognized client-protocol : {config.client_protocol}"
+        )
+
+    return client
+
+
+def get_http_ssl_options(config):
+    """
+    Returns HTTP SSL options dictionary
+
+    Parameters
+    ----------
+    config : namespace
+        Arguments parsed from the CLI
+    """
+
+    ssl_option_keys = [
+        "ssl-https-verify-peer",
+        "ssl-https-verify-host",
+        "ssl-https-ca-certificates-file",
+        "ssl-https-client-certificate-file",
+        "ssl-https-client-certificate-type",
+        "ssl-https-private-key-file",
+        "ssl-https-private-key-type",
+    ]
+
+    return {
+        key: config.perf_analyzer_flags[key]
+        for key in ssl_option_keys
+        if key in config.perf_analyzer_flags
+    }
+
+
+def get_grpc_ssl_options(config):
+    """
+    Returns gRPC SSL options dictionary
+
+    Parameters
+    ----------
+    config : namespace
+        Arguments parsed from the CLI
+    """
+
+    ssl_option_keys = [
+        "ssl-grpc-use-ssl",
+        "ssl-grpc-root-certifications-file",
+        "ssl-grpc-private-key-file",
+        "ssl-grpc-certificate-chain-file",
+    ]
+
+    return {
+        key: config.perf_analyzer_flags[key]
+        for key in ssl_option_keys
+        if key in config.perf_analyzer_flags
+    }
+
+
+def get_triton_handles(config, gpus):
+    """
+    Creates a TritonServer and starts it. Creates a TritonClient
+
+    Parameters
+    ----------
+    config : namespace
+        The arguments passed into the CLI
+    gpus : list of str
+        Available, supported, visible requested GPU UUIDs
+
+    Returns
+    -------
+    TritonClient, TritonServer
+        Handles for triton client/server pair.
+    """
+
+    client = get_client_handle(config)
+    fail_if_server_already_running(client, config)
+    server = TritonServerFactory.get_server_handle(config, gpus)
+
+    return client, server
+
+
+def get_cli_and_config_options():
+    """
+    Parses CLI/Yaml Config file options
+    into Namespace and Config objects for
+    the correct subcommand
+
+    Returns
+    -------
+    args : Namespace
+        Object that contains the parse CLI commands
+        Used for the global options
+    config: CommandConfig
+        The config corresponding to the command being run,
+        already filled in with values from CLI or YAML.
+    """
+
+    # Parse CLI options
+    try:
+        config_profile = ConfigCommandProfile()
+        config_report = ConfigCommandReport()
+
+        cli = CLI()
+        cli.add_subcommand(
+            cmd="profile",
+            help="Run model inference profiling based on specified CLI or config options.",
+            config=config_profile,
+        )
+        cli.add_subcommand(
+            cmd="analyze",
+            help="DEPRECATED: Aliased to profile - please use profile subcommand.",
+            config=config_profile,
+        )
+        cli.add_subcommand(
+            cmd="report",
+            help="Generate detailed reports for a single config",
+            config=config_report,
+        )
+        return cli.parse()
+
+    except TritonModelAnalyzerException as e:
+        logger.error(f"Model Analyzer encountered an error: {e}")
+        sys.exit(1)
+
+
+def create_output_model_repository(config):
+    """
+    Creates output model repository
+
+    Parameters
+    ----------
+    ConfigCommandProfile
+        The config containing the output_model_repository_path
+    """
+
+    try:
+        os.mkdir(config.output_model_repository_path)
+    except FileExistsError:
+        if not config.override_output_model_repository:
+            raise TritonModelAnalyzerException(
+                f'Path "{config.output_model_repository_path}" already exists. '
+                'Please set or modify "--output-model-repository-path" flag or remove this directory.'
+                " You can also allow overriding of the output directory using"
+                ' the "--override-output-model-repository" flag.'
+            )
+        else:
+            shutil.rmtree(config.output_model_repository_path)
+            logger.warning(
+                "Overriding the output model repo path "
+                f'"{config.output_model_repository_path}"'
+            )
+            os.mkdir(config.output_model_repository_path)
+
+
+def fail_if_server_already_running(client, config):
+    """
+    Checks if there is already a Triton server running
+    If there is and the launch mode is not 'remote' or 'c_api', throw an exception
+    Else, nothing will happen
+    """
+    if config.triton_launch_mode == "remote" or config.triton_launch_mode == "c_api":
+        return
+
+    is_server_running = True
+    try:
+        client.is_server_ready()
+    except Exception:
+        is_server_running = False
+    finally:
+        if is_server_running:
+            raise TritonModelAnalyzerException(
+                f"Another application (likely a Triton Server) is already using the desired port. In '{config.triton_launch_mode}' mode, Model Analyzer will launch a Triton Server and requires that the HTTP/GRPC port is not occupied by another application. Please kill the other application or specify a different port."
+            )
+
+
+def main():
+    """
+    Main entrypoint of model_analyzer
+    """
+
+    # Need to create a basic logging format for logs we print
+    # before we have enough information to configure the full logger
+    logging.basicConfig(format="[Model Analyzer] %(message)s")
+
+    args, config = get_cli_and_config_options()
+
+    setup_logging(quiet=args.quiet, verbose=args.verbose)
+
+    logger.debug("\n%s", pformat(config.get_all_config()))
+
+    # Launch subcommand handlers
+    server = None
+    try:
+        # Make calls to correct analyzer subcommand functions
+        if args.subcommand == "profile" or args.subcommand == "analyze":
+            if _is_a_model_repository_required(args, config):
+                raise TritonModelAnalyzerException(
+                    "No model repository specified. Please specify it using the YAML config file or using the --model-repository flag in CLI."
+                )
+
+            # Set up devices
+            if config.dcgm_disable:
+                gpus = []
+            else:
+                gpus = GPUDeviceFactory().verify_requested_gpus(config.gpus)
+
+            # Check/create output model repository
+            create_output_model_repository(config)
+
+            client, server = get_triton_handles(config, gpus)
+            state_manager = AnalyzerStateManager(config=config, server=server)
+
+            # Only check for exit after the events that take a long time.
+            if state_manager.exiting():
+                return
+
+            analyzer = Analyzer(
+                config, server, state_manager, checkpoint_required=False
+            )
+            analyzer.profile(
+                client=client, gpus=gpus, mode=args.mode, verbose=args.verbose
+            )
+        elif args.subcommand == "report":
+            analyzer = Analyzer(
+                config,
+                server,
+                AnalyzerStateManager(config=config, server=server),
+                checkpoint_required=True,
+            )
+            analyzer.report(mode=args.mode)
+    finally:
+        if server is not None:
+            server.stop()
+
+
+def _is_a_model_repository_required(args, config):
+    model_repository_required = (
+        args.subcommand == "profile"
+        and not config.model_repository
+        and not config.triton_launch_mode == "remote"
+    )
+    return model_repository_required
+
+
+if __name__ == "__main__":
+    main()

model_analyzer/log_formatter.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+
+from model_analyzer.constants import LOGGER_NAME
+
+
+class LogFormatter(logging.Formatter):
+    """Class to handle formatting of the logger outputs"""
+
+    def __init__(self):
+        logger = logging.getLogger(LOGGER_NAME)
+        self._log_level = logger.getEffectiveLevel()
+        super().__init__(datefmt="%H:%M:%S")
+
+    def format(self, record):
+        front = "%(asctime)s " if self._log_level is logging.DEBUG else ""
+        if record.levelno == logging.INFO:
+            self._style._fmt = f"{front}[Model Analyzer] %(message)s"
+        else:
+            self._style._fmt = f"{front}[Model Analyzer] %(levelname)s: %(message)s"
+        return super().format(record)
+
+
+def setup_logging(quiet, verbose):
+    """
+    Setup logger format
+
+    Parameters
+    ----------
+    quiet : bool
+        If true, don't print anything other than errors
+    verbose : bool
+        If true and quiet is not true, print debug information
+    """
+
+    if quiet:
+        log_level = logging.ERROR
+    elif verbose:
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.INFO
+
+    logger = logging.getLogger(LOGGER_NAME)
+    logger.setLevel(level=log_level)
+
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(LogFormatter())
+    logger.addHandler(handler)
+    logger.propagate = False

model_analyzer/model_analyzer_exceptions.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class TritonModelAnalyzerException(Exception):
+    """
+    A custom exception specific to the Triton Model
+    Analyzer
+    """
+
+    pass