triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/triton/server/server_factory.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from urllib.parse import urlparse
+
+from model_analyzer.config.input.config_utils import binary_path_validator
+from model_analyzer.constants import CONFIG_PARSER_FAILURE, LOGGER_NAME
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+from .server_config import TritonServerConfig
+from .server_docker import TritonServerDocker
+from .server_local import TritonServerLocal
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class TritonServerFactory:
+    """
+    A factory for creating TritonServer instances
+    """
+
+    @staticmethod
+    def create_server_docker(
+        image,
+        config,
+        gpus,
+        log_path=None,
+        mounts=None,
+        labels=None,
+        shm_size=None,
+        args=None,
+    ):
+        """
+        Parameters
+        ----------
+        image : str
+            The tritonserver docker image to pull and run
+        config : TritonServerConfig
+            the config object containing arguments for this server instance
+        gpus : list of str
+            List of GPU UUIDs to be mounted and used in the container
+            Use ["all"] to include all GPUs
+        log_path: str
+            Absolute path to the triton log file
+        mounts: list of str
+            The volumes to be mounted to the tritonserver container
+        labels: dict
+            name-value pairs for label to set metadata for triton docker
+            container. (Not the same as environment variables)
+        shm-size: str
+            The size of /dev/shm for the triton docker container.
+        args: dict
+            name-value pairs of triton docker args
+        Returns
+        -------
+        TritonServerDocker
+        """
+
+        return TritonServerDocker(
+            image=image,
+            config=config,
+            gpus=gpus,
+            log_path=log_path,
+            mounts=mounts,
+            labels=labels,
+            shm_size=shm_size,
+            args=args,
+        )
+
+    @staticmethod
+    def create_server_local(path, config, gpus, log_path=None):
+        """
+        Parameters
+        ----------
+        path  : str
+            The absolute path to the tritonserver executable
+        config : TritonServerConfig
+            the config object containing arguments for this server instance
+        gpus: list of str
+            List of GPU UUIDs to be made visible to Triton
+            Use ["all"] to include all GPUs
+        log_path: str
+            Absolute path to the triton log file
+
+        Returns
+        -------
+        TritonServerLocal
+        """
+
+        return TritonServerLocal(path=path, config=config, gpus=gpus, log_path=log_path)
+
+    @staticmethod
+    def get_server_handle(config, gpus, use_model_repository=False):
+        """
+        Creates and returns a TritonServer
+        with specified arguments
+
+        Parameters
+        ----------
+        config : namespace
+            Arguments parsed from the CLI
+        gpus : list of str
+            Available, supported, visible requested GPU UUIDs
+        use_model_repository : bool
+            Optional flag to use the model directory instead
+            of the output model directory
+        Returns
+        -------
+        TritonServer
+            Handle to the Triton Server
+        """
+
+        if config.triton_launch_mode == "remote":
+            server = TritonServerFactory._get_remote_server_handle(
+                config, print_warning_message=use_model_repository
+            )
+        elif config.triton_launch_mode == "local":
+            server = TritonServerFactory._get_local_server_handle(
+                config, gpus, use_model_repository=True
+            )
+        elif config.triton_launch_mode == "docker":
+            server = TritonServerFactory._get_docker_server_handle(
+                config, gpus, use_model_repository=True
+            )
+        elif config.triton_launch_mode == "c_api":
+            server = TritonServerFactory._get_c_api_server_handle(
+                config, use_model_repository
+            )
+        else:
+            raise TritonModelAnalyzerException(
+                f"Unrecognized triton-launch-mode : {config.triton_launch_mode}"
+            )
+
+        return server
+
+    @staticmethod
+    def _get_remote_server_handle(config, print_warning_message=True):
+        triton_config = TritonServerConfig()
+        triton_config.update_config(config.triton_server_flags)
+        triton_config["model-repository"] = "remote-model-repository"
+        logger.info("Using remote Triton Server")
+        server = TritonServerFactory.create_server_local(
+            path=None, config=triton_config, gpus=[], log_path=""
+        )
+        if print_warning_message:
+            logger.warning(
+                "GPU memory metrics reported in the remote mode are not"
+                " accurate. Model Analyzer uses Triton explicit model control to"
+                " load/unload models. Some frameworks do not release the GPU"
+                " memory even when the memory is not being used. Consider"
+                ' using the "local" or "docker" mode if you want to accurately'
+                " monitor the GPU memory usage for different models."
+            )
+
+        return server
+
+    @staticmethod
+    def _get_local_server_handle(config, gpus, use_model_repository):
+        TritonServerFactory._validate_triton_server_path(config)
+
+        triton_config = TritonServerConfig()
+        triton_config.update_config(config.triton_server_flags)
+
+        triton_config["model-repository"] = (
+            config.model_repository
+            if use_model_repository and config.model_repository
+            else config.output_model_repository_path
+        )
+
+        if use_model_repository and config.model_repository:
+            triton_config["strict-model-config"] = "false"
+
+        triton_config["http-port"] = config.triton_http_endpoint.split(":")[-1]
+        triton_config["grpc-port"] = config.triton_grpc_endpoint.split(":")[-1]
+        triton_config["metrics-port"] = urlparse(config.triton_metrics_url).port
+        triton_config["model-control-mode"] = "explicit"
+        triton_config["metrics-interval-ms"] = int(config.monitoring_interval * 1e3)
+        logger.info("Starting a local Triton Server")
+        server = TritonServerFactory.create_server_local(
+            path=config.triton_server_path,
+            config=triton_config,
+            gpus=gpus,
+            log_path=config.triton_output_path,
+        )
+
+        return server
+
+    @staticmethod
+    def _get_docker_server_handle(config, gpus, use_model_repository):
+        triton_config = TritonServerConfig()
+        triton_config.update_config(config.triton_server_flags)
+
+        if use_model_repository:
+            triton_config["model-repository"] = os.path.abspath(config.model_repository)
+        else:
+            triton_config["model-repository"] = os.path.abspath(
+                config.output_model_repository_path
+            )
+
+        if use_model_repository:
+            triton_config["strict-model-config"] = "false"
+
+        triton_config["http-port"] = config.triton_http_endpoint.split(":")[-1]
+        triton_config["grpc-port"] = config.triton_grpc_endpoint.split(":")[-1]
+        triton_config["metrics-port"] = urlparse(config.triton_metrics_url).port
+        triton_config["model-control-mode"] = "explicit"
+        triton_config["metrics-interval-ms"] = int(config.monitoring_interval * 1e3)
+        logger.info("Starting a Triton Server using docker")
+        server = TritonServerFactory.create_server_docker(
+            image=config.triton_docker_image,
+            config=triton_config,
+            gpus=gpus,
+            log_path=config.triton_output_path,
+            mounts=config.triton_docker_mounts,
+            labels=config.triton_docker_labels,
+            shm_size=config.triton_docker_shm_size,
+            args=config.triton_docker_args,
+        )
+
+        return server
+
+    @staticmethod
+    def _get_c_api_server_handle(config, use_model_repository):
+        TritonServerFactory._validate_triton_install_path(config)
+
+        triton_config = TritonServerConfig()
+
+        if use_model_repository:
+            triton_config["model-repository"] = os.path.abspath(config.model_repository)
+        else:
+            triton_config["model-repository"] = os.path.abspath(
+                config.output_model_repository_path
+            )
+
+        if use_model_repository:
+            triton_config["strict-model-config"] = "false"
+
+        logger.info("Starting a Triton Server using perf_analyzer's C_API")
+        server = TritonServerFactory.create_server_local(
+            path=None, config=triton_config, gpus=[], log_path=""
+        )
+        logger.warning(
+            "When profiling with perf_analyzer's C_API, some metrics may be "
+            "affected. Triton is not launched with explicit model control "
+            "mode, and as a result, loads all model config variants as they "
+            "are created in the output_model_repository."
+        )
+
+        return server
+
+    @staticmethod
+    def _validate_triton_server_path(config):
+        """
+        Raises an exception if 'triton_server_path' doesn't exist
+
+        Parameters
+        ----------
+        config : namespace
+            Arguments parsed from the CLI
+        """
+        path = config.get_config()["triton_server_path"].value()
+        config_status = binary_path_validator(path)
+        if config_status.status() == CONFIG_PARSER_FAILURE:
+            raise TritonModelAnalyzerException(config_status.message())
+
+    @staticmethod
+    def _validate_triton_install_path(config):
+        """
+        Raises an exception in the following cases:
+          - 'triton_install_path' doesn't exist
+          - 'trtion_install_path' exists, but contains no files
+
+        Parameters
+        ----------
+        config : namespace
+            Arguments parsed from the CLI
+        """
+        path = config.get_config()["triton_install_path"].value()
+
+        # Check the file system
+        if not path or not os.path.exists(path) or not os.path.isdir(path):
+            raise TritonModelAnalyzerException(
+                f"triton_install_path {path} is not specified, does not exist, "
+                "or is not a directory."
+            )
+
+        # Make sure that files exist in the install directory
+        if len(os.listdir(path)) == 0:
+            raise TritonModelAnalyzerException(
+                f"triton_install_path {path} should not be empty."
+            )

model_analyzer/triton/server/server_local.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import tempfile
+from io import TextIOWrapper
+from subprocess import DEVNULL, STDOUT, Popen, TimeoutExpired
+
+import psutil
+
+from model_analyzer.constants import LOGGER_NAME, SERVER_OUTPUT_TIMEOUT_SECS
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+from .server import TritonServer
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class TritonServerLocal(TritonServer):
+    """
+    Concrete Implementation of TritonServer interface that runs
+    tritonserver locally as as subprocess.
+    """
+
+    def __init__(self, path, config, gpus, log_path):
+        """
+        Parameters
+        ----------
+        path  : str
+            The absolute path to the tritonserver executable
+        config : TritonServerConfig
+            the config object containing arguments for this server instance
+        gpus: list of str
+            List of GPU UUIDs to be made visible to Triton
+        log_path: str
+            Absolute path to the triton log file
+        """
+
+        self._tritonserver_process = None
+        self._server_config = config
+        self._server_path = path
+        self._gpus = gpus
+        self._log_path = log_path
+        self._log_file = DEVNULL
+        self._is_first_time_starting_server = True
+
+        assert self._server_config[
+            "model-repository"
+        ], "Triton Server requires --model-repository argument to be set."
+
+    def start(self, env=None):
+        """
+        Starts the tritonserver container locally
+        """
+
+        if self._server_path:
+            # Create command list and run subprocess
+            cmd = [self._server_path]
+            cmd += self._server_config.to_args_list()
+
+            # Set environment, update with user config env
+            triton_env = os.environ.copy()
+
+            if env:
+                # Filter env variables that use env lookups
+                for variable, value in env.items():
+                    if value.find("$") == -1:
+                        triton_env[variable] = value
+                    else:
+                        # Collect the ones that need lookups to give to the shell
+                        triton_env[variable] = os.path.expandvars(value)
+
+            # List GPUs to be used by tritonserver
+            triton_env["CUDA_VISIBLE_DEVICES"] = ",".join(
+                [gpu.device_uuid() for gpu in self._gpus]
+            )
+
+            if self._log_path:
+                try:
+                    if self._is_first_time_starting_server:
+                        if os.path.exists(self._log_path):
+                            os.remove(self._log_path)
+                    self._log_file = open(self._log_path, "a+")
+                except OSError as e:
+                    raise TritonModelAnalyzerException(e)
+            else:
+                self._log_file = tempfile.NamedTemporaryFile()
+
+            self._is_first_time_starting_server = False
+
+            # Construct Popen command
+            try:
+                self._tritonserver_process = Popen(
+                    cmd,
+                    stdout=self._log_file,
+                    stderr=STDOUT,
+                    start_new_session=True,
+                    universal_newlines=True,
+                    env=triton_env,
+                )
+
+                logger.debug("Triton Server started.")
+            except Exception as e:
+                raise TritonModelAnalyzerException(e)
+
+    def stop(self):
+        """
+        Stops the running tritonserver
+        """
+
+        # Terminate process, capture output
+        if self._tritonserver_process is not None:
+            self._tritonserver_process.terminate()
+            try:
+                self._tritonserver_process.communicate(
+                    timeout=SERVER_OUTPUT_TIMEOUT_SECS
+                )
+            except TimeoutExpired:
+                self._tritonserver_process.kill()
+                self._tritonserver_process.communicate()
+            self._tritonserver_process = None
+            if self._log_path:
+                self._log_file.close()
+            logger.debug("Stopped Triton Server.")
+
+    def cpu_stats(self):
+        """
+        Returns the CPU memory usage and CPU available memory in MB
+        """
+
+        if self._tritonserver_process:
+            server_process = psutil.Process(self._tritonserver_process.pid)
+            process_memory_info = server_process.memory_full_info()
+            system_memory_info = psutil.virtual_memory()
+
+            # Divide by 1.0e6 to convert from bytes to MB
+            return (process_memory_info.uss // 1.0e6), (
+                system_memory_info.available // 1.0e6
+            )
+        else:
+            return 0.0, 0.0
+
+    def log_file(self) -> TextIOWrapper:
+        return self._log_file

triton_model_analyzer-1.48.0.dist-info/METADATA

@@ -0,0 +1,52 @@
+Metadata-Version: 2.4
+Name: triton-model-analyzer
+Version: 1.48.0
+Summary: Triton Model Analyzer is a tool to profile and analyze the runtime performance of one or more models on the Triton Inference Server
+Author-email: "NVIDIA Inc." <sw-dl-triton@nvidia.com>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://developer.nvidia.com/nvidia-triton-inference-server
+Project-URL: Repository, https://github.com/triton-inference-server/model_analyzer
+Keywords: triton,tensorrt,inference,server,service,analyzer,nvidia
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Image Recognition
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Environment :: Console
+Classifier: Natural Language :: English
+Classifier: Operating System :: POSIX :: Linux
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: cryptography>=3.3.2
+Requires-Dist: distro>=1.5.0
+Requires-Dist: docker>=4.3.1
+Requires-Dist: gevent>=22.08.0
+Requires-Dist: grpcio<1.68,>=1.63.0
+Requires-Dist: httplib2>=0.19.0
+Requires-Dist: importlib_metadata>=7.1.0
+Requires-Dist: matplotlib>=3.3.4
+Requires-Dist: numba>=0.51.2
+Requires-Dist: optuna==3.6.1
+Requires-Dist: pdfkit>=0.6.1
+Requires-Dist: prometheus_client>=0.9.0
+Requires-Dist: protobuf
+Requires-Dist: psutil>=5.8.0
+Requires-Dist: pyyaml>=5.3.1
+Requires-Dist: requests>=2.24.0
+Requires-Dist: tritonclient[all]>=2.4.0
+Requires-Dist: urllib3>=2.0.7
+Provides-Extra: perf-analyzer
+Requires-Dist: perf-analyzer; extra == "perf-analyzer"
+Dynamic: license-file
+
+See the Model Analyzer's [installation documentation](https://github.com/triton-inference-server/model_analyzer/blob/main/docs/install.md#using-pip3) for package details. The [quick start](https://github.com/triton-inference-server/model_analyzer/blob/main/docs/quick_start.md) documentation describes how to get started with profiling and analysis using Triton Model Analyzer.