triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/monitor/monitor.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+from abc import ABC, abstractmethod
+from multiprocessing.pool import ThreadPool
+
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+
+class Monitor(ABC):
+    """
+    Monitor abstract class is a parent class used for monitoring devices.
+    """
+
+    def __init__(self, frequency, metrics):
+        """
+        Parameters
+        ----------
+        frequency : float
+            How often the metrics should be monitored.
+        metrics : list
+            A list of Record objects that will be monitored.
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+        """
+
+        self._frequency = frequency
+
+        # Is the background thread active
+        self._thread_active = False
+
+        # Background thread collecting results
+        self._thread = None
+
+        # Thread pool
+        self._thread_pool = ThreadPool(processes=1)
+        self._metrics = metrics
+
+    def _monitoring_loop(self):
+        frequency = self._frequency
+
+        while self._thread_active:
+            begin = time.time()
+            # Monitoring iteration implemented by each of the subclasses
+            self._monitoring_iteration()
+
+            duration = time.time() - begin
+            if duration < frequency:
+                time.sleep(frequency - duration)
+
+    @abstractmethod
+    def _monitoring_iteration(self):
+        """
+        Each of the subclasses must implement this.
+        This is called to execute a single round of monitoring.
+        """
+
+        pass
+
+    @abstractmethod
+    def _collect_records(self):
+        """
+        This method is called to collect all the monitoring records.
+        It is called in the stop_recording_metrics function after
+        the background thread has stopped.
+
+        Returns
+        -------
+        List of Records
+            The list of records collected by the monitor
+        """
+
+        pass
+
+    @abstractmethod
+    def is_monitoring_connected(self) -> bool:
+        """
+        This method is called to determine if we can connect to the
+        monitor
+
+        Returns
+        -------
+        bool
+           True if connection to the monitor was successful
+        """
+
+        pass
+
+    def start_recording_metrics(self):
+        """
+        Start recording the metrics.
+        """
+
+        self._thread_active = True
+        self._thread = self._thread_pool.apply_async(self._monitoring_loop)
+
+    def stop_recording_metrics(self):
+        """
+        Stop recording metrics. This will stop monitoring all the metrics.
+
+        Returns
+        ------
+        List of Records
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+        """
+
+        if not self._thread_active:
+            raise TritonModelAnalyzerException(
+                "start_recording_metrics should be "
+                "called before stop_recording_metrics"
+            )
+
+        self._thread_active = False
+        self._thread = None
+
+        return self._collect_records()
+
+    def destroy(self):
+        """
+        Cleanup threadpool resources
+        """
+
+        self._thread_pool.terminate()
+        self._thread_pool.close()

model_analyzer/monitor/remote_monitor.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import requests
+from prometheus_client.parser import text_string_to_metric_families
+
+from model_analyzer.constants import LOGGER_NAME
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+from model_analyzer.record.types.gpu_free_memory import GPUFreeMemory
+from model_analyzer.record.types.gpu_power_usage import GPUPowerUsage
+from model_analyzer.record.types.gpu_used_memory import GPUUsedMemory
+from model_analyzer.record.types.gpu_utilization import GPUUtilization
+
+from .monitor import Monitor
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class RemoteMonitor(Monitor):
+    """
+    Requests metrics from Triton's metrics
+    endpoint
+    """
+
+    gpu_metrics = {
+        "nv_gpu_utilization": GPUUtilization,
+        "nv_gpu_memory_used_bytes": GPUUsedMemory,
+        "nv_gpu_power_usage": GPUPowerUsage,
+        "nv_gpu_memory_total_bytes": GPUFreeMemory,
+    }
+
+    def __init__(self, metrics_url, frequency, metrics):
+        super().__init__(frequency, metrics)
+        self._metrics_url = metrics_url
+        self._metrics_responses = []
+
+        allowed_metrics = set(self.gpu_metrics.values())
+        if not set(metrics).issubset(allowed_metrics):
+            unsupported_metrics = set(metrics) - allowed_metrics
+            raise TritonModelAnalyzerException(
+                f"GPU monitoring does not currently support the following metrics: {unsupported_metrics}]"
+            )
+
+    def is_monitoring_connected(self) -> bool:
+        try:
+            status_code = requests.get(self._metrics_url, timeout=10).status_code
+        except Exception as ex:
+            return False
+
+        return status_code == requests.codes["okay"]
+
+    def _monitoring_iteration(self):
+        """
+        When this function runs, it requests all the metrics
+        that triton has collected and organizes them into
+        the dict. This function should run as fast
+        as possible
+        """
+
+        self._metrics_responses.append(
+            str(requests.get(self._metrics_url, timeout=10).content, encoding="ascii")
+        )
+
+    def _collect_records(self):
+        """
+        This function will organize the metrics responses
+        and create Records out of them
+        """
+
+        records = []
+
+        for response in self._metrics_responses:
+            metrics = text_string_to_metric_families(response)
+            processed_gpu_used_memory = False
+            calculate_free_memory_after_pass = False
+            gpu_memory_used_bytes = None
+            for metric in metrics:
+                if (
+                    metric.name in self.gpu_metrics
+                    and self.gpu_metrics[metric.name] in self._metrics
+                ):
+                    for sample in metric.samples:
+                        if sample.name == "nv_gpu_memory_used_bytes":
+                            processed_gpu_used_memory = True
+                            gpu_memory_used_bytes = sample.value
+                            self._create_and_add_record(
+                                records, sample, gpu_memory_used_bytes // 1.0e6
+                            )
+                        elif sample.name == "nv_gpu_memory_total_bytes":
+                            if processed_gpu_used_memory:
+                                self._create_and_add_record(
+                                    records,
+                                    sample,
+                                    (sample.value - gpu_memory_used_bytes) // 1.0e6,
+                                )
+                            else:
+                                total_memory_metric = metric
+                                calculate_free_memory_after_pass = True
+                        elif sample.name == "nv_gpu_utilization":
+                            self._create_and_add_record(
+                                records, sample, sample.value * 100
+                            )
+                        else:
+                            self._create_and_add_record(records, sample, sample.value)
+            if calculate_free_memory_after_pass:
+                for sample in total_memory_metric.samples:
+                    self._create_and_add_record(
+                        records, sample, (sample.value - gpu_memory_used_bytes) // 1.0e6
+                    )
+
+        return records
+
+    def _create_and_add_record(self, records, sample, sample_value):
+        """
+        Adds a record to given dict
+        """
+
+        records.append(
+            self.gpu_metrics[sample.name](
+                value=sample_value, device_uuid=sample.labels["gpu_uuid"]
+            )
+        )

model_analyzer/output/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/output/file_writer.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+from .output_writer import OutputWriter
+
+
+class FileWriter(OutputWriter):
+    """
+    Writes table to a file or stdout
+    """
+
+    def __init__(self, filename=None):
+        """
+        Parameters
+        ----------
+        filename : File
+            The full path to the file or stream to write the output to.
+            Writes to stdout if filename is None
+        """
+
+        self._filename = filename
+
+    def write(self, out, append=False):
+        """
+        Writes the output to a file or stdout
+
+        Parameters
+        ----------
+        out : str
+            The string to be written to the
+            file or stdout
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If there is an error or exception while writing
+            the output.
+        """
+
+        write_mode = "a+" if append else "w+"
+        if self._filename:
+            try:
+                with open(self._filename, write_mode) as f:
+                    f.write(out)
+            except OSError as e:
+                raise TritonModelAnalyzerException(e)
+        else:
+            print(out, end="")

model_analyzer/output/output_writer.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+
+class OutputWriter(ABC):
+    """
+    Interface that receives a table
+    and writes the table to a file or stream.
+    """
+
+    @abstractmethod
+    def write(self, out):
+        """
+        Writes the output to a file
+        (stdout, .txt, .csv etc.)
+
+        Parameters
+        ----------
+        out : str
+            The string to be written out
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If there is an error or exception while writing
+            the output.
+        """

model_analyzer/perf_analyzer/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/perf_analyzer/genai_perf_config.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException
+
+
+class GenaiPerfConfig:
+    """
+    A config class to set arguments to the genai_perf.
+    An argument set to None will use the genai_perf's default.
+    """
+
+    genai_perf_args = [
+        "backend",
+        "endpoint",
+        "service-kind",
+        "url",
+        "expected-output-tokens",
+        "input-dataset",
+        "input-tokens-mean",
+        "input-tokens-stddev",
+        "input-type",
+        "num-of-output-prompts",
+        "random-seed",
+        "streaming",
+        "tokenizer",
+    ]
+
+    boolean_args = ["streaming"]
+
+    def __init__(self):
+        """
+        Construct a GenaiPerfConfig
+        """
+
+        self._args = {k: None for k in self.genai_perf_args}
+
+    @classmethod
+    def allowed_keys(cls):
+        """
+        Returns
+        -------
+        list of str
+            The keys that are allowed to be
+            passed into perf_analyzer
+        """
+
+        return cls.genai_perf_args
+
+    def update_config(self, params=None):
+        """
+        Allows setting values from a params dict
+
+        Parameters
+        ----------
+        params: dict
+            keys are allowed args to perf_analyzer
+        """
+
+        if params and type(params) is dict:
+            for key in params:
+                self[key] = params[key]
+
+    @classmethod
+    def from_dict(cls, genai_perf_config_dict):
+        genai_perf_config = GenaiPerfConfig()
+        for key in [
+            "_args",
+        ]:
+            if key in genai_perf_config_dict:
+                setattr(genai_perf_config, key, genai_perf_config_dict[key])
+        return genai_perf_config
+
+    def representation(self):
+        """
+        Returns
+        -------
+        str
+            a string representation of the Genai Perf config
+            that removes values which can vary between
+            runs, but should be ignored when determining
+            if a previous (checkpointed) run can be used
+        """
+        cli_string = self.to_cli_string()
+
+        return cli_string
+
+    def to_cli_string(self) -> str:
+        """
+        Utility function to convert a config into a
+        string of arguments to the perf_analyzer with CLI.
+
+        Returns
+        -------
+        str
+            cli command string consisting of all arguments
+            to the perf_analyzer set in the config, without
+            the executable name.
+        """
+
+        # single dashed options, then verbose flags, then main args
+        args = []
+        args.extend(self._parse_options())
+
+        return " ".join(args)
+
+    def _parse_options(self):
+        """
+        Parse the genai perf args
+        """
+        temp_args = []
+        for key, value in self._args.items():
+            if key in self.boolean_args:
+                temp_args = self._parse_boolean_args(key, value, temp_args)
+            elif value:
+                temp_args.append(f"--{key}={value}")
+        return temp_args
+
+    def _parse_boolean_args(self, key, value, temp_args):
+        """
+        Parse genai perf args that should not add a value to the cli string
+        """
+        assert type(value) in [
+            str,
+            type(None),
+        ], f"Data type for arg {key} must be a (boolean) string instead of {type(value)}"
+        if value != None and value.lower() == "true":
+            temp_args.append(f"--{key}")
+        return temp_args
+
+    def __getitem__(self, key):
+        """
+        Gets an arguments value in config
+
+        Parameters
+        ----------
+        key : str
+            The name of the argument to the genai perf config
+
+        Returns
+        -------
+        object
+            The value that the argument is set to in this config
+
+        Raises
+        ------
+        KeyError
+            If argument not found in the config
+        """
+
+        if key in self._args:
+            return self._args[key]
+        else:
+            raise TritonModelAnalyzerException(
+                f"Key {key} does not exist in genai_perf_flags."
+            )
+
+    def __setitem__(self, key, value):
+        """
+        Sets an arguments value in config
+        after checking if defined/supported.
+
+        Parameters
+        ----------
+        key : str
+            The name of the argument in genai_perf
+        value : (any)
+            The value to which the argument is being set
+
+        Raises
+        ------
+        TritonModelAnalyzerException
+            If key is unsupported or undefined in the
+            config class
+        """
+
+        if key in self._args:
+            self._args[key] = value
+        else:
+            raise TritonModelAnalyzerException(
+                f"The argument '{key}' to the genai_perf "
+                "is not supported by model analyzer."
+            )
+
+    def __contains__(self, key):
+        """
+        Returns
+        -------
+        True if key is in perf_config i.e. the key is a
+        genai perf config argument
+        """
+
+        return key in GenaiPerfConfig.allowed_keys()