triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/record/types/cpu_used_ram.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.record import DecreasingRecord
+
+
+@total_ordering
+class CPUUsedRAM(DecreasingRecord):
+    """
+    The CPU memory usage record
+    """
+
+    tag = "cpu_used_ram"
+
+    def __init__(self, value, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            CPU used memory
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+
+        super().__init__(value, timestamp)
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Max " if aggregation_tag else "") + "RAM Usage (MB)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is better than
+        the other
+        """
+
+        return self.value() > other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return CPUUsedRAM(value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return CPUUsedRAM(value=(other.value() - self.value()))

model_analyzer/record/types/gpu_free_memory.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.gpu_record import IncreasingGPURecord
+
+
+@total_ordering
+class GPUFreeMemory(IncreasingGPURecord):
+    """
+    The free memory in the GPU.
+    """
+
+    tag = "gpu_free_memory"
+
+    def __init__(self, value, device_uuid=None, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The value of the GPU metrtic
+        device_uuid : str
+            The  GPU device uuid this metric is associated
+            with.
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+
+        super().__init__(value, device_uuid, timestamp)
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Max " if aggregation_tag else "") + "GPU Memory Available (MB)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return GPUFreeMemory(device_uuid=None, value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return GPUFreeMemory(device_uuid=None, value=(self.value() - other.value()))

model_analyzer/record/types/gpu_power_usage.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.gpu_record import DecreasingGPURecord
+
+
+@total_ordering
+class GPUPowerUsage(DecreasingGPURecord):
+    """
+    GPU Power Usage
+    """
+
+    tag = "gpu_power_usage"
+
+    def __init__(self, value, device_uuid=None, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float-compatible value
+            The value of the GPU metrtic
+        device_uuid : str
+            The  GPU device uuid this metric is associated
+            with.
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+        value = float(value)
+        super().__init__(value, device_uuid, timestamp)
+
+    @staticmethod
+    def aggregation_function():
+        """
+        The function that is used to aggregate
+        this type of record
+        """
+
+        def average(seq):
+            return sum(seq[1:], start=seq[0]) / len(seq)
+
+        return average
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed as part of the header
+            indicating that this record has been aggregated using max, min or
+            average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Average " if aggregation_tag else "") + "GPU Power Usage (W)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return other.value() < self.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return GPUPowerUsage(device_uuid=None, value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return GPUPowerUsage(device_uuid=None, value=(other.value() - self.value()))

model_analyzer/record/types/gpu_total_memory.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.gpu_record import IncreasingGPURecord
+
+
+@total_ordering
+class GPUTotalMemory(IncreasingGPURecord):
+    """
+    The total memory in the GPU.
+    """
+
+    tag = "gpu_total_memory"
+
+    def __init__(self, value, device_uuid=None, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The value of the GPU metrtic
+        device_uuid : str
+            The  GPU device uuid this metric is associated
+            with.
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+
+        super().__init__(value, device_uuid, timestamp)
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Max " if aggregation_tag else "") + "GPU Memory Available (MB)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return GPUTotalMemory(device_uuid=None, value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return GPUTotalMemory(device_uuid=None, value=(self.value() - other.value()))

model_analyzer/record/types/gpu_used_memory.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.gpu_record import DecreasingGPURecord
+
+
+@total_ordering
+class GPUUsedMemory(DecreasingGPURecord):
+    """
+    The used memory in the GPU.
+    """
+
+    tag = "gpu_used_memory"
+
+    def __init__(self, value, device_uuid=None, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The value of the GPU metrtic
+        device_uuid : str
+            The  GPU device uuid this metric is associated
+            with.
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+
+        super().__init__(value, device_uuid, timestamp)
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Max " if aggregation_tag else "") + "GPU Memory Usage (MB)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() > other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return GPUUsedMemory(device_uuid=None, value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return GPUUsedMemory(device_uuid=None, value=(other.value() - self.value()))

model_analyzer/record/types/gpu_utilization.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.gpu_record import IncreasingGPURecord
+
+
+@total_ordering
+class GPUUtilization(IncreasingGPURecord):
+    """
+    GPU utilization record
+    """
+
+    tag = "gpu_utilization"
+
+    def __init__(self, value, device_uuid=None, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            The value of the GPU metrtic
+        device_uuid : str
+            The  GPU device uuid this metric is associated
+            with.
+        timestamp : int
+            The timestamp for the record in nanoseconds
+        """
+
+        super().__init__(value, device_uuid, timestamp)
+
+    @staticmethod
+    def aggregation_function():
+        """
+        The function that is used to aggregate
+        this type of record
+        """
+
+        def average(seq):
+            return sum(seq[1:], start=seq[0]) / len(seq)
+
+        return average
+
+    @staticmethod
+    def header(aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return ("Average " if aggregation_tag else "") + "GPU Utilization (%)"
+
+    def __eq__(self, other):
+        """
+        Allows checking for
+        equality between two records
+        """
+
+        return self.value() == other.value()
+
+    def __lt__(self, other):
+        """
+        Allows checking if
+        this record is less than
+        the other
+        """
+
+        return self.value() < other.value()
+
+    def __add__(self, other):
+        """
+        Allows adding two records together
+        to produce a brand new record.
+        """
+
+        return GPUUtilization(device_uuid=None, value=(self.value() + other.value()))
+
+    def __sub__(self, other):
+        """
+        Allows subtracting two records together
+        to produce a brand new record.
+        """
+
+        return GPUUtilization(device_uuid=None, value=(self.value() - other.value()))

model_analyzer/record/types/inter_token_latency_avg.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from functools import total_ordering
+
+from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase
+
+
+@total_ordering
+class InterTokenLatencyAvg(InterTokenLatencyBase):
+    """
+    A record for perf_analyzer Inter token latency metric
+    """
+
+    tag = "inter_token_latency_avg"
+
+    def __init__(self, value, timestamp=0):
+        """
+        Parameters
+        ----------
+        value : float
+            the latency extracted from the perf analyzer output
+        timestamp : float
+            Elapsed time from start of program
+        """
+
+        super().__init__(value, timestamp)
+
+    @classmethod
+    def header(cls, aggregation_tag=False):
+        """
+        Parameters
+        ----------
+        aggregation_tag: bool
+            An optional tag that may be displayed
+            as part of the header indicating that
+            this record has been aggregated using
+            max, min or average etc.
+
+        Returns
+        -------
+        str
+            The full name of the
+            metric.
+        """
+
+        return "Avg Inter Token Latency (ms)"