triton-model-analyzer 1.48.0__py3-none-any.whl

model_analyzer/monitor/dcgm/DcgmSystem.py

@@ -0,0 +1,412 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import model_analyzer.monitor.dcgm.pydcgm as pydcgm
+import model_analyzer.monitor.dcgm.dcgm_agent as dcgm_agent
+import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs
+import model_analyzer.monitor.dcgm.dcgm_fields as dcgm_fields
+import ctypes
+
+
+class DcgmSystemDiscovery:
+    '''
+    Constructor
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+    '''
+    Get all IDs of the GPUs that DCGM knows about. To get only GPUs that DCGM support,
+    use GetAllSupportedGpuIds().
+
+    Returns an array of GPU IDs. Each of these can be passed to DcgmGroup::AddGpu()
+    '''
+
+    def GetAllGpuIds(self):
+        gpuIds = dcgm_agent.dcgmGetAllDevices(self._dcgmHandle.handle)
+        return gpuIds
+
+    '''
+    Get all of IDs of the GPUs that DCGM supports. This will exclude unsupported
+    GPUs
+
+    Returns an array of GPU IDs. Each of these can be passed to DcgmGroup::AddGpu()
+    '''
+
+    def GetAllSupportedGpuIds(self):
+        gpuIds = dcgm_agent.dcgmGetAllSupportedDevices(self._dcgmHandle.handle)
+        return gpuIds
+
+    '''
+    Get some basic GPU attributes for a given GPU ID.
+
+    Returns a dcgm_structs.c_dcgmDeviceAttributes_v3() object for the given GPU
+    '''
+
+    def GetGpuAttributes(self, gpuId):
+        return dcgm_agent.dcgmGetDeviceAttributes(self._dcgmHandle.handle,
+                                                  gpuId)
+
+    '''
+    Get topology information for a given GPU ID
+
+    Returns a dcgm_structs.c_dcgmDeviceTopology_v1 structure representing the topology for the given GPU
+    '''
+
+    def GetGpuTopology(self, gpuId):
+        return dcgm_agent.dcgmGetDeviceTopology(self._dcgmHandle.handle, gpuId)
+
+    '''
+    Get all entityIds of the entities that DCGM knows about.
+
+    entityGroupId IN: DCGM_FE_? constant of the entity group to fetch the entities of
+    onlyActive    IN: Boolean as to whether to fetch entities that are supported by DCGM (True)
+                      or all entity IDs (False)
+
+    Returns an array of entity IDs. Each of these can be passed to DcgmGroup::AddEntity()
+    '''
+
+    def GetEntityGroupEntities(self, entityGroupId, onlySupported):
+        flags = 0
+        if onlySupported:
+            flags |= dcgm_structs.DCGM_GEGE_FLAG_ONLY_SUPPORTED
+        entityIds = dcgm_agent.dcgmGetEntityGroupEntities(
+            self._dcgmHandle.handle, entityGroupId, flags)
+        return entityIds
+
+    '''
+    Get the status of all of the NvLink links in the system.
+
+    Returns a dcgm_structs.c_dcgmNvLinkStatus_v3 object.
+    '''
+
+    def GetNvLinkLinkStatus(self):
+        return dcgm_agent.dcgmGetNvLinkLinkStatus(self._dcgmHandle.handle)
+
+    '''
+    From a bitmask of input gpu ids, return a bitmask of numGpus GPUs which identifies the topologically
+    closest GPUs to use for a single job. DCGM will consider CPU affinities and NVLink connection speeds
+    to determine the closest.
+    hintFlags can instruct DCGM to consider GPU health or not. By default, unhealthy GPUs are excluded from
+    consideration.
+    '''
+
+    def SelectGpusByTopology(self, inputGpuIds, numGpus, hintFlags):
+        return dcgm_agent.dcgmSelectGpusByTopology(self._dcgmHandle.handle,
+                                                   inputGpuIds, numGpus,
+                                                   hintFlags)
+
+
+class DcgmSystemIntrospect:
+    '''
+    Class to access the system-wide introspection modules of DCGM
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._handle = dcgmHandle
+        self.memory = DcgmSystemIntrospectMemory(dcgmHandle)
+        self.cpuUtil = DcgmSystemIntrospectCpuUtil(dcgmHandle)
+
+    def UpdateAll(self, waitForUpdate=True):
+        dcgm_agent.dcgmIntrospectUpdateAll(self._handle.handle, waitForUpdate)
+
+
+class DcgmSystemIntrospectMemory:
+    '''
+    Class to access information about the memory usage of DCGM itself
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+    def GetForHostengine(self, waitIfNoData=True):
+        '''
+        Retrieve the total amount of virtual memory that the hostengine process is currently using.
+        This measurement represents both the resident set size (what is currently in RAM) and
+        the swapped memory that belongs to the process.
+        
+        waitIfNoData:      wait for metadata to be updated if it's not available
+                      
+        Returns a dcgm_structs.c_dcgmIntrospectMemory_v1 object
+        Raises an exception for DCGM_ST_NO_DATA if no data is available yet and \ref waitIfNoData is False
+        '''
+        return dcgm_agent.dcgmIntrospectGetHostengineMemoryUsage(
+            self._dcgmHandle.handle, waitIfNoData)
+
+
+class DcgmSystemIntrospectCpuUtil:
+    '''
+    Class to access information about the CPU Utilization of DCGM
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+    def GetForHostengine(self, waitIfNoData=True):
+        '''
+        Get the current CPU Utilization of the hostengine process.
+        
+        waitIfNoData:      wait for metadata to be updated if it's not available
+                      
+        Returns a dcgm_structs.c_dcgmIntrospectCpuUtil_v1 object
+        Raises an exception for DCGM_ST_NO_DATA if no data is available yet and \ref waitIfNoData is False
+        '''
+        return dcgm_agent.dcgmIntrospectGetHostengineCpuUtilization(
+            self._dcgmHandle.handle, waitIfNoData)
+
+
+'''
+Class to encapsulate DCGM field-metadata requests
+'''
+
+
+class DcgmSystemFields:
+
+    def GetFieldById(self, fieldId):
+        '''
+        Get a field's metadata by its dcgm_fields.DCGM_FI_* field ID
+
+        fieldId: dcgm_fields.DCGM_FI_* field ID of the field
+
+        Returns a dcgm_fields.c_dcgm_field_meta_t struct on success or None on error.
+        '''
+        return dcgm_fields.DcgmFieldGetById(fieldId)
+
+    def GetFieldByTag(self, tag):
+        '''
+        Get a field's metadata by its tag name. Ex: 'brand'
+
+        tag: Tag name of the field
+
+        Returns a dcgm_fields.c_dcgm_field_meta_t struct on success or None on error.
+        '''
+        return dcgm_fields.DcgmFieldGetByTag(tag)
+
+
+'''
+Class to encapsulate DCGM module management and introspection
+'''
+
+
+class DcgmSystemModules:
+    '''
+    Constructor
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+    '''
+    Denylist a module from being loaded by DCGM.
+
+    moduleId a dcgm_structs.dcgmModuleId* ID of the module to denylist
+
+    Returns: Nothing.
+    Raises a DCGM_ST_IN_USE exception if the module was already loaded
+    '''
+
+    def Denylist(self, moduleId):
+        dcgm_agent.dcgmModuleDenylist(self._dcgmHandle.handle, moduleId)
+
+    '''
+    Get the statuses of all of the modules in DCGM
+
+    Returns: a dcgm_structs.c_dcgmModuleGetStatuses_v1 structure.
+    '''
+
+    def GetStatuses(self):
+        return dcgm_agent.dcgmModuleGetStatuses(self._dcgmHandle.handle)
+
+
+'''
+Class to encapsulate DCGM profiling
+'''
+
+
+class DcgmSystemProfiling:
+    '''
+    Constructor
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+    '''
+    Pause profiling activities in DCGM. This should be used when you are monitoring profiling fields
+    from DCGM but want to be able to still run developer tools like nvprof, nsight systems, and nsight compute.
+    Profiling fields start with DCGM_PROF_ and are in the field ID range 1001-1012.
+    
+    Call this API before you launch one of those tools and Resume() after the tool has completed.
+    
+    DCGM will save BLANK values while profiling is paused. 
+    Calling this while profiling activities are already paused is fine and will be treated as a no-op.
+    '''
+
+    def Pause(self):
+        return dcgm_agent.dcgmProfPause(self._dcgmHandle.handle)
+
+    '''
+    Resume profiling activities in DCGM that were previously paused with Pause().
+
+    Call this API after you have completed running other NVIDIA developer tools to reenable DCGM
+    profiling metrics.
+    
+    DCGM will save BLANK values while profiling is paused. 
+    
+    Calling this while profiling activities have already been resumed is fine and will be treated as a no-op.
+    '''
+
+    def Resume(self):
+        return dcgm_agent.dcgmProfResume(self._dcgmHandle.handle)
+
+
+'''
+Class to encapsulate global DCGM methods. These apply to a single DcgmHandle, provided to the constructor
+'''
+
+
+class DcgmSystem:
+    '''
+    Constructor
+
+    dcgmHandle is a pydcgm.DcgmHandle instance of the connection that will be used by all methods of this class
+    '''
+
+    def __init__(self, dcgmHandle):
+        self._dcgmHandle = dcgmHandle
+
+        #Child classes
+        self.discovery = DcgmSystemDiscovery(self._dcgmHandle)
+        self.introspect = DcgmSystemIntrospect(self._dcgmHandle)
+        self.fields = DcgmSystemFields()
+        self.modules = DcgmSystemModules(self._dcgmHandle)
+        self.profiling = DcgmSystemProfiling(self._dcgmHandle)
+
+    '''
+    Request that the host engine perform a field value update cycle. If the host
+    engine was starting in DCGM_OPERATION_MODE_MANUAL, calling this method is
+    the only way that field values will be updated.
+
+    Note that performing a field value update cycle does not update every field.
+    It only update fields that are newly watched or fields that haven't updated
+    in enough time to warrant updating again, based on their update frequency.
+
+    waitForUpdate specifies whether this function call should block until the
+    field value update loop is complete or not. Use True if you intend to query
+    values immediately after calling this.
+    '''
+
+    def UpdateAllFields(self, waitForUpdate):
+        ret = dcgm_agent.dcgmUpdateAllFields(self._dcgmHandle.handle,
+                                             waitForUpdate)
+        #Throw an exception on error
+        dcgm_structs._dcgmCheckReturn(ret)
+
+    '''
+    Get a DcgmGroup instance for the default all-GPUs group. This object is used to
+    perform operations on a group of GPUs. See DcgmGroup.py for details.
+
+    AddGpu() and RemoveGpu() operations are not allowed on the default group
+    '''
+
+    def GetDefaultGroup(self):
+        return pydcgm.DcgmGroup(self._dcgmHandle,
+                                groupId=dcgm_structs.DCGM_GROUP_ALL_GPUS)
+
+    '''
+    Get an instance of DcgmGroup with no GPUs. Call AddGpu() on the returned
+    object with GPU IDs from GetAllGpuIds() before performing actions on
+    the returned DcgmGroup instance.
+
+    groupName is the name of the group to create in the host engine. This name must be
+    unique.
+
+    Note: The group will be deleted from the host engine when the returned object goes out of scope
+    '''
+
+    def GetEmptyGroup(self, groupName):
+        return pydcgm.DcgmGroup(self._dcgmHandle, groupName=groupName)
+
+    '''
+    Get an instance of DcgmGroup populated with the gpuIds provided
+
+    groupName is the name of the group to create in the host engine. This name must be
+    unique.
+    gpuIds is the list of GPU IDs to add to the group
+
+    Note: The group will be deleted from the host engine when the returned object goes out of scope
+    '''
+
+    def GetGroupWithGpuIds(self, groupName, gpuIds):
+        newGroup = pydcgm.DcgmGroup(self._dcgmHandle, groupName=groupName)
+        for gpuId in gpuIds:
+            newGroup.AddGpu(gpuId)
+        return newGroup
+
+    '''
+    Get an instance of DcgmGroup populated with the provided entities
+
+    groupName is the name of the group to create in the host engine. This name must be
+    unique.
+    entities is the list of entity pairs (type and id) to add to the group
+
+    Note: The group will be deleted from the host engine when the returned object goes out of scope
+    '''
+
+    def GetGroupWithEntities(self, groupName, entities):
+        group = pydcgm.DcgmGroup(self._dcgmHandle, groupName=groupName)
+        for entity in entities:
+            group.AddEntity(entity.entityGroupId, entity.entityId)
+
+        return group
+
+    '''
+    Get ids of all DcgmGroups of GPUs. This returns a list containing the ids of the DcgmGroups.
+    '''
+
+    def GetAllGroupIds(self):
+        return dcgm_agent.dcgmGroupGetAllIds(self._dcgmHandle.handle)
+
+    '''
+    Get all all of the field groups in the system
+    '''
+
+    def GetAllFieldGroups(self):
+        return dcgm_agent.dcgmFieldGroupGetAll(self._dcgmHandle.handle)
+
+    '''
+    Get a field group's id by its name.
+
+    Returns: Field group ID if found
+             None if not found
+    '''
+
+    def GetFieldGroupIdByName(self, name):
+        allGroups = self.GetAllFieldGroups()
+        for i in range(0, allGroups.numFieldGroups):
+            if allGroups.fieldGroups[i].fieldGroupName == name:
+                return ctypes.c_void_p(allGroups.fieldGroups[i].fieldGroupId)
+
+        return None
+
+    def PauseTelemetryForDiag(self):
+        """Pause DCGM modules from updating field values."""
+        import dcgm_agent_internal
+        dcgm_agent_internal.dcgmPauseTelemetryForDiag(self._dcgmHandle.handle)
+
+    def ResumeTelemetryForDiag(self):
+        """Resume previously paused DCGM modules so that they can update field values."""
+        import dcgm_agent_internal
+        dcgm_agent_internal.dcgmResumeTelemetryForDiag(self._dcgmHandle.handle)

model_analyzer/monitor/dcgm/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/monitor/dcgm/common/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

model_analyzer/monitor/dcgm/common/dcgm_client_cli_parser.py

@@ -0,0 +1,194 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from os import environ
+import argparse
+import logging
+import sys
+
+
+###############################################################################
+def create_parser(
+    publish_port=8000,
+    interval=10,
+    name='the monitoring tool',  # Replace with 'prometheus', 'telegraf', etc.
+    field_ids=None,
+    log_file=None,
+    log_level='INFO',
+    dcgm_hostname=environ.get('DCGM_HOSTNAME') or 'localhost',
+):
+    '''
+    Create a parser that defaults to sane parameters.
+
+    The default parameters can be overridden through keyword arguments.
+
+    Note: if DCGM_HOSTNAME is set as an environment variable, it is used as
+    the default instead of localhost
+    '''
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p',
+        '--publish-port',
+        dest='publish_port',
+        type=int,
+        default=publish_port,
+        help='TCP port that the client should publish to. Default={}.'.format(
+            publish_port))
+    parser.add_argument(
+        '-i',
+        '--interval',
+        dest='interval',
+        type=int,
+        default=interval,
+        help=
+        'How often the client should retrieve new values from DCGM in seconds. Default={}.'
+        .format(interval))
+    parser.add_argument(
+        '-f',
+        '--field-ids',
+        dest='field_ids',
+        type=str,
+        default=field_ids,
+        help=
+        'Comma-separated list of field IDs that should be retrieved from DCGM. '
+        +
+        'The full list of available field IDs can be obtained from dcgm_fields.h, dcgm_fields.py, '
+        + 'or running \'dcgmi dmon -l\'.')
+    parser.add_argument(
+        '--log-file',
+        dest='logfile',
+        type=str,
+        default=log_file,
+        help=
+        'A path to a log file for recording what information is being sent to {}'
+        .format(name))
+    parser.add_argument(
+        '--log-level',
+        dest='loglevel',
+        type=str,
+        default=log_level,
+        help=
+        'Specify a log level to use for logging.\n\tCRITICAL (0) - log only critical errors that drastically affect execution'
+        +
+        '\n\tERROR (1) - Log any error in execution\n\tWARNING (2) - Log all warnings and errors that occur'
+        +
+        '\n\tINFO (3) - Log informational messages about program execution in addition to warnings and errors'
+        +
+        '\n\tDEBUG (4) - Log debugging information in addition to all information about execution'
+        + '\nDefault: {}'.format(log_level))
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '-n',
+        '--hostname',
+        dest='hostname',
+        type=str,
+        default=dcgm_hostname,
+        help=
+        'IP/hostname where the client should query DCGM for values. Default={} (all interfaces).'
+        .format(dcgm_hostname))
+    group.add_argument(
+        '-e',
+        '--embedded',
+        dest='embedded',
+        action='store_true',
+        help=
+        'Launch DCGM from within this process instead of connecting to nv-hostengine.'
+    )
+
+    return parser
+
+
+def add_custom_argument(parser, *args, **kwargs):
+    parser.add_argument(*args, **kwargs)
+
+
+###############################################################################
+def add_target_host_argument(name, parser, default_target='localhost'):
+    parser.add_argument(
+        '-t',
+        '--publish-hostname',
+        dest='publish_hostname',
+        type=str,
+        default=default_target,
+        help='The hostname at which the client will publish the readings to {}'.
+        format(name))
+
+
+###############################################################################
+def run_parser(parser):
+    '''
+    Run a parser created using create_parser
+    '''
+    return parser.parse_args()
+
+
+###############################################################################
+def get_field_ids(args):
+    # This indicates the user supplied a string, so we should override the
+    # default
+    if isinstance(args.field_ids, str):
+        tokens = args.field_ids.split(",")
+        field_ids = [int(token) for token in tokens]
+        return field_ids
+    # The default object should already be an array of ints. Just return it
+    else:
+        return args.field_ids
+
+
+###############################################################################
+def get_log_level(args):
+    levelStr = args.loglevel.upper()
+    if levelStr == '0' or levelStr == 'CRITICAL':
+        numeric_log_level = logging.CRITICAL
+    elif levelStr == '1' or levelStr == 'ERROR':
+        numeric_log_level = logging.ERROR
+    elif levelStr == '2' or levelStr == 'WARNING':
+        numeric_log_level = logging.WARNING
+    elif levelStr == '3' or levelStr == 'INFO':
+        numeric_log_level = logging.INFO
+    elif levelStr == '4' or levelStr == 'DEBUG':
+        numeric_log_level = logging.DEBUG
+    else:
+        print("Could not understand the specified --log-level '%s'" %
+              (args.loglevel))
+        args.print_help()
+        sys.exit(2)
+    return numeric_log_level
+
+
+###############################################################################
+def parse_command_line(name, default_port, add_target_host=False):
+    # Fields we accept raw from the CLI
+    FIELDS_AS_IS = ['publish_port', 'interval', 'logfile', 'publish_hostname']
+
+    parser = create_parser(
+        name=name,
+        publish_port=default_port,
+    )
+
+    if add_target_host:
+        add_target_host_argument(name, parser)
+
+    args = run_parser(parser)
+    field_ids = get_field_ids(args)
+    log_level = get_log_level(args)
+
+    args_as_dict = vars(args)
+    settings = {i: args_as_dict[i] for i in FIELDS_AS_IS}
+    settings['dcgm_hostname'] = None if args.embedded else args.hostname
+    settings['field_ids'] = field_ids
+    settings['log_level'] = log_level
+
+    return settings