wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/Services/PyCondor/PyCondorAPI.py

@@ -0,0 +1,154 @@
+"""
+_PyCondorAPI_
+
+Class used to interact with Condor daemons on the agent
+"""
+
+from __future__ import print_function, division
+
+from builtins import str, object
+from past.builtins import basestring
+import logging
+try:
+    # This module has dependency with python binding for condor package (condor)
+    import htcondor
+except ImportError:
+    pass
+
+
+class PyCondorAPI(object):
+    """
+    Some APIs to interact with HTCondor via the HTCondor python bindings.
+    """
+    def __init__(self):
+        self.schedd = htcondor.Schedd()
+        self.coll = htcondor.Collector()
+
+    def recreateSchedd(self):
+        """
+        In case our current schedd object is in a "strange" state, we
+        better recreate it.
+        """
+        logging.warning("Recreating Schedd instance due to query error...")
+        self.schedd = htcondor.Schedd()
+
+    def getCondorJobsSummary(self):
+        """
+        Retrieves a job summary from the HTCondor Schedd object
+        :return: a list of classads representing the matching jobs, or None if failed
+        """
+        jobs = None  # return None to signalize the query failed
+        queryOpts = htcondor.htcondor.QueryOpts.SummaryOnly
+        try:
+            return self.schedd.query(opts=queryOpts)
+        except Exception:
+            self.recreateSchedd()
+
+        try:
+            jobs = self.schedd.query(opts=queryOpts)
+        except Exception as ex:
+            logging.exception("Failed to fetch summary of jobs from Condor Schedd. Error: %s", str(ex))
+        return jobs
+
+    def getCondorJobs(self, constraint='true', attrList=None, limit=-1, opts="Default"):
+        """
+        Given a job/schedd constraint, return a list of job classad.
+        :param constraint: the query constraint (str or ExprTree). Defaults to 'true'
+        :param attrList: a list of attribute strings to be returned in the call.
+                         It defaults to all attributes.
+        :param limit: a limit on the number of matches to return. Defaults to -1 (all)
+        :param opts: string with additional flags for the query. Defaults to Default.
+            https://htcondor.readthedocs.io/en/v8_9_7/apis/python-bindings/api/htcondor.html#htcondor.QueryOpts
+        :return: returns an iterator to the job classads
+        """
+        attrList = attrList or []
+        # if option parameter is invalid, default it to the standard behavior
+        opts = getattr(htcondor.htcondor.QueryOpts, opts, "Default")
+        msg = "Querying condor schedd with params: constraint=%s, attrList=%s, limit=%s, opts=%s"
+        logging.info(msg, constraint, attrList, limit, opts)
+        try:
+            return self.schedd.query(constraint, attrList, limit, opts=opts)
+        except Exception:
+            self.recreateSchedd()
+
+        # if we hit another exception, let it be raised up in the chain
+        return self.schedd.query(constraint, attrList, limit, opts=opts)
+
+    def editCondorJobs(self, job_spec, attr, value):
+        """
+        _editCondorJobs_
+
+        Edit a set of condor jobs given an attribute and value
+        job_spec can be a list of job IDs or a string specifying a constraint
+        """
+        success = False
+        try:
+            self.schedd.edit(job_spec, attr, value)
+            success = True
+        except Exception as ex:
+            # edit doesn't distinguish between an error and not matching any jobs
+            # check for this message and assume it just didn't match any jobs
+            if isinstance(ex, RuntimeError) and str(ex) == "Unable to edit jobs matching constraint":
+                success = True
+                msg = "Condor constraint did not match any jobs. "
+                msg += "Message from schedd: %s" % str(ex)
+                logging.info(msg)
+            else:
+                msg = "Condor failed to edit the jobs. "
+                msg += "Error message: %s" % str(ex)
+                logging.exception(msg)
+
+        return success
+
+    def isScheddOverloaded(self):
+        """
+        check whether job limit is reached in local schedd.
+        Condition is check by following logic.
+        ( ShadowsRunning > 9.700000000000000E-01 * MAX_RUNNING_JOBS) )
+        || ( RecentDaemonCoreDutyCycle > 9.800000000000000E-01 )
+        """
+        try:
+            scheddAd = self.coll.locate(htcondor.DaemonTypes.Schedd)
+            q = self.coll.query(htcondor.AdTypes.Schedd, 'Name == "%s"' % scheddAd['Name'],
+                                projection=['CurbMatchmaking'])[0]
+            isOverloaded = q['CurbMatchmaking'].eval()
+            return isOverloaded
+        except Exception:
+            # if there is an error, try to recreate the collector instance
+            logging.info("Recreating Collector instance due to query error...")
+            self.coll = htcondor.Collector()
+        try:
+            scheddAd = self.coll.locate(htcondor.DaemonTypes.Schedd)
+            q = self.coll.query(htcondor.AdTypes.Schedd, 'Name == "%s"' % scheddAd['Name'],
+                                projection=['CurbMatchmaking'])[0]
+            isOverloaded = q['CurbMatchmaking'].eval()
+        except Exception as ex:
+            msg = "Condor failed to fetch schedd attributes."
+            msg += "Error message: %s" % str(ex)
+            logging.exception(msg)
+            # since it failed, assume it's overloaded
+            isOverloaded = True
+
+        return isOverloaded
+
+
+def getScheddParamValue(param):
+    """
+    _getScheddParamValue_
+
+    Given a schedd parameter, retrieve it's value with htcondor, e.g.:
+    MAX_JOBS_RUNNING, MAX_JOBS_PER_OWNER, etc
+    """
+    paramResult = None
+    if not isinstance(param, basestring):
+        logging.error("Parameter %s must be string type", param)
+        return paramResult
+
+    try:
+        paramResult = htcondor.param[param]
+    except Exception as ex:
+        msg = "Condor failed to fetch schedd parameter: %s" % param
+        msg += "Error message: %s" % str(ex)
+        logging.exception(msg)
+
+    return paramResult

WMCore/Services/PyCondor/PyCondorUtils.py

@@ -0,0 +1,105 @@
+from __future__ import print_function, division
+
+from builtins import str, object
+
+from future.utils import viewitems
+
+import logging
+import os
+import pickle
+import signal
+import time
+import traceback
+
+import htcondor
+
+
+class OutputObj(object):
+    """
+    Class used when AuthenticatedSubprocess is created with pickleOut
+    It stores the output message of the subprocess, any output object provided
+    and extra information for debugging purposes, such as the environment
+    """
+
+    def __init__(self, outputMessage, outputObj):
+        self.outputMessage = outputMessage
+        self.outputObj = outputObj
+        self.environmentStr = ""
+        for key, val in viewitems(os.environ):
+            self.environmentStr += "%s=%s\n" % (key, val)
+
+
+class AuthenticatedSubprocess(object):
+    """
+    Context manager for execution of condor commands in a forked, especially useful for
+    those commands where a different proxy credential is needed.
+    """
+
+    def __init__(self, proxy=None, pickleOut=False, outputObj=None, logger=logging):
+        """
+        Basic setup of the context manager
+        :param proxy: optional path to the proxy file
+        :param pickleOut: boolean flag which enables pickled output data
+        :param outputObj: structure storing the actual output of the htcondor call
+        :param logger: logger object
+        """
+        self.proxy = proxy
+        self.pickleOut = pickleOut
+        self.outputObj = outputObj
+        self.timedout = False
+        self.logger = logger
+
+    def __enter__(self):
+        self.r, self.w = os.pipe()
+        self.rpipe = os.fdopen(self.r, 'r')
+        self.wpipe = os.fdopen(self.w, 'w')
+        self.pid = os.fork()
+        if self.pid == 0 and self.proxy:
+            # CRAB case
+            htcondor.SecMan().invalidateAllSessions()
+            htcondor.param['SEC_CLIENT_AUTHENTICATION_METHODS'] = 'FS,GSI'
+            htcondor.param['DELEGATE_FULL_JOB_GSI_CREDENTIALS'] = 'true'
+            htcondor.param['DELEGATE_JOB_GSI_CREDENTIALS_LIFETIME'] = '0'
+            os.environ['X509_USER_PROXY'] = self.proxy
+            self.rpipe.close()
+        elif self.pid == 0:
+            self.rpipe.close()
+        else:
+            self.wpipe.close()
+        return self.pid, self.rpipe
+
+    def __exit__(self, a, b, c):
+        if self.pid == 0:
+            if a is None and b is None and c is None:
+                if self.pickleOut:
+                    oo = OutputObj("OK", self.outputObj)
+                    self.wpipe.write(pickle.dumps(oo))
+                else:
+                    self.wpipe.write("OK")
+                self.wpipe.close()
+                os._exit(0)
+            else:
+                tracebackString = str('\n'.join(traceback.format_tb(c)))
+                msg = "Trapped exception in AuthenticatedSubprocess.Fork: %s %s %s \n%s" % \
+                      (a, b, c, tracebackString)
+                if self.pickleOut:
+                    oo = OutputObj(msg, self.outputObj)
+                    self.wpipe.write(pickle.dumps(oo))
+                else:
+                    self.wpipe.write(msg)
+                self.wpipe.close()
+                os._exit(1)
+        else:
+            timestart = time.time()
+            self.timedout = True
+            while (time.time() - timestart) < 3600:
+                res = os.waitpid(self.pid, os.WNOHANG)
+                if res != (0, 0):
+                    self.timedout = False
+                    break
+                time.sleep(0.100)
+            if self.timedout:
+                self.logger.warning(
+                    "Subprocess with PID %s (executed in AuthenticatedSubprocess) timed out. Killing it." % self.pid)
+                os.kill(self.pid, signal.SIGTERM)
+                # we should probably wait again and send SIGKILL if the kill does not work

WMCore/Services/ReqMgr/ReqMgr.py

@@ -0,0 +1,261 @@
+from builtins import str, bytes
+from future.utils import viewitems
+
+import json
+import logging
+
+from WMCore.Services.Service import Service
+from WMCore.Cache.GenericDataCache import MemoryCacheStruct
+
+
+class ReqMgr(Service):
+    """
+    API for dealing with retrieving information from RequestManager dataservice
+
+    """
+
+    def __init__(self, url, header=None, logger=None):
+        """
+        responseType will be either xml or json
+        """
+
+        httpDict = {}
+        header = header or {}
+        # url is end point
+        httpDict['endpoint'] = "%s/data" % url
+        httpDict['logger'] = logger if logger else logging.getLogger()
+
+        # cherrypy converts request.body to params when content type is set
+        # application/x-www-form-urlencodeds
+        httpDict.setdefault("content_type", 'application/json')
+        httpDict.setdefault('cacheduration', 0)
+        httpDict.setdefault("accept_type", "application/json")
+        httpDict.update(header)
+        self.encoder = json.dumps
+        Service.__init__(self, httpDict)
+        # This is only for the unittest: never set it true unless it is unittest
+        self._noStale = False
+
+    def _getResult(self, callname, clearCache=True, args=None, verb="GET",
+                   encoder=json.loads, decoder=json.loads, contentType=None):
+        """
+        _getResult_
+        """
+        result = ''
+        cfile = callname.replace("/", "_")
+        if clearCache:
+            self.clearCache(cfile, args, verb)
+
+        f = self.refreshCache(cfile, callname, args, encoder=encoder,
+                              verb=verb, contentType=contentType)
+        result = f.read()
+        f.close()
+
+        if result and decoder:
+            result = decoder(result)
+
+        return result['result']
+
+    def _createQuery(self, queryDict):
+        """
+        _createQuery
+        :param queryDict: is the format of {name: values} fair. value can be sting, int or list
+        :type queryDict: dict
+        :returns: url query string
+        """
+        if self._noStale:
+            args = "_nostale=true&"
+        else:
+            args = ""
+        for name, values in viewitems(queryDict):
+            if isinstance(values, (str, bytes, int)):
+                values = [values]
+            for val in values:
+                args += '%s=%s&' % (name, val)
+
+        return args.rstrip('&')
+
+    def getParentLocks(self):
+        """
+        _getParentLocks_
+
+        A public method to return the parent locks from ReqMgr.
+        :returns: A list of datasets
+
+        """
+        callname = 'parentlocks'
+        result = self._getResult(callname, clearCache=True, verb="GET")
+        return result[0][callname]
+
+    def getRequestByNames(self, names):
+
+        """
+        _getRequestByNames_
+
+        :param names: list or sting of request name(s)
+        :type statusList: list, str
+        :returns:  list of dict or list of request names depending on the detail value
+         -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
+                                                                    'BlockWhitelist': [],
+                                                                    'CMSSWVersion': 'CMSSW_4_4_2_patch2',
+                                                                    ....
+                                                                    '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
+                                                                    'inputMode': 'couchDB'}}]
+        TODO: need proper error handling if status is not 200 from orignal reporting.
+        """
+
+        query = self._createQuery({'name': names})
+        callname = 'request?%s' % query
+        return self._getResult(callname, verb="GET")
+
+    def getRequestByStatus(self, statusList, detail=True):
+        """
+        _getRequestByStatus_
+
+        :param statusList: list of status
+        :type statusList: list
+        :param detail: boolean of request list.
+        :type detail: boolean
+        :returns:  list of dict or list of request names depending on the detail value
+         -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
+                                                                    'BlockWhitelist': [],
+                                                                    'CMSSWVersion': 'CMSSW_4_4_2_patch2',
+                                                                    ....
+                                                                    '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
+                                                                    'inputMode': 'couchDB'}}]
+        TODO: need proper error handling if status is not 200 from orignal reporting.
+        """
+
+        query = self._createQuery({'status': statusList, 'detail': detail})
+        callname = 'request?%s' % query
+        return self._getResult(callname, verb="GET")
+
+    def getRequestTasks(self, name):
+
+        """
+        _getRequestTasks_
+
+        :param name: request name
+        :type string: str
+        :returns:  list of dict or list of request names depending on the detail value
+         -- {result:[{requestTask}, {requestTask}]}
+        """
+
+        callname = 'splitting/%s' % name
+        return self._getResult(callname, verb="GET")
+
+    def getConfig(self, name):
+
+        """
+        _getConfig_
+
+        :param name: request name
+        :type string: str
+        :returns:  list of dict or list of request names depending on the detail value
+         -- {result:[config_string]}
+        """
+
+        callname = 'workload_config/%s' % name
+        return self._getResult(callname, verb="GET")
+
+    def insertRequests(self, requestDict):
+        """
+        _insertRequests_
+
+        :param requestDict: request argument dictionary
+        :type requestDict: dict
+        :returns:  list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
+                                                                    'BlockWhitelist': [],
+                                                                    'CMSSWVersion': 'CMSSW_4_4_2_patch2',
+                                                                    ....
+                                                                    '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
+                                                                    'inputMode': 'couchDB'}}]
+        TODO: need proper error handling if status is not 200 from orignal reporting.
+        """
+        return self["requests"].post('request', requestDict)[0]['result']
+
+    def updateRequestStatus(self, request, status):
+        """
+        _updateRequestStatus_
+
+        :param request: request(workflow name)
+        :type reqeust: str
+        :param status: status of workflow to update (i.e. 'assigned')
+        :type status: str
+        :returns:  list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
+                                                                    'BlockWhitelist': [],
+                                                                    'CMSSWVersion': 'CMSSW_4_4_2_patch2',
+                                                                    ....
+                                                                    '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
+                                                                    'inputMode': 'couchDB'}}]
+        TODO: need proper error handling if status is not 200 from orignal reporting.
+        """
+
+        status = {"RequestStatus": status}
+        status["RequestName"] = request
+        return self["requests"].put('request', status)[0]['result']
+
+    def updateRequestStats(self, request, stats):
+        """
+        put initial stats for request
+        param: stats: dict of {'total_jobs': 100, 'input_lumis': 100,
+                               'input_events': 100, 'input_num_files': 100}
+        specific to ReqMgr app. not implemented for T0
+        """
+        # this stats dict will be validated on the server side
+        self.updateRequestProperty(request, stats)
+
+    def updateRequestProperty(self, request, propDict):
+        """
+        _updateRequestProperty_
+        :param request: request(workflow name)
+        :type reqeust: str
+        :param propDict: request property with key value -- {"SiteWhitelist": ["ABC"], "SiteBlacklist": ["A"], "RequestStatus": "assigned"}
+        :type propDict: dict
+        :returns:  list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
+                                                                    'BlockWhitelist': [],
+                                                                    'CMSSWVersion': 'CMSSW_4_4_2_patch2',
+                                                                    ....
+                                                                    '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
+                                                                    'inputMode': 'couchDB'}}]
+        """
+        propDict["RequestName"] = request
+        return self["requests"].put('request/%s' % request, propDict)[0]['result']
+
+    def getAbortedAndForceCompleteRequestsFromMemoryCache(self, expire=0):
+        """
+        _getAbortedAndForceCompleteRequestsFromMemoryCache_
+        """
+
+        maskStates = ["aborted", "aborted-completed", "force-complete"]
+        return self.getRequestByStatusFromMemoryCache(maskStates, expire)
+
+    def getRequestByStatusFromMemoryCache(self, statusList, expire=0):
+
+        return MemoryCacheStruct(expire=expire, func=self.getRequestByStatus, initCacheValue=[],
+                                 logger=self['logger'], kwargs={'statusList': statusList, "detail": False})
+
+    def cloneRequest(self, requestName, overwrittenParams=None):
+        """
+        _cloneRequest_
+
+        :param requestName: request name which need to be cloned
+        :type reqeust: str
+        :param overwrittenParams: status of workflow to update (i.e. 'assigned')
+        :type overwrittenParams: dict (dictionary format of parameters need to be overwritten)
+        :returns list of dict [{'request': cloned_request_name}]
+        """
+        params = overwrittenParams or {}
+        uri = 'request/clone/%s' % requestName
+        return self["requests"].post(uri, params)[0]['result']
+
+    def getGenericRequestInfo(self, queryDict):
+        """
+        _getGenericRequestInfo_
+        :param queryDict: query dictionary which will be converted to url query string.
+        :type queryDict: dict
+        :returns list of dict [{'request': { any info request by mask}]
+        """
+        query = self._createQuery(queryDict)
+        callname = 'request?%s' % query
+        return self._getResult(callname, verb="GET")