wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/McM/McM.py

@@ -0,0 +1,173 @@
+#! /bin/env python
+
+"""
+A service class for retrieving data from McM using
+an SSO cookie since it sits behind CERN SSO
+"""
+
+from builtins import str, object
+import json
+import os
+import pycurl
+import subprocess
+import logging
+from typing import List
+from io import BytesIO
+from WMCore.WMException import WMException
+
+
+class McMNoDataError(WMException):
+    """
+    _McMNoDataError_
+        McM responded but has no data for the request
+    """
+
+    def __init__(self):
+        WMException.__init__(self, 'McM responded correctly but has no data')
+
+
+class McM(object):
+    """
+    A service class for retrieving data from McM using
+    an SSO cookie since it sits behind CERN SSO
+    'key' must be unencrypted
+    """
+
+    def __init__(self, url='https://cms-pdmv.cern.ch/mcm', tmpDir='/tmp'):
+        self.url = url
+        self.tmpDir = tmpDir
+        self.logger = self._get_logger()
+        self.cookieFile = None
+
+    def __enter__(self):
+        self._krb_ticket()
+        self._get_cookie()
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        if self.cookieFile:
+            try:
+                os.remove(self.cookieFile)
+            except OSError:
+                return
+        return
+
+    def _get_logger(self) -> logging.Logger:
+        """
+        Create a logger for McM client
+        """
+        logger: logging.Logger = logging.getLogger("mcm_client")
+        date_format: str = "%Y-%m-%d %H:%M:%S %z"
+        format: str = "[%(levelname)s][%(name)s][%(asctime)s]: %(message)s"
+        formatter: logging.Formatter = logging.Formatter(fmt=format, datefmt=date_format)
+        handler: logging.StreamHandler = logging.StreamHandler()
+
+        handler.setLevel(logging.INFO)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        return logger
+
+    def _krb_ticket(self) -> None:
+        """
+        Check there is a valid Kerberos ticket for requesting a SSO
+        cookie. Raise a RuntimeError if there is not one.
+        """
+        process = subprocess.Popen(["klist", "-f"],
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
+        stdout: str = process.communicate()[0].decode("utf-8")
+        if process.returncode != 0:
+            msg: str = ("There is no valid Kerberos ticket for requesting a SSO cookie. "
+                        "Please make sure to provide one for your runtime environment"
+                        )
+            self.logger.error(msg)
+            raise RuntimeError("FATAL -- %s\nError msg: %s" % (msg, stdout))
+
+    def _get_cookie(self) -> None:
+        """
+        Request a SSO cookie to authenticate to McM.
+        """
+        self.cookieFile = os.path.join(self.tmpDir, 'ssoCookie.txt')
+        command: List[str] = ["auth-get-sso-cookie", "-u", self.url, "-o", self.cookieFile, "-vv"]
+        callback_not_invoked: str = "DEBUG: Not automatically redirected: trying SAML authentication"
+        cookie_stored: str = "INFO: Saving cookies"
+
+        process = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                   stderr=subprocess.STDOUT, shell=False)
+        stdout: str = process.communicate()[0].decode("utf-8")
+        stdout_list: List[str] = stdout.strip().split("\n")
+        if process.returncode != 0:
+            raise RuntimeError("FATAL -- Error requesting SSO cookie\nError msg: %s" % (stdout))
+
+        if callback_not_invoked in stdout:
+            callback_idx: int = stdout_list.index(callback_not_invoked)
+            stored_after_issue: bool = cookie_stored in stdout_list[callback_idx + 1]
+            if stored_after_issue:
+                msg: str = ("Callback method was not invoked. "
+                            "Please make sure the provided Kerberos ticket is not linked to an account with 2FA"
+                            )
+                raise RuntimeError(msg)
+
+    def _getURL(self, extendURL):
+        """
+        Fetch an MCM URL with CURL using the SSO cookie.
+        Only intended to be used internally
+        """
+
+        try:
+            b = BytesIO()
+            c = pycurl.Curl()
+
+            fullUrl = '%s/%s' % (self.url, extendURL)
+
+            c.setopt(c.URL, fullUrl)
+            c.setopt(c.SSL_VERIFYPEER, False)
+            c.setopt(c.SSL_VERIFYHOST, False)
+            c.setopt(c.FOLLOWLOCATION, True)
+            c.setopt(c.COOKIEJAR, self.cookieFile)
+            c.setopt(c.COOKIEFILE, self.cookieFile)
+            c.setopt(c.WRITEFUNCTION, b.write)
+            c.perform()
+            if c.getinfo(pycurl.HTTP_CODE) != 200:
+                raise IOError
+        except:
+            c.close()
+            raise IOError('Was not able to fetch or decode URL from McM')
+
+        try:
+            body = b.getvalue()
+            res = json.loads(body)
+        except ValueError:
+            c.close()
+            raise IOError('Was not able to decode JSON from McM')
+
+        c.close()
+        return res
+
+    def getHistory(self, prepID):
+        """
+        Get the history record which has who did what to an McM request
+        """
+
+        try:
+            url = 'search?db_name=batches&contains=%s&get_raw' % prepID
+            res = self._getURL(url)
+            history = res.get("results", [])[0].get("history", [])
+            return history
+        except IndexError:
+            raise McMNoDataError
+
+    def getRequest(self, prepID):
+        """
+        Get the request record which has, among other things,
+        the number of requested events
+        """
+
+        url = 'public/restapi/requests/get/%s' % prepID
+        res = self._getURL(url)
+        return res['results']
+
+
+if __name__ == '__main__':
+    with McM() as mcm:
+        history = mcm.getHistory(prepID='BTV-Upg2023SHCAL14DR-00002')
+        request = mcm.getRequest(prepID='BTV-Upg2023SHCAL14DR-00002')

WMCore/Services/McM/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_McM_
+
+API for McM
+
+"""
+__all__ = []

WMCore/Services/MonIT/Grafana.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# coding=utf-8
+"""
+Wrapper class, based on the PyCuRL module, providing an interface
+to CERN MonIT Grafana APIs
+"""
+from __future__ import division, print_function, absolute_import
+from builtins import str, object
+from future import standard_library
+standard_library.install_aliases()
+
+import logging
+import json
+from copy import copy
+from pprint import pformat
+from urllib.parse import urljoin
+
+from WMCore.Services.pycurl_manager import RequestHandler
+
+class Grafana(object):
+    """
+    Service class providing functionality to CERN monitoring Grafana APIs
+    """
+
+    def __init__(self, token, configDict=None):
+        """
+        Constructs a Grafana object.
+        :param token: mandatory string with the key/token string
+        :param configDict: dictionary with extra parameters, such as:
+          logger: logger object
+          endpoint: string with the url/endpoint to be used
+          cacheduration: float with the cache duration, in hours
+          headers: dictionary with the headers to be used
+        """
+        self._token = token
+        self.configDict = configDict or {}
+        self.configDict.setdefault('endpoint', "https://monit-grafana.cern.ch")
+        self.configDict.setdefault('cacheduration', 0)  # in hours
+        if 'headers' not in self.configDict:
+            self.configDict.setdefault('headers', {})
+            self.configDict['headers'].update({"Accept": "application/json"})
+            self.configDict['headers'].update({"Content-Type": "application/json"})
+
+        self.logger = self.configDict.get("logger", logging.getLogger())
+        self.logger.info("MonIT service initialized with parameters: %s", self.configDict)
+
+    def updateToken(self, token):
+        """
+        Update token to be used in requests made through this module
+        :param token: string with the new token
+        """
+        self._token = token
+
+    def _postRequest(self, url, params, verb='POST', verbose=0):
+        "Helper function to POST request to given URL"
+        mgr = RequestHandler(logger=self.logger)
+        headers = copy(self.configDict['headers'])
+        headers.update({"Authorization": self._token})
+
+        try:
+            data = mgr.getdata(url, params, headers, verb=verb, verbose=verbose)
+            return json.loads(data)
+        except Exception as exc:
+            self.logger.error("Failed to retrieve data from MonIT. Error: %s", str(exc))
+            return None
+
+    def getAPIData(self, apiName, queryStr=''):
+        """
+        _getAPIData_
+
+        Retrieve data from a given Grafana API and index name
+        :param apiName: string with the API name/number
+        :param queryStr: string with the whole query logic
+        :return: data as retrieved from Grafana
+        """
+        uri = urljoin(self.configDict['endpoint'], "/api/datasources/proxy/%s/_msearch" % apiName)
+        return self._postRequest(uri, queryStr)
+
+    def getSSBData(self, pathName, metricName, indexName="monit_prod_cmssst_*",
+                   apiName="9475", queryStr=''):
+        """
+        _getSSBData_
+
+        Retrieve data from the SSB index in Grafana
+        :param pathName: string with the path to the correct metric, mapped from SSB as:
+            columnid=237   ProdStatus   path="sts15min"  data.prod_status
+            columnid=160   CPU bound    path="scap15min" data.core_cpu_intensive
+            columnid=161   I/O bound    path="scap15min" data.core_io_intensive
+            columnid=159   Prod Cores   path="scap15min" data.core_production
+            columnid=136   Real Cores   path="scap15min" data.core_max_used
+            columnid=107   Tape Pledge  path="scap15min" data.tape_pledge
+        :param indexName: optional string with the grafana index name
+        :param apiName: optional string with the Grafana API number
+        :param queryStr: optional string with the full query logic
+        :return: dictionary with the site names and the metric value
+        """
+        # TODO: allow flexible time range instead of the hardwired last 24h
+        # TODO: allow flexible number of rows, instead of the hardwired 500
+        results = {}
+        if metricName not in ('prod_status', 'core_cpu_intensive', 'core_io_intensive',
+                              'core_production', 'core_max_used', 'tape_pledge'):
+            self.logger.error("SSB metric name '%s' is NOT supported.", metricName)
+            return results
+
+        uri = urljoin(self.configDict['endpoint'], "/api/datasources/proxy/%s/_msearch" % apiName)
+        if not queryStr:
+            ### NOTE: these '\n' new lines are mandatory to get it working...
+            queryStr = '{"search_type":"query_then_fetch","ignore_unavailable":true,"index":["%s"]}\n' % indexName
+            queryStr += '{"size":500,"query":{"bool":{"filter":[{"range":{"metadata.timestamp":{"gte":"now-1d","lte":"now","format":"epoch_millis"}}},' \
+                        '{"query_string":{"analyze_wildcard":true,' \
+                        '"query":"metadata.type: ssbmetric AND metadata.type_prefix:raw AND metadata.monit_hdfs_path: %s"}}]}},' \
+                        '"sort":{"metadata.timestamp":{"order":"desc","unmapped_type":"boolean"}},' \
+                        '"script_fields":{},"docvalue_fields":["metadata.timestamp"]}\n' % pathName
+        self.logger.info("Calling API: %s with query: %s", uri, pformat(queryStr))
+
+        data = self._postRequest(uri, queryStr)
+        #self.logger.debug("Data retrieved: %s", pformat(data))
+        if data:
+            # now parse the ugly output
+            hits = data['responses'][0]['hits']['hits']
+            for item in hits:
+                site = item['_source']['data']['name']
+                metricValue = item['_source']['data'][metricName]
+                tStamp = int(item['sort'][0])
+
+                results.setdefault(site, {})
+                if tStamp > results[site].get("timeStamp", 0):
+                    results[site][metricName] = metricValue
+                    results[site]["timeStamp"] = tStamp
+
+            data = results
+
+        return data

WMCore/Services/PyCondor/PyCondorAPI.py

@@ -0,0 +1,154 @@
+"""
+_PyCondorAPI_
+
+Class used to interact with Condor daemons on the agent
+"""
+
+from __future__ import print_function, division
+
+from builtins import str, object
+from past.builtins import basestring
+import logging
+try:
+    # This module has dependency with python binding for condor package (condor)
+    import htcondor2 as htcondor
+except ImportError:
+    pass
+
+
+class PyCondorAPI(object):
+    """
+    Some APIs to interact with HTCondor via the HTCondor python bindings.
+    """
+    def __init__(self):
+        self.schedd = htcondor.Schedd()
+        self.coll = htcondor.Collector()
+
+    def recreateSchedd(self):
+        """
+        In case our current schedd object is in a "strange" state, we
+        better recreate it.
+        """
+        logging.warning("Recreating Schedd instance due to query error...")
+        self.schedd = htcondor.Schedd()
+
+    def getCondorJobsSummary(self):
+        """
+        Retrieves a job summary from the HTCondor Schedd object
+        :return: a list of classads representing the matching jobs, or None if failed
+        """
+        jobs = None  # return None to signalize the query failed
+        queryOpts = htcondor.htcondor.QueryOpts.SummaryOnly
+        try:
+            return self.schedd.query(opts=queryOpts)
+        except Exception:
+            self.recreateSchedd()
+
+        try:
+            jobs = self.schedd.query(opts=queryOpts)
+        except Exception as ex:
+            logging.exception("Failed to fetch summary of jobs from Condor Schedd. Error: %s", str(ex))
+        return jobs
+
+    def getCondorJobs(self, constraint='true', attrList=None, limit=-1, opts="Default"):
+        """
+        Given a job/schedd constraint, return a list of job classad.
+        :param constraint: the query constraint (str or ExprTree). Defaults to 'true'
+        :param attrList: a list of attribute strings to be returned in the call.
+                         It defaults to all attributes.
+        :param limit: a limit on the number of matches to return. Defaults to -1 (all)
+        :param opts: string with additional flags for the query. Defaults to Default.
+            https://htcondor.readthedocs.io/en/v8_9_7/apis/python-bindings/api/htcondor.html#htcondor.QueryOpts
+        :return: returns an iterator to the job classads
+        """
+        attrList = attrList or []
+        # if option parameter is invalid, default it to the standard behavior
+        opts = getattr(htcondor.htcondor.QueryOpts, opts, "Default")
+        msg = "Querying condor schedd with params: constraint=%s, attrList=%s, limit=%s, opts=%s"
+        logging.info(msg, constraint, attrList, limit, opts)
+        try:
+            return self.schedd.query(constraint, attrList, limit, opts=opts)
+        except Exception:
+            self.recreateSchedd()
+
+        # if we hit another exception, let it be raised up in the chain
+        return self.schedd.query(constraint, attrList, limit, opts=opts)
+
+    def editCondorJobs(self, job_spec, attr, value):
+        """
+        _editCondorJobs_
+
+        Edit a set of condor jobs given an attribute and value
+        job_spec can be a list of job IDs or a string specifying a constraint
+        """
+        success = False
+        try:
+            self.schedd.edit(job_spec, attr, value)
+            success = True
+        except Exception as ex:
+            # edit doesn't distinguish between an error and not matching any jobs
+            # check for this message and assume it just didn't match any jobs
+            if isinstance(ex, RuntimeError) and str(ex) == "Unable to edit jobs matching constraint":
+                success = True
+                msg = "Condor constraint did not match any jobs. "
+                msg += "Message from schedd: %s" % str(ex)
+                logging.info(msg)
+            else:
+                msg = "Condor failed to edit the jobs. "
+                msg += "Error message: %s" % str(ex)
+                logging.exception(msg)
+
+        return success
+
+    def isScheddOverloaded(self):
+        """
+        check whether job limit is reached in local schedd.
+        Condition is check by following logic.
+        ( ShadowsRunning > 9.700000000000000E-01 * MAX_RUNNING_JOBS) )
+        || ( RecentDaemonCoreDutyCycle > 9.800000000000000E-01 )
+        """
+        try:
+            scheddAd = self.coll.locate(htcondor.DaemonTypes.Schedd)
+            q = self.coll.query(htcondor.AdTypes.Schedd, 'Name == "%s"' % scheddAd['Name'],
+                                projection=['CurbMatchmaking'])[0]
+            isOverloaded = q['CurbMatchmaking'].eval(q)
+            return isOverloaded
+        except Exception:
+            # if there is an error, try to recreate the collector instance
+            logging.info("Recreating Collector instance due to query error...")
+            self.coll = htcondor.Collector()
+        try:
+            scheddAd = self.coll.locate(htcondor.DaemonTypes.Schedd)
+            q = self.coll.query(htcondor.AdTypes.Schedd, 'Name == "%s"' % scheddAd['Name'],
+                                projection=['CurbMatchmaking'])[0]
+            isOverloaded = q['CurbMatchmaking'].eval(q)
+        except Exception as ex:
+            msg = "Condor failed to fetch schedd attributes."
+            msg += "Error message: %s" % str(ex)
+            logging.exception(msg)
+            # since it failed, assume it's overloaded
+            isOverloaded = True
+
+        return isOverloaded
+
+
+def getScheddParamValue(param):
+    """
+    _getScheddParamValue_
+
+    Given a schedd parameter, retrieve it's value with htcondor, e.g.:
+    MAX_JOBS_RUNNING, MAX_JOBS_PER_OWNER, etc
+    """
+    paramResult = None
+    if not isinstance(param, basestring):
+        logging.error("Parameter %s must be string type", param)
+        return paramResult
+
+    try:
+        paramResult = htcondor.param[param]
+    except Exception as ex:
+        msg = "Condor failed to fetch schedd parameter: %s" % param
+        msg += "Error message: %s" % str(ex)
+        logging.exception(msg)
+
+    return paramResult