wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/HTTPS/HTTPSAuthHandler.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Basic interface to HTTPS requests using ssl object managers, for python >= 2.7.9.
+See usage example in:
+src/python/WMCore/WMSpec/Steps/Executors/DQMUpload.py
+"""
+from __future__ import print_function, division
+
+import logging
+import ssl
+try:
+    # python2
+    import urllib2
+    import httplib
+    HTTPSHandler = urllib2.HTTPSHandler
+    HTTPSConnection = httplib.HTTPSConnection
+except:
+    # python3
+    import urllib.request
+    import http.client
+    HTTPSHandler = urllib.request.HTTPSHandler
+    HTTPSConnection = http.client.HTTPSConnection
+
+
+class HTTPSAuthHandler(HTTPSHandler):
+    """
+    HTTPS authentication class to provide a ssl context with the certificates.
+    """
+    def __init__(self, key=None, cert=None, capath='/etc/grid-security/certificates/', level=0):
+        self.logger = logging.getLogger(__name__)
+        if cert:
+            # then create a default ssl context manager to carry the credentials.
+            # It also loads the default CA certificates
+            self.ctx = ssl.create_default_context()
+            self.ctx.load_cert_chain(cert, keyfile=key)
+            self.ctx.load_verify_locations(None, capath)
+
+            self.logger.info("Found %d default trusted CA certificates.", len(self.ctx.get_ca_certs()))
+            ### DEBUG start ###
+            #for ca in self.ctx.get_ca_certs():
+            #    if 'CERN' in str(ca['subject']):
+            #        print("  %s" % str(ca['subject']))
+            ### DEBUG end ###
+            self.logger.info("SSL context manager created with the following settings:")
+            self.logger.info("  check_hostname : %s", self.ctx.check_hostname)  # default to True
+            self.logger.info("  options : %s", self.ctx.options)  # default to 2197947391
+            self.logger.info("  protocol : %s", self.ctx.protocol)  # default to 2 (PROTOCOL_SSLv23)
+            self.logger.info("  verify_flags : %s", self.ctx.verify_flags)  # default to 0 (VERIFY_DEFAULT)
+            self.logger.info("  verify_mode : %s", self.ctx.verify_mode)  # default to 2 (CERT_REQUIRED)
+            HTTPSHandler.__init__(self, debuglevel=level, context=self.ctx)
+        else:
+            self.logger.info("Certificate not provided for HTTPSHandler")
+            HTTPSHandler.__init__(self, debuglevel=level)
+
+    def get_connection(self, host, **kwargs):
+        if self.ctx:
+            return HTTPSConnection(host, context=self.ctx, **kwargs)
+        return HTTPSConnection(host)
+
+    def https_open(self, req):
+        """
+        Overwrite the default https_open.
+        """
+        self.logger.debug("%s method to %s", req.get_method(), req.get_full_url())
+        self.logger.debug("  with the following headers: %s", req.headers)
+        return self.do_open(self.get_connection, req)

WMCore/Services/LogDB/LogDB.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+LogDB provides functionality to post/search messages into LogDB.
+https://github.com/dmwm/WMCore/issues/5705
+"""
+# futures
+from builtins import object
+from future.utils import viewitems
+
+from future import standard_library
+standard_library.install_aliases()
+
+# standard modules
+import logging
+import re
+import threading
+from collections import defaultdict
+from http.client import HTTPException
+
+# project modules
+from WMCore.Lexicon import splitCouchServiceURL
+from WMCore.Services.LogDB.LogDBBackend import LogDBBackend
+
+
+def getLogDBInstanceFromThread():
+    """This function only gets to call when LogDB is instantiated before hand
+       All the WMComponentWorkers instatntiate LogDB automatically
+    """
+    myThread = threading.currentThread()
+    if not hasattr(myThread, "logdbClient") or not isinstance(myThread.logdbClient, LogDB):
+        # logdb is not set do anything
+        return None
+    return myThread.logdbClient
+
+
+class LogDB(object):
+    """
+    _LogDB_
+
+    LogDB object - interface to LogDB functionality.
+    """
+
+    def __init__(self, url, identifier, logger=None, **kwds):
+        self.logger = logger if logger else logging.getLogger()
+        self.url = url if url else 'https://cmsweb.cern.ch/couchdb/wmstats_logdb'
+        self.identifier = identifier if identifier else 'unknown'
+        self.default_user = "HEARTBEAT"
+
+        try:
+            self.thread_name = kwds.pop('thread_name')
+        except KeyError:
+            self.thread_name = threading.currentThread().getName()
+
+        self.user_pat = re.compile(r'^/[a-zA-Z][a-zA-Z0-9/\=\s()\']*\=[a-zA-Z0-9/\=\.\-_/#:\s\']*$')
+        self.agent = 0 if self.user_pat.match(self.identifier) else 1
+        couch_url, db_name = splitCouchServiceURL(self.url)
+        self.backend = LogDBBackend(couch_url, db_name, identifier,
+                                    self.thread_name, agent=self.agent, **kwds)
+        self.logger.info(self)
+
+    def __repr__(self):
+        "Return representation for class"
+        return "<LogDB(url=%s, identifier=%s, agent=%d)>" \
+               % (self.url, self.identifier, self.agent)
+
+    def post(self, request=None, msg="", mtype="comment"):
+        """Post new entry into LogDB for given request"""
+        res = 'post-error'
+        try:
+            if request is None:
+                request = self.default_user
+            if self.user_pat.match(self.identifier):
+                res = self.backend.user_update(request, msg, mtype)
+            else:
+                res = self.backend.agent_update(request, msg, mtype)
+        except HTTPException as ex:
+            msg = "Failed to post doc to LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+        except Exception as exc:
+            self.logger.error("LogDBBackend post API failed, error=%s", str(exc))
+        self.logger.debug("LogDB post request, res=%s", res)
+        return res
+
+    def get(self, request=None, mtype=None):
+        """Retrieve all entries from LogDB for given request"""
+        res = []
+        try:
+            if request is None:
+                request = self.default_user
+            if self.user_pat.match(self.identifier):
+                agent = False
+            else:
+                agent = True
+            for row in self.backend.get(request, mtype, agent=agent).get('rows', []):
+                request = row['doc']['request']
+                identifier = row['doc']['identifier']
+                thr = row['doc']['thr']
+                mtype = row['doc']['type']
+                for rec in row['doc']['messages']:
+                    rec.update({'request': request, 'identifier': identifier, 'thr': thr, 'type': mtype})
+                    res.append(rec)
+        except HTTPException as ex:
+            msg = "Failed to get doc from LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+            res = 'get-error'
+        except Exception as exc:
+            self.logger.error("LogDBBackend get API failed, error=%s", str(exc))
+            res = 'get-error'
+        self.logger.debug("LogDB get request, res=%s", res)
+        return res
+
+    def get_all_requests(self):
+        """Retrieve all entries from LogDB for given request"""
+        try:
+            results = self.backend.get_all_requests()
+            res = []
+            for row in results['rows']:
+                res.append(row["key"])
+        except Exception as exc:
+            self.logger.error("LogDBBackend get_all_requests API failed, error=%s", str(exc))
+            res = 'get-error'
+        self.logger.debug("LogDB get_all_requests request, res=%s", res)
+        return res
+
+    def delete(self, request=None, mtype=None, this_thread=False, agent=True):
+        """
+        Delete entry in LogDB for given request
+        if mtype == None - delete all the log for that request
+        mtype != None - only delete specified mtype
+        """
+        res = 'delete-error'
+        try:
+            if request is None:
+                request = self.default_user
+            res = self.backend.delete(request, mtype, this_thread, agent)
+        except HTTPException as ex:
+            msg = "Failed to delete doc in LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+        except Exception as ex:
+            self.logger.error("LogDBBackend delete API failed, error=%s", str(ex))
+        self.logger.debug("LogDB delete request, res=%s", res)
+        return res
+
+    def cleanup(self, thr, backend='local'):
+        """Clean-up back-end LogDB"""
+        docs = []
+        try:
+            docs = self.backend.cleanup(thr)
+        except Exception as exc:
+            self.logger.error('LogDBBackend cleanup API failed, backend=%s, error=%s', backend, str(exc))
+        return docs
+
+    def heartbeat_report(self):
+        report = defaultdict(dict)
+        if self.user_pat.match(self.identifier):
+            self.logger.error("User %s: doesn't allow this function", self.identifier)
+            return report
+
+        for row in self.backend.get(self.default_user, None, agent=True).get('rows', []):
+            identifier = row['doc']['identifier']
+            # wmstats thread can run in multiple boxed.
+            if self.identifier == identifier or identifier.startswith(self.identifier):
+                # this will handle wmstats DataCacheUpdate thread with multiple machine
+                postfix = identifier.replace(self.identifier, "")
+                thr = "%s%s" % (row['doc']['thr'], postfix)
+                mtype = row['doc']['type']
+                ts = row['doc']['messages'][-1]['ts']
+                msg = row['doc']['messages'][-1]['msg']
+                if (thr in report) and ('ts' in report[thr]) and ts <= report[thr]['ts']:
+                    continue
+                else:
+                    report[thr]['type'] = mtype
+                    report[thr]['msg'] = msg
+                    report[thr]['ts'] = ts
+        return report
+
+    def _append_down_component_detail(self, report, thr, msg, ts=0, state="error"):
+        report['down_components'].append(thr)
+        detail = {'name': thr, 'worker_name': thr, 'state': state,
+                  'last_error': ts, 'error_message': msg,
+                  'pid': 'N/A'}
+        report['down_component_detail'].append(detail)
+        return
+
+    def wmstats_down_components_report(self, thread_list):
+        report = {}
+        report['down_components'] = []
+        report['down_component_detail'] = []
+
+        hbinfo = self.heartbeat_report()
+        for thr in thread_list:
+            # skip DataCacheUpdate thread. It will have multiple with post fix.
+            # i.e. DataCacheUpdate-vocms111
+            # TODO, need a better way to check
+            if thr != "DataCacheUpdate" and thr not in hbinfo:
+                self._append_down_component_detail(report, thr, "Thread not running")
+
+        for thr, info in viewitems(hbinfo):
+            if info['type'] == 'agent-error':
+                self._append_down_component_detail(report, thr, info['msg'], info['ts'])
+        return report

WMCore/Services/LogDB/LogDBBackend.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+"""
+LogDBBackend
+
+Interface to LogDB persistent storage
+"""
+
+from builtins import object, bytes
+
+from Utils.PythonVersion import PY3
+from Utils.Utilities import encodeUnicodeToBytes, encodeUnicodeToBytesConditional
+
+# system modules
+import datetime
+import hashlib
+import time
+
+# WMCore modules
+from WMCore.Database.CMSCouch import CouchServer, CouchNotFoundError
+from WMCore.Services.LogDB.LogDBExceptions import LogDBError
+
+# define full list of supported LogDB types
+LOGDB_MSG_TYPES = ['info', 'error', 'warning', 'comment']
+
+def gen_hash(key):
+    "Generate hash for given key"
+    key = encodeUnicodeToBytes(key)  # if key is not unicode, then it is not changed
+    if not isinstance(key, bytes):
+        raise NotImplementedError
+    keyhash = hashlib.md5()
+    keyhash.update(encodeUnicodeToBytesConditional(key, condition=PY3))
+    return keyhash.hexdigest()
+
+def tstamp():
+    "Return timestamp with microseconds"
+    now = datetime.datetime.now()
+    base = str(time.mktime(now.timetuple())).split('.')[0]
+    ctime = '%s.%s' % (base, now.microsecond)
+    return float(ctime)
+
+def clean_entry(doc):
+    """Clean document from CouchDB attributes"""
+    for attr in ['_rev', '_id']:
+        if  attr in doc:
+            del doc[attr]
+    return doc
+
+class LogDBBackend(object):
+    """
+    Represents persistent storage for LogDB
+    """
+    def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
+        self.db_url = db_url
+        self.server = CouchServer(db_url)
+        self.db_name = db_name
+        self.dbid = identifier
+        self.thread_name = thread_name
+        self.agent = kwds.get('agent', 0)
+        self.db = self.server.connectDatabase(db_name, create=False)
+        self.design = 'LogDB' # name of design document
+        self.view = 'requests' # name of view to look-up requests
+        self.tsview = 'tstamp' # name of tsview to look-up requests
+        self.threadview = 'logByRequestAndThread'
+        self.requestview = 'logByRequest'
+
+    def deleteDatabase(self):
+        """Delete back-end database"""
+        if  self.db_name in self.server.listDatabases():
+            self.server.deleteDatabase(self.db_name)
+
+    def check(self, request, mtype=None):
+        """Check that given request name is valid"""
+        # TODO: we may add some logic to check request name, etc.
+        if  not request:
+            raise LogDBError("Request name is empty")
+        if  mtype and mtype not in LOGDB_MSG_TYPES:
+            raise LogDBError("Unsupported message type: '%s', supported types %s" \
+                    % (mtype, LOGDB_MSG_TYPES))
+
+    def docid(self, request, mtype):
+        """Generate doc id, we use double dash to avoid dashes from thread names"""
+        return gen_hash('--'.join((request, self.dbid, self.thread_name, mtype)))
+
+    def prefix(self, mtype):
+        """Generate agent specific prefix for given message type"""
+        if  self.agent:
+            # we add prefix for agent messages, all others will not have this index
+            mtype = 'agent-%s' % mtype
+        return mtype
+
+    def agent_update(self, request, msg='', mtype="info"):
+        """Update agent info in LogDB for given request"""
+        self.check(request, mtype)
+        mtype = self.prefix(mtype)
+        rec = {"ts":tstamp(), "msg":msg}
+        doc = {"_id": self.docid(request, mtype), "messages": [rec],
+                "request":request, "identifier":self.dbid,
+                "thr":self.thread_name, "type":mtype}
+        try:
+            exist_doc = self.db.document(doc["_id"])
+            doc["_rev"] = exist_doc["_rev"]
+        except CouchNotFoundError:
+            # this means document is not exist so we will just insert
+            pass
+        finally:
+            res = self.db.commitOne(doc)
+        return res
+
+    def user_update(self, request, msg, mtype='comment'):
+        """Update user info in LogDB for given request"""
+        rec = {"ts":tstamp(), "msg":msg}
+        doc = {"_id": self.docid(request, mtype), "messages": [rec],
+                "request":request, "identifier":self.dbid,
+                "thr":self.thread_name, "type":mtype}
+        try:
+            exist_doc = self.db.document(doc["_id"])
+            doc["_rev"] = exist_doc["_rev"]
+            doc["messages"] += exist_doc["messages"]
+        except CouchNotFoundError:
+            # this means document is not exist so we will just insert
+            pass
+        finally:
+            res = self.db.commitOne(doc)
+        return res
+
+    def get(self, request, mtype=None, detail=True, agent=True):
+        """Retrieve all entries from LogDB for given request"""
+        self.check(request, mtype)
+        if agent and mtype:
+            mtype = self.prefix(mtype)
+        options = {'reduce':False}
+        if mtype:
+            keys = [[request, mtype]]
+        else:
+            keys=[]
+            options.update({'startkey': [request], 'endkey':[request, {}]})
+        if detail:
+            options.update({'include_docs': True})
+        docs = self.db.loadView(self.design, self.view, options, keys=keys)
+        return docs
+
+    def get_by_thread(self, request, mtype='error', detail=False, agent=True):
+        self.check(request, mtype)
+        if agent and mtype:
+            mtype = self.prefix(mtype)
+        keys = [[request, self.dbid, self.thread_name, mtype]]
+        options = {'reduce':False}
+        if detail:
+            options.update({'include_docs': True})
+        docs = self.db.loadView(self.design, self.threadview, options, keys)
+        return docs
+
+    def get_by_request(self, request):
+        keys = [request]
+        options = {'reduce':False}
+        docs = self.db.loadView(self.design, self.requestview, options, keys)
+        return docs
+
+    def get_all_requests(self):
+        """Retrieve all entries from LogDB"""
+        options = {'reduce':True, 'group_level':1}
+        docs = self.db.loadView(self.design, self.view, options)
+        return docs
+
+    def delete(self, request, mtype=None, this_thread=False, agent=True):
+        """Delete entry in LogDB for given request"""
+        if mtype:
+            self.check(request, mtype)
+        else:
+            self.check(request)
+        if this_thread:
+            docs = self.get_by_thread(request, mtype=mtype, detail=False, agent=agent)
+        else:
+            docs = self.get(request, mtype=mtype, detail=False, agent=agent)
+        ids = [r['id'] for r in docs.get('rows', [])]
+        res = self.db.bulkDeleteByIDs(ids)
+        return res
+
+    def cleanup(self, thr):
+        """
+        Clean-up docs older then given threshold (thr should be specified in seconds).
+        This is done via tstamp view end endkey, e.g.
+        curl "http://127.0.0.1:5984/logdb/_design/LogDB/_view/tstamp?endkey=1427912282"
+        """
+        cutoff = int(round(time.time()-thr))
+        #docs = self.db.allDocs() # may need another view to look-up old docs
+        spec = {'endkey':cutoff, 'reduce':False}
+        docs = self.db.loadView(self.design, self.tsview, spec)
+        ids = [d['id'] for d in docs.get('rows', [])]
+        self.db.bulkDeleteByIDs(ids)
+        return ids

WMCore/Services/LogDB/LogDBExceptions.py

@@ -0,0 +1,11 @@
+"""LogDB Exceptions"""
+
+class LogDBError(Exception):
+    """Standard error baseclass"""
+    def __init__(self, error):
+        Exception.__init__(self, error)
+        self.msg = LogDBError.__class__.__name__
+        self.error = error
+
+    def __str__(self):
+        return "%s: %s" % (self.msg, self.error)

WMCore/Services/LogDB/LogDBReport.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : LogDBReport.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: LogDB report class to represent LogDB messages
+"""
+from __future__ import print_function
+
+from builtins import range
+from builtins import object
+
+class LogDBReport(object):
+    """LogDBReport class to represent LogDB messages"""
+    def __init__(self, logdb):
+        self.logdb = logdb
+
+    def docs(self, request):
+        """Fetch LogDB messages for given request"""
+        if  request == 'all':
+            docs = self.logdb.get_all_requests()
+        else:
+            docs = self.logdb.get(request)
+        return docs
+
+    def orderby(self, docs, order):
+        """Order LogDB messages by given type"""
+        odict = {}
+        for item in docs:
+            odict[item['ts']] = item
+        keys = sorted(odict.keys())
+        keys.reverse()
+        out = []
+        for key in keys:
+            out.append(odict[key])
+        return out
+
+    def to_json(self, request, order='ts'):
+        """Represent given messages in JSON data-format for given set of requests"""
+        docs = self.orderby(self.docs(request), order)
+        return docs
+
+    def to_txt(self, request, order='ts', sep=' '):
+        """Represent given messages in ASCII text format for given set of requests"""
+        docs = self.orderby(self.docs(request), order)
+        keys = list(docs[0])
+        out = sep.join(keys) + '\n'
+        for doc in docs:
+            values = []
+            for key in keys:
+                values.append('%s' % doc[key])
+            out += sep.join(values) + '\n'
+        return out
+
+    def to_html(self, request, order='ts'):
+        """Represent given messages in ASCII text format for given set of requests"""
+        out = '<table id="logdb-report">\n'
+        for doc in self.to_txt(request, order, sep='</td><td>').split('\n'):
+            if  doc:
+                out += '<tr><td>'+doc+'</td></tr>\n'
+        out += '</table>'
+        return out
+
+    def to_stdout(self, request, order='ts'):
+        """Yield to stdout LogDB messages for given request/type"""
+        msg = '\nReport for %s' % request
+        print(msg, '\n', '-'*len(msg))
+        docs = self.orderby(self.docs(request), order)
+        times = []
+        messages = []
+        mtypes = []
+        for doc in docs:
+            times.append(str(doc['ts']))
+            messages.append(doc['msg'])
+            mtypes.append(doc['type'])
+        tstpad = max([len(t) for t in times])
+        msgpad = max([len(m) for m in messages])
+        mtppad = max([len(m) for m in mtypes])
+        out = []
+        for idx in range(len(times)):
+            tcol = '%s%s' % (times[idx], ' '*(tstpad-len(times[idx])))
+            mcol = '%s%s' % (messages[idx], ' '*(msgpad-len(messages[idx])))
+            ecol = '%s%s' % (mtypes[idx], ' '*(mtppad-len(mtypes[idx])))
+            print("%s %s %s" % (tcol, mcol, ecol))

WMCore/Services/MSUtils/MSUtils.py

@@ -0,0 +1,54 @@
+"""
+This module contains a few basic WMCore-related utilitarian functions
+"""
+
+from Utils.CertTools import ckey, cert
+
+from WMCore.Services.pycurl_manager import RequestHandler
+
+
+def makeHttpRequest(srvUrl, payload=None, method='GET', headers=None, encode=True, decode=True, payloadSizeToDump=50):
+    """
+    Perform HTTP call to provided service URL. This can be any of supported methods like
+    GET, POST, PUT, DELETE. It will return HTTP response from upstream MicroService
+    and parse it accordingly (the WMCore MS service returns results as {'result': {'data':...}}
+    HTTP response.
+
+    :param srvUrl: string with the service url
+    :param payload: payload query or document, in case of GET HTTP request it is query dictionary,
+        in case of POST/PUT HTTP requests it is JSON payload to the service
+    :param method: string, defines which HTTP method to use
+    :param headers: HTTP headers (presented as dictionarY)
+    :param encode: boolean to reflect if data should be encoded, see pycurl_manager.py
+    :param decode: boolean to reflect if data should be decoded, see pycurl_manager.py
+    :param payloadSizeToDump: amount of characters to dump in a log from payload (to make log entries readable)
+    :return: returns a list with all objects, or raises
+        an exception in case of failure
+    """
+    payload = payload or {}
+    mgr = RequestHandler()
+    headers = headers or {'Content-Type': 'application/json'}
+    data = mgr.getdata(srvUrl, payload, headers, verb=method,
+                       ckey=ckey(), cert=cert(), encode=encode, decode=decode)
+    if data and data.get("result", []):
+        if "error" in data["result"][0]:
+            # strip off part of payload to make readable log message
+            sdata = f"{payload}"
+            if len(sdata) > payloadSizeToDump:
+                sdata = sdata[:payloadSizeToDump] + "..."
+            msg = f"Failed to contact {srvUrl} via {method} request with {sdata}"
+            msg += f" and error message: {data}"
+            raise RuntimeError(msg)
+    return data["result"]
+
+def getPileupDocs(mspileupUrl, queryDict=None, method='GET'):
+    """
+    Fetch documents from MSPileup according to the query passed in using POST.
+
+    :param mspileupUrl: string with the MSPileup url
+    :param queryDict: dictionary with the MongoDB query to run
+    :param method: string, defines which HTTP method to use
+    :return: returns a list with all the pileup objects, or raises
+        an exception in case of failure
+    """
+    return makeHttpRequest(mspileupUrl, payload=queryDict, method=method)