wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/Services/DBS/ProdException.py

@@ -0,0 +1,133 @@
+#!/usr/bin/python
+"""
+_ProdException_
+
+General Exception class for Prod modules
+
+"""
+
+from future.utils import viewitems
+
+try:
+    import exceptions
+except ImportError:
+    import builtins as exceptions
+
+import inspect
+import logging
+
+
+class ProdException(exceptions.Exception):
+    """
+    _ProdException_
+
+    Exception class which works out details of where
+    it was raised.
+
+    """
+    def __init__(self, message, errorNo=1000,**data):
+        self.name = str(self.__class__.__name__)
+        exceptions.Exception.__init__(self, self.name,
+                                      message)
+
+        #  //
+        # // Init data dictionary with defaults
+        #//
+        self.data = {}
+        self.data.setdefault("ClassName", None)
+        self.data.setdefault("ModuleName", None)
+        self.data.setdefault("MethodName", None)
+        self.data.setdefault("ClassInstance", None)
+        self.data.setdefault("FileName", None)
+        self.data.setdefault("LineNumber", None)
+        if errorNo==None:
+            self.data.setdefault("ErrorNr",0)
+        else:
+            self.data.setdefault("ErrorNr",errorNo)
+
+        self.message = message
+        self.data.update(data)
+
+        #  //
+        # // Automatically determine the module name
+        #//  if not set
+        if self.data['ModuleName'] == None:
+            frame = inspect.currentframe()
+            lastframe = inspect.getouterframes(frame)[1][0]
+            excepModule = inspect.getmodule(lastframe)
+            if excepModule != None:
+                modName = excepModule.__name__
+                self.data['ModuleName'] = modName
+
+
+        #  //
+        # // Find out where the exception came from
+        #//
+        stack = inspect.stack(1)[1]
+        self.data['FileName'] = stack[1]
+        self.data['LineNumber'] = stack[2]
+        self.data['MethodName'] = stack[3]
+
+        #  //
+        # // ClassName if ClassInstance is passed
+        #//
+        if self.data['ClassInstance'] != None:
+            self.data['ClassName'] = \
+              self.data['ClassInstance'].__class__.__name__
+
+        logging.error(str(self))
+
+    def __getitem__(self, key):
+        """
+        make exception look like a dictionary
+        """
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        """
+        make exception look like a dictionary
+        """
+        self.data[key] = value
+
+    def addInfo(self, **data):
+        """
+        _addInfo_
+
+        Add key=value information pairs to an
+        exception instance
+        """
+        for key, value in viewitems(data):
+            self.data[key] = value
+        return
+
+    def xml(self):
+        """create a xml string rep of this exception"""
+        strg ="<Exception>\n"
+        strg +="<Object>\n"
+        strg += "%s\n" % self.name
+        strg +="</Object>\n"
+        strg +="<Message>\n"
+        strg += self.message
+        strg +="</Message>\n"
+        strg +="<DataItems>\n"
+        for key, value in viewitems(self.data):
+            strg +="<DataItem>\n"
+            strg += "<Key>\n"
+            strg += str(key)
+            strg += "</Key>\n"
+            strg += "<Value>\n"
+            strg += str(value)
+            strg += "</Value>\n"
+            strg +="</DataItem>\n"
+        strg +="</DataItems>\n"
+        strg +="</Exception>\n"
+        logging.error(strg)
+        return strg
+
+    def __str__(self):
+        """create a string rep of this exception"""
+        strg = "%s\n" % self.name
+        strg += "Message: %s\n" % self.message
+        for key, value in viewitems(self.data):
+            strg += "\t%s : %s\n" % (key, value, )
+        return strg

WMCore/Services/DBS/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_DBSUpload.DBSInterface_
+
+DBS APIs for DBSUpload
+
+"""
+__all__ = []

WMCore/Services/FWJRDB/FWJRDBAPI.py

@@ -0,0 +1,118 @@
+from __future__ import (division, print_function)
+
+from builtins import str, bytes, object
+
+from WMCore.Database.CMSCouch import CouchServer, Database
+from WMCore.Lexicon import splitCouchServiceURL
+
+class FWJRDBAPI(object):
+
+    def __init__(self, couchURL, dbName=None):
+        """
+        setting up comon variables for inherited class.
+        inherited class should call this in their init function
+        """
+        if isinstance(couchURL, Database):
+            self.couchDB = couchURL
+            self.couchURL = self.couchDB['host']
+            self.dbName = self.couchDB.name
+            self.couchServer = CouchServer(self.couchURL)
+        else:
+            if dbName == None:
+                self.couchURL, self.dbName = splitCouchServiceURL(couchURL)
+            else:
+                self.couchURL = couchURL
+                self.dbName = dbName
+            self.couchServer = CouchServer(self.couchURL)
+            self.couchDB = self.couchServer.connectDatabase(self.dbName, False)
+        self.couchapp = "FWJRDump"
+        self.defaultStale = {"stale": "update_after"}
+
+
+    def setDefaultStaleOptions(self, options):
+        if not options:
+            options = {}
+        if 'stale' not in options:
+            options.update(self.defaultStale)
+        return options
+
+    def _setNoStale(self):
+        """
+        Use this only for the unittest
+        """
+        self.defaultStale = {}
+
+    def _getCouchView(self, view, options, keys = []):
+
+        options = self.setDefaultStaleOptions(options)
+
+        if keys and isinstance(keys, (str, bytes)):
+            keys = [keys]
+        return self.couchDB.loadView(self.couchapp, view, options, keys)
+
+
+    def _filterCouchInfo(self, couchInfo):
+        # remove the couch specific information
+        for key in ['_rev', '_attachments']:
+            if  key in couchInfo:
+                del couchInfo[key]
+        return
+
+
+    def _formatCouchData(self, data, key = "id", detail = True, filterCouch = True,
+                         returnDict = False):
+        result = {}
+        for row in data['rows']:
+            if 'error' in row:
+                continue
+            if "doc" in row:
+                if filterCouch:
+                    self._filterCouchInfo(row["doc"])
+                result[row[key]] = row["doc"]
+            else:
+                result[row[key]] = row["value"]
+        if detail or returnDict:
+            return result
+        else:
+            return list(result)
+
+    def getFWJRByArchiveStatus(self, status, limit=None, skip=None):
+        """
+        'status': list of the status or status string
+        """
+        options = {}
+        options["include_docs"] = True
+
+        if limit != None:
+            options["limit"] = limit
+        if skip != None:
+            options["skip"] = skip
+        keys = status
+        return self._getCouchView("reportsByArchiveStatus", options, keys)
+
+    def updateArchiveUploadedStatus(self, docID):
+
+        return self.couchDB.updateDocument(docID, self.couchapp, "archiveStatus")
+
+    def isAllFWJRArchived(self, workflow):
+        keys = [[workflow, "ready"]]
+        options = {"reduce": True, "group": True}
+        result = self._getCouchView("byWorkflowAndArchiveStatus", options, keys)
+        if len(result["rows"]) == 0:
+            return True
+        else:
+            return False
+
+
+    def outputByWorkflowName(self):
+
+        options = {"group": True, "stale": "ok", "reduce": True}
+        # site of data should be relatively small (~1M) for put in the memory
+        # If not, find a way to stream
+        return self._getCouchView("outputByWorkflowName", options)
+
+
+    def getFWJRWithSkippedFiles(self):
+        options = {"reduce": True, "group": True, "include_docs": False}
+        return self._getCouchView("skippedFileInfoByTaskAndSite", options)
+

WMCore/Services/HTTPS/HTTPSAuthHandler.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Basic interface to HTTPS requests using ssl object managers, for python >= 2.7.9.
+See usage example in:
+src/python/WMCore/WMSpec/Steps/Executors/DQMUpload.py
+"""
+from __future__ import print_function, division
+
+import logging
+import ssl
+try:
+    # python2
+    import urllib2
+    import httplib
+    HTTPSHandler = urllib2.HTTPSHandler
+    HTTPSConnection = httplib.HTTPSConnection
+except:
+    # python3
+    import urllib.request
+    import http.client
+    HTTPSHandler = urllib.request.HTTPSHandler
+    HTTPSConnection = http.client.HTTPSConnection
+
+
+class HTTPSAuthHandler(HTTPSHandler):
+    """
+    HTTPS authentication class to provide a ssl context with the certificates.
+    """
+    def __init__(self, key=None, cert=None, capath='/etc/grid-security/certificates/', level=0):
+        self.logger = logging.getLogger(__name__)
+        if cert:
+            # then create a default ssl context manager to carry the credentials.
+            # It also loads the default CA certificates
+            self.ctx = ssl.create_default_context()
+            self.ctx.load_cert_chain(cert, keyfile=key)
+            self.ctx.load_verify_locations(None, capath)
+
+            self.logger.info("Found %d default trusted CA certificates.", len(self.ctx.get_ca_certs()))
+            ### DEBUG start ###
+            #for ca in self.ctx.get_ca_certs():
+            #    if 'CERN' in str(ca['subject']):
+            #        print("  %s" % str(ca['subject']))
+            ### DEBUG end ###
+            self.logger.info("SSL context manager created with the following settings:")
+            self.logger.info("  check_hostname : %s", self.ctx.check_hostname)  # default to True
+            self.logger.info("  options : %s", self.ctx.options)  # default to 2197947391
+            self.logger.info("  protocol : %s", self.ctx.protocol)  # default to 2 (PROTOCOL_SSLv23)
+            self.logger.info("  verify_flags : %s", self.ctx.verify_flags)  # default to 0 (VERIFY_DEFAULT)
+            self.logger.info("  verify_mode : %s", self.ctx.verify_mode)  # default to 2 (CERT_REQUIRED)
+            HTTPSHandler.__init__(self, debuglevel=level, context=self.ctx)
+        else:
+            self.logger.info("Certificate not provided for HTTPSHandler")
+            HTTPSHandler.__init__(self, debuglevel=level)
+
+    def get_connection(self, host, **kwargs):
+        if self.ctx:
+            return HTTPSConnection(host, context=self.ctx, **kwargs)
+        return HTTPSConnection(host)
+
+    def https_open(self, req):
+        """
+        Overwrite the default https_open.
+        """
+        self.logger.debug("%s method to %s", req.get_method(), req.get_full_url())
+        self.logger.debug("  with the following headers: %s", req.headers)
+        return self.do_open(self.get_connection, req)

WMCore/Services/LogDB/LogDB.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+LogDB provides functionality to post/search messages into LogDB.
+https://github.com/dmwm/WMCore/issues/5705
+"""
+# futures
+from builtins import object
+from future.utils import viewitems
+
+from future import standard_library
+standard_library.install_aliases()
+
+# standard modules
+import logging
+import re
+import threading
+from collections import defaultdict
+from http.client import HTTPException
+
+# project modules
+from WMCore.Lexicon import splitCouchServiceURL
+from WMCore.Services.LogDB.LogDBBackend import LogDBBackend
+
+
+def getLogDBInstanceFromThread():
+    """This function only gets to call when LogDB is instantiated before hand
+       All the WMComponentWorkers instatntiate LogDB automatically
+    """
+    myThread = threading.currentThread()
+    if not hasattr(myThread, "logdbClient") or not isinstance(myThread.logdbClient, LogDB):
+        # logdb is not set do anything
+        return None
+    return myThread.logdbClient
+
+
+class LogDB(object):
+    """
+    _LogDB_
+
+    LogDB object - interface to LogDB functionality.
+    """
+
+    def __init__(self, url, identifier, logger=None, **kwds):
+        self.logger = logger if logger else logging.getLogger()
+        self.url = url if url else 'https://cmsweb.cern.ch/couchdb/wmstats_logdb'
+        self.identifier = identifier if identifier else 'unknown'
+        self.default_user = "HEARTBEAT"
+
+        try:
+            self.thread_name = kwds.pop('thread_name')
+        except KeyError:
+            self.thread_name = threading.currentThread().getName()
+
+        self.user_pat = re.compile(r'^/[a-zA-Z][a-zA-Z0-9/\=\s()\']*\=[a-zA-Z0-9/\=\.\-_/#:\s\']*$')
+        self.agent = 0 if self.user_pat.match(self.identifier) else 1
+        couch_url, db_name = splitCouchServiceURL(self.url)
+        self.backend = LogDBBackend(couch_url, db_name, identifier,
+                                    self.thread_name, agent=self.agent, **kwds)
+        self.logger.info(self)
+
+    def __repr__(self):
+        "Return representation for class"
+        return "<LogDB(url=%s, identifier=%s, agent=%d)>" \
+               % (self.url, self.identifier, self.agent)
+
+    def post(self, request=None, msg="", mtype="comment"):
+        """Post new entry into LogDB for given request"""
+        res = 'post-error'
+        try:
+            if request is None:
+                request = self.default_user
+            if self.user_pat.match(self.identifier):
+                res = self.backend.user_update(request, msg, mtype)
+            else:
+                res = self.backend.agent_update(request, msg, mtype)
+        except HTTPException as ex:
+            msg = "Failed to post doc to LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+        except Exception as exc:
+            self.logger.error("LogDBBackend post API failed, error=%s", str(exc))
+        self.logger.debug("LogDB post request, res=%s", res)
+        return res
+
+    def get(self, request=None, mtype=None):
+        """Retrieve all entries from LogDB for given request"""
+        res = []
+        try:
+            if request is None:
+                request = self.default_user
+            if self.user_pat.match(self.identifier):
+                agent = False
+            else:
+                agent = True
+            for row in self.backend.get(request, mtype, agent=agent).get('rows', []):
+                request = row['doc']['request']
+                identifier = row['doc']['identifier']
+                thr = row['doc']['thr']
+                mtype = row['doc']['type']
+                for rec in row['doc']['messages']:
+                    rec.update({'request': request, 'identifier': identifier, 'thr': thr, 'type': mtype})
+                    res.append(rec)
+        except HTTPException as ex:
+            msg = "Failed to get doc from LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+            res = 'get-error'
+        except Exception as exc:
+            self.logger.error("LogDBBackend get API failed, error=%s", str(exc))
+            res = 'get-error'
+        self.logger.debug("LogDB get request, res=%s", res)
+        return res
+
+    def get_all_requests(self):
+        """Retrieve all entries from LogDB for given request"""
+        try:
+            results = self.backend.get_all_requests()
+            res = []
+            for row in results['rows']:
+                res.append(row["key"])
+        except Exception as exc:
+            self.logger.error("LogDBBackend get_all_requests API failed, error=%s", str(exc))
+            res = 'get-error'
+        self.logger.debug("LogDB get_all_requests request, res=%s", res)
+        return res
+
+    def delete(self, request=None, mtype=None, this_thread=False, agent=True):
+        """
+        Delete entry in LogDB for given request
+        if mtype == None - delete all the log for that request
+        mtype != None - only delete specified mtype
+        """
+        res = 'delete-error'
+        try:
+            if request is None:
+                request = self.default_user
+            res = self.backend.delete(request, mtype, this_thread, agent)
+        except HTTPException as ex:
+            msg = "Failed to delete doc in LogDB. Reason: %s, status: %s" % (ex.reason, ex.status)
+            self.logger.error(msg)
+        except Exception as ex:
+            self.logger.error("LogDBBackend delete API failed, error=%s", str(ex))
+        self.logger.debug("LogDB delete request, res=%s", res)
+        return res
+
+    def cleanup(self, thr, backend='local'):
+        """Clean-up back-end LogDB"""
+        docs = []
+        try:
+            docs = self.backend.cleanup(thr)
+        except Exception as exc:
+            self.logger.error('LogDBBackend cleanup API failed, backend=%s, error=%s', backend, str(exc))
+        return docs
+
+    def heartbeat_report(self):
+        report = defaultdict(dict)
+        if self.user_pat.match(self.identifier):
+            self.logger.error("User %s: doesn't allow this function", self.identifier)
+            return report
+
+        for row in self.backend.get(self.default_user, None, agent=True).get('rows', []):
+            identifier = row['doc']['identifier']
+            # wmstats thread can run in multiple boxed.
+            if self.identifier == identifier or identifier.startswith(self.identifier):
+                # this will handle wmstats DataCacheUpdate thread with multiple machine
+                postfix = identifier.replace(self.identifier, "")
+                thr = "%s%s" % (row['doc']['thr'], postfix)
+                mtype = row['doc']['type']
+                ts = row['doc']['messages'][-1]['ts']
+                msg = row['doc']['messages'][-1]['msg']
+                if (thr in report) and ('ts' in report[thr]) and ts <= report[thr]['ts']:
+                    continue
+                else:
+                    report[thr]['type'] = mtype
+                    report[thr]['msg'] = msg
+                    report[thr]['ts'] = ts
+        return report
+
+    def _append_down_component_detail(self, report, thr, msg, ts=0, state="error"):
+        report['down_components'].append(thr)
+        detail = {'name': thr, 'worker_name': thr, 'state': state,
+                  'last_error': ts, 'error_message': msg,
+                  'pid': 'N/A'}
+        report['down_component_detail'].append(detail)
+        return
+
+    def wmstats_down_components_report(self, thread_list):
+        report = {}
+        report['down_components'] = []
+        report['down_component_detail'] = []
+
+        hbinfo = self.heartbeat_report()
+        for thr in thread_list:
+            # skip DataCacheUpdate thread. It will have multiple with post fix.
+            # i.e. DataCacheUpdate-vocms111
+            # TODO, need a better way to check
+            if thr != "DataCacheUpdate" and thr not in hbinfo:
+                self._append_down_component_detail(report, thr, "Thread not running")
+
+        for thr, info in viewitems(hbinfo):
+            if info['type'] == 'agent-error':
+                self._append_down_component_detail(report, thr, info['msg'], info['ts'])
+        return report