wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/WMFactory.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""
+A factory Class that is 'not thread safe' but is intended to work in threads
+(no sharing). The class dynamically loads objects from files when needed and
+caches them (or not). It is a generalized factory object. If needed this class
+can be made threadsafe.
+"""
+
+from builtins import object
+import threading
+
+
+class WMFactory(object):
+    """
+    A factory Class that is 'not thread safe' but is intended to work in
+    threads (no sharing). The class dynamically loads objects from files
+    when needed and caches them.
+    """
+
+    def __init__(self, name, namespace=''):
+        """
+        Initializes the factory, and checks if this thread already
+        has an attribute for storing registries. It uses the reserved
+        'registries' attribute in the thread.
+        """
+        self.namespace = namespace
+        self.objectList = {}
+
+        myThread = threading.currentThread()
+        if not hasattr(myThread, "factory"):
+            myThread.factory = {}
+        myThread.factory[name] = self
+
+    def loadObject(self, classname, args=None, storeInCache=True,
+                   getFromCache=True, listFlag=False, alteredClassName=None):
+        """
+        Dynamically loads the object from file.
+        For this to work the class name has to
+        be the same as the file name (minus the .py)
+
+        Be default objects are loaded from cache. However if you
+        want several different instances of the same object in one
+        thread, you set cache to False.
+        """
+        if getFromCache:
+            if classname in self.objectList:
+                return self.objectList[classname]
+
+        if self.namespace == '':
+            module = classname
+            # FIXME: hoky way of doing this! Change this please!
+            errModule = classname
+        else:
+            module = "%s.%s" % (self.namespace, classname)
+            errModule = "%s.%s" % (self.namespace, classname)
+        if alteredClassName:
+            classname = alteredClassName
+        module = __import__(module, globals(), locals(), [classname])
+        obj = getattr(module, classname.split('.')[-1])
+        if args is None:
+            classinstance = obj()
+        else:
+            # This handles the passing of list-style arguments instead of dicts
+            # Primarily for setting the schema
+            # Or anywhere you need arguments of the form (a,b,c,...)
+            if isinstance(args, list) and listFlag:
+                classinstance = obj(*args)
+            elif isinstance(args, dict):
+                classinstance = obj(**args)
+            else:
+                # But if you actually need to pass a list, better do it the old fashioned way
+                classinstance = obj(args)
+        if storeInCache:
+            self.objectList[classname] = classinstance
+
+        return classinstance

WMCore/WMInit.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+"""
+_WMInit
+
+Init class that can be used by external projects
+that only use part of the libraries
+"""
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+import threading
+import traceback
+
+from WMCore.Configuration import loadConfigurationFile
+from WMCore.DAOFactory import DAOFactory
+from WMCore.Database.DBFactory import DBFactory
+from WMCore.Database.Transaction import Transaction
+from WMCore.WMBase import getWMBASE
+from WMCore.WMException import WMException
+from WMCore.WMFactory import WMFactory
+
+
+class WMInitException(WMException):
+    """
+    WMInitException
+
+    You should never, ever see one of these.
+    I'm not optimistic that this will be the case.
+    """
+
+
+def connectToDB():
+    """
+    _connectToDB_
+
+    Connect to the database specified in the WMAgent config.
+    """
+    if "WMAGENT_CONFIG" not in os.environ:
+        print("Please set WMAGENT_CONFIG to point at your WMAgent configuration.")
+        sys.exit(1)
+
+    if not os.path.exists(os.environ["WMAGENT_CONFIG"]):
+        print("Can't find config: %s" % os.environ["WMAGENT_CONFIG"])
+        sys.exit(1)
+
+    wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
+
+    if not hasattr(wmAgentConfig, "CoreDatabase"):
+        print("Your config is missing the CoreDatabase section.")
+        sys.exit(1)
+
+    socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
+    connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
+    (dialect, junk) = connectUrl.split(":", 1)
+
+    myWMInit = WMInit()
+    myWMInit.setDatabaseConnection(dbConfig=connectUrl, dialect=dialect,
+                                   socketLoc=socketLoc)
+    return
+
+
+class WMInit(object):
+    def __init__(self):
+        return
+
+    def getWMBASE(self):
+        """ for those that don't want to use the static version"""
+        return getWMBASE()
+
+    def setLogging(self, logFile=None, logName=None, logLevel=logging.INFO, logExists=True):
+        """
+        Sets logging parameters, depending on the settings,
+        this method will create a logging file.
+        """
+        # use logName as name for file is no log file is given
+        if not logExists:
+            logging.basicConfig(level=logLevel, \
+                                format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', \
+                                datefmt='%m-%d %H:%M', \
+                                filename='%s.log' % logFile, \
+                                filemode='w')
+            logging.debug("Log file ready")
+
+        myThread = threading.currentThread()
+        if logName != None:
+            myThread.logger = logging.getLogger(logName)
+        else:
+            myThread.logger = logging.getLogger()
+
+    def setDatabaseConnection(self, dbConfig, dialect, socketLoc=None):
+        """
+        Sets the default connection parameters, without having to worry
+        much on what attributes need to be set. This is esepcially
+        advantagous for developers of third party projects that want
+        to use only parts of the WMCore lib.
+
+        The class differentiates between different formats used by external
+        projects. External project formats that are supported can activated
+        it by setting the flavor flag.
+        """
+        myThread = threading.currentThread()
+        if getattr(myThread, "dialect", None) != None:
+            # Database is already initialized, we'll create a new
+            # transaction and move on.
+            if hasattr(myThread, "transaction"):
+                if myThread.transaction != None:
+                    myThread.transaction.commit()
+
+            myThread.transaction = Transaction(myThread.dbi)
+            return
+
+        options = {}
+        if dialect.lower() == 'mysql':
+            dialect = 'MySQL'
+            if socketLoc != None:
+                options['unix_socket'] = socketLoc
+        elif dialect.lower() == 'oracle':
+            dialect = 'Oracle'
+        elif dialect.lower() == 'http':
+            dialect = 'CouchDB'
+        else:
+            msg = "Unsupported dialect %s !" % dialect
+            logging.error(msg)
+            raise WMInitException(msg)
+
+        myThread.dialect = dialect
+        myThread.logger = logging
+        myThread.dbFactory = DBFactory(logging, dbConfig, options)
+        myThread.dbi = myThread.dbFactory.connect()
+
+        # The transaction object will begin a transaction as soon as it is
+        # initialized.  I'd rather have the user handle that, so we'll commit
+        # it here.
+        myThread.transaction = Transaction(myThread.dbi)
+        myThread.transaction.commit()
+
+        return
+
+    def setSchema(self, modules=None, params=None):
+        """
+        Creates the schema in the database based on the modules
+        input.
+
+        This method needs to have been preceded by the
+        setDatabaseConnection.
+        """
+        modules = modules or []
+        myThread = threading.currentThread()
+
+        parameters = None
+        flag = False
+        # Set up for typical DBCreator format: logger, dbi, params
+        if params != None:
+            parameters = [None, None, params]
+            flag = True
+
+        myThread.transaction.begin()
+        for factoryName in modules:
+            # need to create these tables for testing.
+            # notice the default structure: <dialect>/Create
+            factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
+
+            create = factory.loadObject("Create", args=parameters, listFlag=flag)
+            createworked = create.execute(conn=myThread.transaction.conn,
+                                          transaction=myThread.transaction)
+            if createworked:
+                logging.debug("Tables for " + factoryName + " created")
+            else:
+                logging.debug("Tables " + factoryName + " could not be created.")
+        myThread.transaction.commit()
+
+    def clearDatabase(self, modules=None):
+        """
+        Database deletion. Global, ignore modules.
+        """
+        myThread = threading.currentThread()
+        if hasattr(myThread, 'transaction') and getattr(myThread.transaction, 'transaction', None):
+            # Then we have an open transaction
+            # We should try and close it first
+            try:
+                myThread.transaction.commit()
+            except:
+                try:
+                    myThread.transaction.rollback()
+                except:
+                    pass
+
+        # Setup the DAO
+        daoFactory = DAOFactory(package="WMCore.Database",
+                                logger=myThread.logger,
+                                dbinterface=myThread.dbi)
+        destroyDAO = daoFactory(classname="Destroy")
+
+        # Actually run a transaction and delete the DB
+        try:
+            destroyDAO.execute()
+        except Exception as ex:
+            msg = "Critical error while attempting to delete entire DB!\n"
+            msg += str(ex)
+            msg += str(traceback.format_exc())
+            logging.error(msg)
+            raise WMInitException(msg)
+
+        return
+
+    def checkDatabaseContents(self):
+        """
+        _checkDatabaseContents_
+
+        Check and see if anything is in the database.
+        This should be called by methods about to build the schema to make sure
+        that the DB itself is empty.
+        """
+
+        myThread = threading.currentThread()
+        daoFactory = DAOFactory(package="WMCore.Database",
+                                logger=myThread.logger,
+                                dbinterface=myThread.dbi)
+
+        testDAO = daoFactory(classname="ListUserContent")
+
+        result = testDAO.execute()
+        myThread.dbi.engine.dispose()
+
+        return result

WMCore/WMLogging.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+"""
+_WMLogging_
+
+Logging facilities used in WMCore.
+"""
+import logging
+import codecs
+from datetime import date, timedelta
+from logging.handlers import HTTPHandler, RotatingFileHandler, TimedRotatingFileHandler
+
+# a new log level which is lower than debug
+# to prevent a tsunami of log messages in debug
+# mode but to have the possibility to see all
+# database queries if necessary.
+logging.SQLDEBUG = 5
+logging.addLevelName(logging.SQLDEBUG,"SQLDEBUG")
+
+def sqldebug(msg):
+    """
+    A convenience method that all default levels
+    have for publishing log messages.
+    """
+    logging.log(logging.SQLDEBUG, msg)
+
+def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3):
+    """
+    _setupRotatingHandler_
+
+    Create a rotating log handler with the given parameters.
+    """
+    handler = RotatingFileHandler(fileName, "a", maxBytes, backupCount)
+    logging.getLogger().addHandler(handler)
+    return
+
+
+def getTimeRotatingLogger(name, logFile, duration = 'midnight'):
+    """ Set the logger for time based lotaing.
+    """
+    logger = logging.getLogger(name)
+    if duration == 'midnight':
+        handler = MyTimedRotatingFileHandler(logFile, duration, backupCount = 10)
+    else:
+        handler = TimedRotatingFileHandler(logFile, duration, backupCount = 10)
+    formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+
+    return logger
+
+
+class MyTimedRotatingFileHandler(TimedRotatingFileHandler):
+    """
+    _MyTimedRotatingFileHandler_
+
+    Overwrite the standard filename functionality from
+    logging.handlers.MyTimedRotatingFileHandler
+    such that it mimics the same behaviour as rotatelogs tool.
+
+    Source code from:
+    https://stackoverflow.com/questions/338450/timedrotatingfilehandler-changing-file-name
+    """
+    def __init__(self, logName, interval, backupCount):
+        super(MyTimedRotatingFileHandler, self).__init__(logName, when=interval,
+                                                         backupCount=backupCount)
+
+    def doRollover(self):
+        """
+        _doRollover_
+
+        Rotate the log file and add the date between the log name
+        and its extension, e.g.:
+        reqmgr2.log becomes reqmgr2-20170815.log
+        """
+        self.stream.close()
+        # replace yesterday's date by today
+        yesterdayStr = (date.today() - timedelta(1)).strftime("%Y%m%d")
+        todayStr = date.today().strftime("%Y%m%d")
+        self.baseFilename = self.baseFilename.replace(yesterdayStr, todayStr)
+        if self.encoding:
+            self.stream = codecs.open(self.baseFilename, 'w', self.encoding)
+        else:
+            self.stream = open(self.baseFilename, 'w')
+        self.rolloverAt = self.rolloverAt + self.interval
+
+
+class CouchHandler(logging.handlers.HTTPHandler):
+    def __init__(self, host, database):
+        HTTPHandler.__init__(self, host, database, 'POST')
+        from WMCore.Database.CMSCouch import CouchServer
+        self.database = CouchServer(dburl=host).connectDatabase(database, size=10)
+
+    def emit(self, record):
+        """
+        Write a document to CouchDB representing the log message.
+        """
+        doc = {}
+        doc['message'] = record.msg
+        doc['threadName'] = record.threadName
+        doc['name'] = record.name
+        doc['created'] = record.created
+        doc['process'] = record.process
+        doc['levelno'] = record.levelno
+        doc['lineno'] = record.lineno
+        doc['processName'] = record.processName
+        doc['levelname'] = record.levelname
+        self.database.commitOne(doc, timestamp=True)