wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/WMFactory.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""
+A factory Class that is 'not thread safe' but is intended to work in threads
+(no sharing). The class dynamically loads objects from files when needed and
+caches them (or not). It is a generalized factory object. If needed this class
+can be made threadsafe.
+"""
+
+from builtins import object
+import threading
+
+
+class WMFactory(object):
+    """
+    A factory Class that is 'not thread safe' but is intended to work in
+    threads (no sharing). The class dynamically loads objects from files
+    when needed and caches them.
+    """
+
+    def __init__(self, name, namespace=''):
+        """
+        Initializes the factory, and checks if this thread already
+        has an attribute for storing registries. It uses the reserved
+        'registries' attribute in the thread.
+        """
+        self.namespace = namespace
+        self.objectList = {}
+
+        myThread = threading.currentThread()
+        if not hasattr(myThread, "factory"):
+            myThread.factory = {}
+        myThread.factory[name] = self
+
+    def loadObject(self, classname, args=None, storeInCache=True,
+                   getFromCache=True, listFlag=False, alteredClassName=None):
+        """
+        Dynamically loads the object from file.
+        For this to work the class name has to
+        be the same as the file name (minus the .py)
+
+        Be default objects are loaded from cache. However if you
+        want several different instances of the same object in one
+        thread, you set cache to False.
+        """
+        if getFromCache:
+            if classname in self.objectList:
+                return self.objectList[classname]
+
+        if self.namespace == '':
+            module = classname
+            # FIXME: hoky way of doing this! Change this please!
+            errModule = classname
+        else:
+            module = "%s.%s" % (self.namespace, classname)
+            errModule = "%s.%s" % (self.namespace, classname)
+        if alteredClassName:
+            classname = alteredClassName
+        module = __import__(module, globals(), locals(), [classname])
+        obj = getattr(module, classname.split('.')[-1])
+        if args is None:
+            classinstance = obj()
+        else:
+            # This handles the passing of list-style arguments instead of dicts
+            # Primarily for setting the schema
+            # Or anywhere you need arguments of the form (a,b,c,...)
+            if isinstance(args, list) and listFlag:
+                classinstance = obj(*args)
+            elif isinstance(args, dict):
+                classinstance = obj(**args)
+            else:
+                # But if you actually need to pass a list, better do it the old fashioned way
+                classinstance = obj(args)
+        if storeInCache:
+            self.objectList[classname] = classinstance
+
+        return classinstance

WMCore/WMInit.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python
+"""
+_WMInit
+
+Init class that can be used by external projects
+that only use part of the libraries
+"""
+from __future__ import print_function
+
+import logging
+import os
+import os.path
+import sys
+import threading
+import traceback
+import warnings
+import wmcoredb
+
+from WMCore.Configuration import loadConfigurationFile
+from WMCore.DAOFactory import DAOFactory
+from WMCore.Database.DBFactory import DBFactory
+from WMCore.Database.Transaction import Transaction
+from WMCore.WMBase import getWMBASE
+from WMCore.WMException import WMException
+
+
+class WMInitException(WMException):
+    """
+    WMInitException
+
+    You should never, ever see one of these.
+    I'm not optimistic that this will be the case.
+    """
+
+
+def connectToDB():
+    """
+    _connectToDB_
+
+    Connect to the database specified in the WMAgent config.
+    """
+    if "WMAGENT_CONFIG" not in os.environ:
+        print("Please set WMAGENT_CONFIG to point at your WMAgent configuration.")
+        sys.exit(1)
+
+    if not os.path.exists(os.environ["WMAGENT_CONFIG"]):
+        print("Can't find config: %s" % os.environ["WMAGENT_CONFIG"])
+        sys.exit(1)
+
+    wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
+
+    if not hasattr(wmAgentConfig, "CoreDatabase"):
+        print("Your config is missing the CoreDatabase section.")
+        sys.exit(1)
+
+    socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
+    connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
+    (dialect, _) = connectUrl.split(":", 1)
+
+    myWMInit = WMInit()
+    myWMInit.setDatabaseConnection(dbConfig=connectUrl, dialect=dialect,
+                                   socketLoc=socketLoc)
+
+    return
+
+
+class WMInit(object):
+    def __init__(self):
+        return
+
+    def getWMBASE(self):
+        """ for those that don't want to use the static version"""
+        return getWMBASE()
+
+    def setLogging(self, logFile=None, logName=None, logLevel=logging.INFO, logExists=True):
+        """
+        Sets logging parameters, depending on the settings,
+        this method will create a logging file.
+        """
+        # use logName as name for file is no log file is given
+        if not logExists:
+            logging.basicConfig(level=logLevel, \
+                                format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', \
+                                datefmt='%m-%d %H:%M', \
+                                filename='%s.log' % logFile, \
+                                filemode='w')
+            logging.debug("Log file ready")
+
+        myThread = threading.currentThread()
+        if logName != None:
+            myThread.logger = logging.getLogger(logName)
+        else:
+            myThread.logger = logging.getLogger()
+
+    def setDatabaseConnection(self, dbConfig, dialect, socketLoc=None):
+        """
+        Sets the default connection parameters, without having to worry
+        much on what attributes need to be set. This is esepcially
+        advantagous for developers of third party projects that want
+        to use only parts of the WMCore lib.
+
+        The class differentiates between different formats used by external
+        projects. External project formats that are supported can activated
+        it by setting the flavor flag.
+        """
+        myThread = threading.currentThread()
+        if getattr(myThread, "dialect", None) != None:
+            # Database is already initialized, we'll create a new
+            # transaction and move on.
+            if hasattr(myThread, "transaction"):
+                if myThread.transaction != None:
+                    myThread.transaction.commit()
+
+            myThread.transaction = Transaction(myThread.dbi)
+            return
+
+        options = {}
+        if dialect.lower() in ['mysql', 'mariadb']:
+            dialect = 'mariadb'  # Both MySQL and MariaDB use the mariadb directory
+            if socketLoc != None:
+                options['unix_socket'] = socketLoc
+        elif dialect.lower() == 'oracle':
+            dialect = 'oracle'  # Keep lowercase for consistency
+        elif dialect.lower() == 'http':
+            dialect = 'CouchDB'
+        else:
+            msg = "Unsupported dialect %s !" % dialect
+            logging.error(msg)
+            raise WMInitException(msg)
+
+        myThread.dialect = dialect
+        myThread.logger = logging
+        myThread.dbFactory = DBFactory(logging, dbConfig, options)
+        myThread.dbi = myThread.dbFactory.connect()
+
+        # The transaction object will begin a transaction as soon as it is
+        # initialized.  I'd rather have the user handle that, so we'll commit
+        # it here.
+        myThread.transaction = Transaction(myThread.dbi)
+        myThread.transaction.commit()
+
+        return
+
+    def setSchema(self, modules=None, params=None):
+        """
+        Creates the schema in the database based on the modules
+        input.
+
+        This method needs to have been preceded by the
+        setDatabaseConnection.
+
+        @deprecated: Use setSchemaFromModules instead
+        """
+        warnings.warn("setSchema is deprecated. Use setSchemaFromModules instead.", DeprecationWarning)
+
+        # create a map of old to new SQL module names
+        moduleMap = {
+            'WMCore.WMBS': 'wmbs',
+            'WMCore.ResourceControl': 'resourcecontrol',
+            'WMCore.BossAir': 'bossair',
+            'WMCore.Agent.Database': 'agent',
+            'WMComponent.DBS3Buffer': 'dbs3buffer',
+            'T0.WMBS': 'tier0',
+            'WMQuality.TestDB': 'testdb'
+        }
+
+        # convert old module names to new format
+        if modules:
+            modules = [moduleMap.get(module, module) for module in modules]
+
+        self.setSchemaFromModules(modules)
+
+    def setSchemaFromModules(self, sqlModules):
+        """
+        Initialize database schema for one or more SQL packages.
+        It finds out which dialect is being used and then looks for the
+        appropriate SQL files in the sql directory.
+
+        :param sqlModules: List of SQL database modules to be initialized.
+            Current supported values are (default is 'wmbs'):
+            - 'wmbs'
+            - 'resourcecontrol'
+            - 'bossair'
+            - 'agent'
+            - 'dbs3buffer'
+            - 'tier0'
+        """
+        myThread = threading.currentThread()
+        if not hasattr(myThread, 'dbi'):
+            raise WMInitException("Database connection not initialized. Call setDatabaseConnection first.")
+
+        # Get the database dialect
+        dialect = myThread.dialect.lower()
+        if dialect not in ['mariadb', 'oracle']:
+            raise WMInitException(f"Unsupported database dialect: {dialect}")
+
+        # Get the base directory (WMCore root)
+        if os.environ.get('WMCORE_ROOT'):
+            baseDir = os.environ['WMCORE_ROOT']
+            logging.info("SQL base directory based on WMCORE_ROOT: %s", baseDir)
+        else:
+            baseDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+            logging.info("SQL base directory based on WMInit-relative path: %s", baseDir)
+
+        # Define the SQL files needed for each module and their dependencies
+        # The order matters - modules that depend on others should come later
+        moduleSQLFiles = {
+            'wmbs': {
+                'files': ['create_wmbs_tables.sql', 'create_wmbs_indexes.sql', 'initial_wmbs_data.sql'],
+                'dependencies': []
+            },
+            'resourcecontrol': {
+                'files': ['create_resourcecontrol.sql'],
+                'dependencies': ['wmbs']
+            },
+            'bossair': {
+                'files': ['create_bossair.sql'],
+                'dependencies': ['wmbs']
+            },
+            'agent': {
+                'files': ['create_agent.sql'],
+                'dependencies': []
+            },
+            'dbs3buffer': {
+                'files': ['create_dbs3buffer.sql'],
+                'dependencies': ['wmbs']
+            },
+            'tier0': {
+                'files': ['create_tier0_tables.sql', 'create_tier0_indexes.sql', 'create_tier0_functions.sql', 'initial_tier0_data.sql'],
+                'dependencies': ['wmbs', 'dbs3buffer']
+            },
+            'testdb': {
+                'files': ['create_testdb.sql'],
+                'dependencies': []
+            },
+        }
+
+        # Validate all requested modules exist
+        for module in sqlModules:
+            if module not in moduleSQLFiles:
+                raise WMInitException(f"Unknown module: {module}")
+
+        # Sort modules based on dependencies
+        sorted_modules = []
+        remaining_modules = set(sqlModules)
+
+        while remaining_modules:
+            # Find modules with no remaining dependencies
+            ready_modules = [
+                mod for mod in remaining_modules
+                if all(dep in sorted_modules for dep in moduleSQLFiles[mod]['dependencies'])
+            ]
+
+            if not ready_modules:
+                # Circular dependency detected
+                raise WMInitException("Circular dependency detected in module dependencies")
+
+            sorted_modules.extend(ready_modules)
+            remaining_modules -= set(ready_modules)
+
+        # Execute SQL files in dependency order
+        for module in sorted_modules:
+            logging.info("Executing SQL files for: %s", module)
+            for sql_file in moduleSQLFiles[module]['files']:
+                dialect_sql_file = wmcoredb.get_sql_file(module_name=module, file_name=sql_file, backend=dialect)
+
+                # now execute each SQL statement
+                for stmt in self._getSQLStatements(dialect_sql_file, dialect):
+                    try:
+                        myThread.dbi.processData(stmt)
+                    except Exception as ex:
+                        msg = f"Error executing SQL file {dialect_sql_file}. "
+                        msg += f"Statement: {stmt}"
+                        msg += str(ex)
+                        raise WMInitException(msg)
+
+        return
+
+    def _getSQLStatements(self, sqlFile, dialect):
+        """
+        Return the SQL statements from the file.
+        For MariaDB, it accepts the whole SQL file content in a single statement.
+        For Oracle, it splits the SQL file content into statements using the slash (/) terminator
+        when it appears as the first character in a line.
+        """
+        if not os.path.exists(sqlFile):
+            raise WMInitException(f"SQL file not found: {sqlFile}")
+
+        with open(sqlFile, 'r', encoding='utf-8') as f:
+            sql = f.read()
+
+        if dialect == 'mariadb':
+            return [sql]
+        elif dialect == 'oracle':
+            statements = []
+            current_statement = []
+
+            for line in sql.split('\n'):
+                line = line.strip()
+                # Skip empty lines and comments
+                if not line or line.startswith('--'):
+                    continue
+
+                # If we find a slash terminator at the start of a line
+                if line == '/':
+                    # Join all lines collected so far into a statement
+                    stmt = '\n'.join(current_statement)
+                    if stmt.strip():
+                        statements.append(stmt)
+                    current_statement = []
+                else:
+                    current_statement.append(line)
+
+            # Add any remaining statement
+            if current_statement:
+                stmt = '\n'.join(current_statement)
+                if stmt.strip():
+                    statements.append(stmt)
+
+            return statements
+        else:
+            raise WMInitException(f"Unsupported database dialect: {dialect}")
+
+    def clearDatabase(self, modules=None):
+        """
+        Database deletion. Global, ignore modules.
+        """
+        myThread = threading.currentThread()
+        if hasattr(myThread, 'transaction') and getattr(myThread.transaction, 'transaction', None):
+            # Then we have an open transaction
+            # We should try and close it first
+            try:
+                myThread.transaction.commit()
+            except:
+                try:
+                    myThread.transaction.rollback()
+                except:
+                    pass
+
+        # Setup the DAO
+        daoFactory = DAOFactory(package="WMCore.Database",
+                                logger=myThread.logger,
+                                dbinterface=myThread.dbi)
+        destroyDAO = daoFactory(classname="Destroy")
+
+        # Actually run a transaction and delete the DB
+        try:
+            destroyDAO.execute()
+        except Exception as ex:
+            msg = "Critical error while attempting to delete entire DB!\n"
+            msg += str(ex)
+            msg += str(traceback.format_exc())
+            logging.error(msg)
+            raise WMInitException(msg)
+
+        return
+
+    def checkDatabaseContents(self):
+        """
+        _checkDatabaseContents_
+
+        Check and see if anything is in the database.
+        This should be called by methods about to build the schema to make sure
+        that the DB itself is empty.
+        """
+
+        myThread = threading.currentThread()
+        daoFactory = DAOFactory(package="WMCore.Database",
+                                logger=myThread.logger,
+                                dbinterface=myThread.dbi)
+
+        testDAO = daoFactory(classname="ListUserContent")
+
+        result = testDAO.execute()
+        myThread.dbi.engine.dispose()
+
+        return result

WMCore/WMLogging.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+"""
+_WMLogging_
+
+Logging facilities used in WMCore.
+"""
+import logging
+from datetime import date, timedelta
+from logging.handlers import HTTPHandler, RotatingFileHandler, TimedRotatingFileHandler
+
+# a new log level which is lower than debug
+# to prevent a tsunami of log messages in debug
+# mode but to have the possibility to see all
+# database queries if necessary.
+logging.SQLDEBUG = 5
+logging.addLevelName(logging.SQLDEBUG,"SQLDEBUG")
+
+def sqldebug(msg):
+    """
+    A convenience method that all default levels
+    have for publishing log messages.
+    """
+    logging.log(logging.SQLDEBUG, msg)
+
+def setupRotatingHandler(fileName, maxBytes = 200000000, backupCount = 3):
+    """
+    _setupRotatingHandler_
+
+    Create a rotating log handler with the given parameters.
+    """
+    handler = RotatingFileHandler(fileName, "a", maxBytes, backupCount)
+    logging.getLogger().addHandler(handler)
+    return
+
+
+def getTimeRotatingLogger(name, logFile, duration = 'midnight'):
+    """ Set the logger for time based lotaing.
+    """
+    logger = logging.getLogger(name)
+    if duration == 'midnight':
+        handler = MyTimedRotatingFileHandler(logFile, duration, backupCount = 10)
+    else:
+        handler = TimedRotatingFileHandler(logFile, duration, backupCount = 10)
+    formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+
+    return logger
+
+
+class MyTimedRotatingFileHandler(TimedRotatingFileHandler):
+    """
+    _MyTimedRotatingFileHandler_
+
+    Overwrite the standard filename functionality from
+    logging.handlers.MyTimedRotatingFileHandler
+    such that it mimics the same behaviour as rotatelogs tool.
+
+    Source code from:
+    https://stackoverflow.com/questions/338450/timedrotatingfilehandler-changing-file-name
+    """
+    def __init__(self, logName, interval, backupCount):
+        super(MyTimedRotatingFileHandler, self).__init__(logName, when=interval,
+                                                         backupCount=backupCount)
+
+    def doRollover(self):
+        """
+        _doRollover_
+
+        Rotate the log file and add the date between the log name
+        and its extension, e.g.:
+        reqmgr2.log becomes reqmgr2-20170815.log
+        """
+        self.stream.close()
+        # replace yesterday's date by today
+        yesterdayStr = (date.today() - timedelta(1)).strftime("%Y%m%d")
+        todayStr = date.today().strftime("%Y%m%d")
+        self.baseFilename = self.baseFilename.replace(yesterdayStr, todayStr)
+        self.stream = open(self.baseFilename, 'w', encoding='utf-8')
+        self.rolloverAt = self.rolloverAt + self.interval
+
+
+class CouchHandler(logging.handlers.HTTPHandler):
+    def __init__(self, host, database):
+        HTTPHandler.__init__(self, host, database, 'POST')
+        from WMCore.Database.CMSCouch import CouchServer
+        self.database = CouchServer(dburl=host).connectDatabase(database, size=10)
+
+    def emit(self, record):
+        """
+        Write a document to CouchDB representing the log message.
+        """
+        doc = {}
+        doc['message'] = record.msg
+        doc['threadName'] = record.threadName
+        doc['name'] = record.name
+        doc['created'] = record.created
+        doc['process'] = record.process
+        doc['levelno'] = record.levelno
+        doc['lineno'] = record.lineno
+        doc['processName'] = record.processName
+        doc['levelname'] = record.levelname
+        self.database.commitOne(doc, timestamp=True)