wmglobalqueue 2.3.10rc10__py3-none-any.whl

WMCore/Database/ConfigDBMap.py

@@ -0,0 +1,29 @@
+from builtins import object
+
+
+class ConfigDBMapInterface(object):
+    """
+    Interface for converting the configuration to
+    db url string and option dictionary for DBFactory parameters
+
+    the implementation of this class should create db url string.
+    and option for DBFactory creation.
+    For details of options, reference WMCore.Database.DBFactory
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def getDBUrl(self):
+        """
+        this should return db string
+        i.e. mysql://username@hostname.fnal.gov:3306/TestDB
+        """
+        raise NotImplementedError("getDBUrl is not implemented")
+
+    def getOption(self):
+        """
+        this should return options of dict format
+        i.e. {'engine_parameters': {'pool_size': 10}}
+        """
+        raise NotImplementedError("getOption is not implemented")

WMCore/Database/CouchUtils.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+CouchUtils.py
+
+Created by Dave Evans on 2010-03-11.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+from __future__ import print_function
+
+from future import standard_library
+standard_library.install_aliases()
+
+from http.client import HTTPException
+
+import functools
+
+import WMCore.Database.CMSCouch as CMSCouch
+
+
+class CouchConnectionError(Exception):
+    """docstring for CouchConnectionError"""
+    def __init__(self, arg):
+        super(CouchConnectionError, self).__init__(arg)
+
+
+
+
+def initialiseCouch(objectRef):
+    if objectRef.couchdb != None:
+        return
+    if objectRef.url == None:
+        msg = "url for couch service not provided"
+        raise CouchConnectionError(msg)
+    if objectRef.database == None:
+        msg = "database name for couch service not provided"
+        raise CouchConnectionError(msg)
+    try:
+        objectRef.server = CMSCouch.CouchServer(objectRef.url)
+        objectRef.couchdb = objectRef.server.connectDatabase(objectRef.database)
+    except HTTPException as e:
+        msg = "%s with status: %s, reason: %s and result: %s" % (repr(e),
+                                                                 getattr(e, 'status', ""),
+                                                                 getattr(e, 'reason', ""),
+                                                                 getattr(e, 'result', ""))
+        raise CouchConnectionError(msg)
+    except Exception as e:
+        msg = "Exception instantiating couch services for :\n"
+        msg += " url = %s\n database = %s\n" % (objectRef.url, objectRef.database)
+        msg += " Exception: %s" % str(e)
+        print(msg)
+        raise CouchConnectionError(msg)
+
+def connectToCouch(funcRef):
+    """
+    _connectToCouch_
+
+    Decorator method to connect the function's class object to couch
+    """
+    @functools.wraps(funcRef)
+    def wrapper(x, *args, **opts):
+        initialiseCouch(x)
+        return funcRef(x, *args, **opts)
+    return wrapper
+
+def requireOwner(func):
+    """
+    _requireOwner_
+
+    Decorator to ensure that the owner attribute of a couch ACDC object is not None
+    """
+
+    def wrapper(self, *args, **opts):
+        if self.owner == None:
+            msg = "Owner not provided for Collection"
+            raise RuntimeError(msg)
+        return func(self, *args, **opts)
+    return wrapper
+
+def requireCollection(func):
+    """
+    _requireCollection_
+
+    Decorator to ensure that the collection attribute of a couch ACDC object is not None
+    """
+
+    def wrapper(self, *args, **opts):
+        if self.collection == None:
+            msg = "Collection not provided for Collection"
+            raise RuntimeError(msg)
+        return func(self, *args, **opts)
+    return wrapper
+
+def requireFilesetName(func):
+    """
+    _requireFilesetName_
+
+    Decorator to require that a fileset has a name that is not None
+
+    """
+    def wrapper(self, *args, **opts):
+        if not 'name' in self or self['name'] == None:
+            raise RuntimeError("Filesets must be named")
+        return func(self, *args, **opts)
+    return wrapper
+
+def requireCollectionName(func):
+    """
+    _requireCollectionName_
+
+    Decorator to require that a collection has a name that is not None
+
+    """
+    def wrapper(self, *args, **opts):
+        if not 'name' in self or self['name'] == None:
+            raise RuntimeError("Filesets must be named")
+        return func(self, *args, **opts)
+    return wrapper

WMCore/Database/DBCore.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+"""
+_Core_
+
+Core Database APIs
+
+
+"""
+from copy import copy
+
+from Utils.IteratorTools import grouper
+import WMCore.WMLogging
+from WMCore.DataStructs.WMObject import WMObject
+from WMCore.Database.ResultSet import ResultSet
+
+class DBInterface(WMObject):
+    """
+    Base class for doing SQL operations using a SQLAlchemy engine, or
+    pre-exisitng connection.
+
+    processData will take a (list of) sql statements and a (list of)
+    bind variable dictionaries and run the statements on the DB. If
+    necessary it will substitute binds into the sql (MySQL).
+
+    TODO:
+        Add in some suitable exceptions in one or two places
+        Test the hell out of it
+        Support executemany()
+    """
+
+    logger = None
+    engine = None
+
+    def __init__(self, logger, engine):
+        self.logger = logger
+        self.logger.info ("Instantiating base WM DBInterface")
+        self.engine = engine
+        self.maxBindsPerQuery = 500
+
+    def buildbinds(self, sequence, thename, therest=[{}]):
+        """
+        Build a list of binds. Can be used recursively, e.g.:
+        buildbinds(file, 'file', buildbinds(pnn, 'location'), {'lumi':123})
+        TODO: replace with an appropriate map function
+        """
+        binds = []
+        for r in sequence:
+            for i in self.makelist(therest):
+                thebind = copy(i)
+                thebind[thename] = r
+                binds.append(thebind)
+        return binds
+
+    def executebinds(self, s=None, b=None, connection=None,
+                     returnCursor=False):
+        """
+        _executebinds_
+
+        returns a list of sqlalchemy.engine.base.ResultProxy objects
+        """
+        if b == None:
+            resultProxy = connection.execute(s)
+        else:
+            resultProxy = connection.execute(s, b)
+
+        if returnCursor:
+            return resultProxy
+
+        result = ResultSet()
+        result.add(resultProxy)
+        resultProxy.close()
+        return result
+
+    def executemanybinds(self, s=None, b=None, connection=None,
+                         returnCursor=False):
+        """
+        _executemanybinds_
+        b is a list of dictionaries for the binds, e.g.:
+
+        b = [ {'bind1':'value1a', 'bind2': 'value2a'},
+        {'bind1':'value1b', 'bind2': 'value2b'} ]
+
+        see: http://www.gingerandjohn.com/archives/2004/02/26/cx_oracle-executemany-example/
+
+        Can't executemany() selects - so do each combination of binds here instead.
+        This will return a list of sqlalchemy.engine.base.ResultProxy object's
+        one for each set of binds.
+
+        returns a list of sqlalchemy.engine.base.ResultProxy objects
+        """
+
+        s = s.strip()
+        if s.lower().endswith('select', 0, 6):
+            """
+            Trying to select many
+            """
+            if returnCursor:
+                result = []
+                for bind in b:
+                    result.append(connection.execute(s, bind))
+            else:
+                result = ResultSet()
+                for bind in b:
+                    resultproxy = connection.execute(s, bind)
+                    result.add(resultproxy)
+                    resultproxy.close()
+
+            return self.makelist(result)
+
+        """
+        Now inserting or updating many
+        """
+        result = connection.execute(s, b)
+        return self.makelist(result)
+
+    def connection(self):
+        """
+        Return a connection to the engine (from the connection pool)
+        """
+        return self.engine.connect()
+
+
+    def processData(self, sqlstmt, binds={}, conn=None,
+                    transaction=False, returnCursor=False):
+        """
+        set conn if you already have an active connection to reuse
+        set transaction = True if you already have an active transaction
+
+        """
+        connection = None
+        try:
+            if not conn:
+                connection = self.connection()
+            else:
+                connection = conn
+
+            result = []
+            # Can take either a single statement or a list of statements and binds
+            sqlstmt = self.makelist(sqlstmt)
+            binds = self.makelist(binds)
+            if len(sqlstmt) > 0 and (len(binds) == 0 or (binds[0] == {} or binds[0] == None)):
+                # Should only be run by create statements
+                if not transaction:
+                    #WMCore.WMLogging.sqldebug("transaction created in DBInterface")
+                    trans = connection.begin()
+
+                for i in sqlstmt:
+                    r = self.executebinds(i, connection=connection,
+                                          returnCursor=returnCursor)
+                    result.append(r)
+
+                if not transaction:
+                    trans.commit()
+            elif len(binds) > len(sqlstmt) and len(sqlstmt) == 1:
+                #Run single SQL statement for a list of binds - use execute_many()
+                if not transaction:
+                    trans = connection.begin()
+                for subBinds in grouper(binds, self.maxBindsPerQuery):
+                    result.extend(self.executemanybinds(sqlstmt[0], subBinds,
+                                                        connection=connection, returnCursor=returnCursor))
+
+                if not transaction:
+                    trans.commit()
+            elif len(binds) == len(sqlstmt):
+                # Run a list of SQL for a list of binds
+                if not transaction:
+                    trans = connection.begin()
+
+                for i, s in enumerate(sqlstmt):
+                    b = binds[i]
+
+                    r = self.executebinds(s, b, connection=connection,
+                                          returnCursor=returnCursor)
+                    result.append(r)
+
+                if not transaction:
+                    trans.commit()
+            else:
+                self.logger.exception(
+                    "DBInterface.processData Nothing executed, problem with your arguments")
+                self.logger.exception(
+                    "DBInterface.processData SQL = %s" % sqlstmt)
+                WMCore.WMLogging.sqldebug('DBInterface.processData  sql is %s items long' % len(sqlstmt))
+                WMCore.WMLogging.sqldebug('DBInterface.processData  binds are %s items long' % len(binds))
+                assert_value = False
+                if len(binds) == len(sqlstmt):
+                    assert_value = True
+                WMCore.WMLogging.sqldebug('DBInterface.processData are binds and sql same length? : %s' % (assert_value))
+                WMCore.WMLogging.sqldebug('sql: %s\n binds: %s\n, connection:%s\n, transaction:%s\n' %
+                                           (sqlstmt, binds, connection, transaction))
+                WMCore.WMLogging.sqldebug('type check:\nsql: %s\n binds: %s\n, connection:%s\n, transaction:%s\n' %
+                                           (type(sqlstmt), type(binds), type(connection), type(transaction)))
+                raise Exception("""DBInterface.processData Nothing executed, problem with your arguments
+                Probably mismatched sizes for sql (%i) and binds (%i)""" % (len(sqlstmt), len(binds)))
+        finally:
+            if not conn and connection != None:
+                connection.close() # Return connection to the pool
+        return result

WMCore/Database/DBCreator.py

@@ -0,0 +1,113 @@
+#!/usr/bin/python
+
+"""
+_DBCreator_
+
+Base class for formatters that create tables.
+
+"""
+
+from builtins import str
+
+from WMCore.Database.DBFormatter import DBFormatter
+from WMCore.WMException import WMException
+from WMCore.WMExceptions import WMEXCEPTION
+
+
+class DBCreator(DBFormatter):
+
+    """
+    _DBCreator_
+
+    Generic class for creating database tables.
+
+    """
+
+    def __init__(self, logger, dbinterface):
+        """
+        _init_
+
+        Call the constructor of the parent class and create empty dictionaries
+        to hold table create statements, constraint statements and insert
+        statements.
+        """
+        DBFormatter.__init__(self, logger, dbinterface)
+        self.create = {}
+        self.constraints = {}
+        self.inserts = {}
+        self.indexes = {}
+
+    def execute(self, conn = None, transaction = False):
+        """
+        _execute_
+
+        Generic method to create tables and constraints by execute
+        sql statements in the create, and constraints dictionaries.
+
+        Before execution the keys assigned to the tables in the self.create
+        dictionary are sorted, to offer the possibilitiy of executing
+        table creation in a certain order.
+
+        """
+        # create tables
+        for i in sorted(self.create.keys()):
+            try:
+                self.dbi.processData(self.create[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.create[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # create indexes
+        for i in self.indexes:
+            try:
+                self.dbi.processData(self.indexes[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.indexes[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # set constraints
+        for i in self.constraints:
+            try:
+                self.dbi.processData(self.constraints[i],
+                                 conn = conn,
+                                 transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.constraints[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # insert permanent data
+        for i in self.inserts:
+            try:
+                self.dbi.processData(self.inserts[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.inserts[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        return True
+
+    def __str__(self):
+        """
+        _str_
+
+        Return a well formatted text representation of the schema held in the
+        self.create, self.constraints, self.inserts, self.indexes dictionaries.
+        """
+        string = ''
+        for i in self.create, self.constraints, self.inserts, self.indexes:
+            for j in i:
+                string = string + i[j].lstrip() + '\n'
+        return string

WMCore/Database/DBExceptionHandler.py

@@ -0,0 +1,57 @@
+from __future__ import division, print_function
+
+from builtins import str
+from functools import wraps
+import logging
+import threading
+
+### Don't crash the components on the following database exceptions
+# ORA-03113: end-of-file on communication channel
+# ORA-03114: Not Connected to Oracle
+# ORA-03135: connection lost contact
+# ORA-12545: Connect failed because target host or object does not exist
+# ORA-00060: deadlock detected while waiting for resource
+# ORA-01033: ORACLE initialization or shutdown in progress
+# (cx_Oracle.InterfaceError) not connected  # same as ORA-03114, in the new SQLAlchemy
+# (cx_Oracle.DatabaseError) ORA-25408: can not safely replay call
+# (cx_Oracle.DatabaseError) ORA-25401: can not continue fetches
+# and those two MySQL exceptions
+DB_CONNECTION_ERROR_STR = ["ORA-03113", "ORA-03114", "ORA-03135", "ORA-12545", "ORA-00060", "ORA-01033",
+                           "MySQL server has gone away", "Lock wait timeout exceeded",
+                           "(cx_Oracle.InterfaceError) not connected", "ORA-25408", "ORA-25401"]
+
+
+def db_exception_handler(f):
+    """
+    :param f: function
+    :return: wrapper fuction
+
+    Only need to handle DB connection problem other db problems which need to rollback the transaction,
+    shouldn't be included in this hanldler
+
+    Warning: This only used, when original function return values are not used, or wrapped around Utils.Timer.timeFunc
+    """
+
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        myThread = threading.currentThread()
+        if hasattr(myThread, "logdbClient") and myThread.logdbClient is not None:
+            myThread.logdbClient.delete("DBConnection_error_handler", "warning", this_thread=True)
+        try:
+            return f(*args, **kwargs)
+        except Exception as ex:
+            msg = str(ex)
+            for errStr in DB_CONNECTION_ERROR_STR:
+                if errStr in msg:
+                    logging.error("%s: Temp error will try later", msg)
+
+                    if hasattr(myThread, "logdbClient") and myThread.logdbClient is not None:
+                        myThread.logdbClient.post("DBConnection_error_handler", "warning")
+
+                    # returns tuples of 3 since timeFunc returns (time to spend, result, func.__name__)
+                    return 0, None, f.__name__
+
+            # for other case raise the same exception
+            raise
+
+    return wrapper

WMCore/Database/DBFactory.py

@@ -0,0 +1,110 @@
+
+from builtins import object
+import threading
+
+from sqlalchemy import create_engine
+from sqlalchemy import __version__ as sqlalchemy_version
+from WMCore.Database.Dialects import MySQLDialect
+from WMCore.Database.Dialects import OracleDialect
+
+class DBFactory(object):
+
+    # class variable
+    _engineMap = {}
+    _defaultEngineParams = {"convert_unicode" : True,
+                            "pool_recycle": 7200}
+
+    def __init__(self, logger, dburl=None, options={}):
+        self.logger = logger
+        # get the engine parameter from option
+        if 'engine_parameters' in options:
+            self._defaultEngineParams.update(options['engine_parameters'])
+            del options['engine_parameters']
+
+        if dburl:
+            self.dburl = dburl
+        else:
+            #This will be deprecated.
+            """
+            Need to make the dburl here. Possible formats are:
+
+            mysql://host/database
+            mysql://username@host/database
+            mysql://username:password@host:port/database
+
+            oracle://username:password@tnsName
+            oracle://username:password@host:port/sidname
+
+            """
+            hostjoin = ''
+            if 'dialect' in options:
+                self.dburl = '%s://' % options['dialect']
+                del options['dialect']
+            if 'user' in options:
+                self.dburl = '%s%s' % (self.dburl, options['user'])
+                hostjoin='@'
+                del options['user']
+            if 'username' in options:
+                self.dburl = '%s%s' % (self.dburl, options['username'])
+                hostjoin='@'
+                del options['username']
+            if 'passwd' in options:
+                self.dburl = '%s:%s' % (self.dburl, options['passwd'])
+                del options['passwd']
+            if 'password' in options:
+                self.dburl = '%s:%s' % (self.dburl, options['password'])
+                del options['password']
+            if 'tnsName' in options:
+                self.dburl = '%s%s%s' % (self.dburl, hostjoin, options['tnsName'])
+                del options['tnsName']
+            elif 'host' in options:
+                self.dburl = '%s%s%s' % (self.dburl, hostjoin, options['host'])
+                del options['host']
+                if 'port' in options:
+                    self.dburl = '%s:%s' % (self.dburl, options['port'])
+                    del options['port']
+            if 'host' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['host'])
+                del options['host']
+            if 'database' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['database'])
+                del options['database']
+            elif 'sid' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['sid'])
+                del options['sid']
+
+        if self.dburl.split(':')[0].lower() == "http":
+
+            self.engine = None
+            self.dia = None
+
+        else:
+            self.engine = self._engineMap.setdefault(self.dburl,
+                                                     create_engine(self.dburl,
+                                                                   connect_args=options,
+                                                                   **self._defaultEngineParams)
+                                                     )
+            self.dia = self.engine.dialect
+
+        self.lock = threading.Condition()
+
+
+    def connect(self):
+        self.lock.acquire()
+
+        if self.engine:
+
+            self.logger.debug("Using SQLAlchemy v.%s" % sqlalchemy_version)
+
+            if isinstance(self.dia, MySQLDialect):
+                from WMCore.Database.MySQLCore import MySQLInterface as DBInterface
+            else:
+                from WMCore.Database.DBCore import DBInterface
+            # we instantiate within the lock so we can safely return the local instance.
+            dbInterface =  DBInterface(self.logger, self.engine)
+
+        else:
+            dbInterface =  None
+
+        self.lock.release()
+        return dbInterface