wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Database/DBCore.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+"""
+_Core_
+
+Core Database APIs
+
+
+"""
+from copy import copy
+
+from Utils.IteratorTools import grouper
+import WMCore.WMLogging
+from WMCore.DataStructs.WMObject import WMObject
+from WMCore.Database.ResultSet import ResultSet
+
+class DBInterface(WMObject):
+    """
+    Base class for doing SQL operations using a SQLAlchemy engine, or
+    pre-exisitng connection.
+
+    processData will take a (list of) sql statements and a (list of)
+    bind variable dictionaries and run the statements on the DB. If
+    necessary it will substitute binds into the sql (MySQL).
+
+    TODO:
+        Add in some suitable exceptions in one or two places
+        Test the hell out of it
+        Support executemany()
+    """
+
+    logger = None
+    engine = None
+
+    def __init__(self, logger, engine):
+        self.logger = logger
+        self.logger.info ("Instantiating base WM DBInterface")
+        self.engine = engine
+        self.maxBindsPerQuery = 500
+
+    def buildbinds(self, sequence, thename, therest=[{}]):
+        """
+        Build a list of binds. Can be used recursively, e.g.:
+        buildbinds(file, 'file', buildbinds(pnn, 'location'), {'lumi':123})
+        TODO: replace with an appropriate map function
+        """
+        binds = []
+        for r in sequence:
+            for i in self.makelist(therest):
+                thebind = copy(i)
+                thebind[thename] = r
+                binds.append(thebind)
+        return binds
+
+    def executebinds(self, s=None, b=None, connection=None,
+                     returnCursor=False):
+        """
+        _executebinds_
+
+        returns a list of sqlalchemy.engine.base.ResultProxy objects
+        """
+        if b == None:
+            resultProxy = connection.execute(s)
+        else:
+            resultProxy = connection.execute(s, b)
+
+        if returnCursor:
+            return resultProxy
+
+        result = ResultSet()
+        result.add(resultProxy)
+        resultProxy.close()
+        return result
+
+    def executemanybinds(self, s=None, b=None, connection=None,
+                         returnCursor=False):
+        """
+        _executemanybinds_
+        b is a list of dictionaries for the binds, e.g.:
+
+        b = [ {'bind1':'value1a', 'bind2': 'value2a'},
+        {'bind1':'value1b', 'bind2': 'value2b'} ]
+
+        see: http://www.gingerandjohn.com/archives/2004/02/26/cx_oracle-executemany-example/
+
+        Can't executemany() selects - so do each combination of binds here instead.
+        This will return a list of sqlalchemy.engine.base.ResultProxy object's
+        one for each set of binds.
+
+        returns a list of sqlalchemy.engine.base.ResultProxy objects
+        """
+
+        s = s.strip()
+        if s.lower().endswith('select', 0, 6):
+            """
+            Trying to select many
+            """
+            if returnCursor:
+                result = []
+                for bind in b:
+                    result.append(connection.execute(s, bind))
+            else:
+                result = ResultSet()
+                for bind in b:
+                    resultproxy = connection.execute(s, bind)
+                    result.add(resultproxy)
+                    resultproxy.close()
+
+            return self.makelist(result)
+
+        """
+        Now inserting or updating many
+        """
+        result = connection.execute(s, b)
+        return self.makelist(result)
+
+    def connection(self):
+        """
+        Return a connection to the engine (from the connection pool)
+        """
+        return self.engine.connect()
+
+
+    def processData(self, sqlstmt, binds={}, conn=None,
+                    transaction=False, returnCursor=False):
+        """
+        set conn if you already have an active connection to reuse
+        set transaction = True if you already have an active transaction
+
+        """
+        connection = None
+        try:
+            if not conn:
+                connection = self.connection()
+            else:
+                connection = conn
+
+            result = []
+            # Can take either a single statement or a list of statements and binds
+            sqlstmt = self.makelist(sqlstmt)
+            binds = self.makelist(binds)
+            if len(sqlstmt) > 0 and (len(binds) == 0 or (binds[0] == {} or binds[0] == None)):
+                # Should only be run by create statements
+                if not transaction:
+                    #WMCore.WMLogging.sqldebug("transaction created in DBInterface")
+                    trans = connection.begin()
+
+                for i in sqlstmt:
+                    r = self.executebinds(i, connection=connection,
+                                          returnCursor=returnCursor)
+                    result.append(r)
+
+                if not transaction:
+                    trans.commit()
+            elif len(binds) > len(sqlstmt) and len(sqlstmt) == 1:
+                #Run single SQL statement for a list of binds - use execute_many()
+                if not transaction:
+                    trans = connection.begin()
+                for subBinds in grouper(binds, self.maxBindsPerQuery):
+                    result.extend(self.executemanybinds(sqlstmt[0], subBinds,
+                                                        connection=connection, returnCursor=returnCursor))
+
+                if not transaction:
+                    trans.commit()
+            elif len(binds) == len(sqlstmt):
+                # Run a list of SQL for a list of binds
+                if not transaction:
+                    trans = connection.begin()
+
+                for i, s in enumerate(sqlstmt):
+                    b = binds[i]
+
+                    r = self.executebinds(s, b, connection=connection,
+                                          returnCursor=returnCursor)
+                    result.append(r)
+
+                if not transaction:
+                    trans.commit()
+            else:
+                self.logger.exception(
+                    "DBInterface.processData Nothing executed, problem with your arguments")
+                self.logger.exception(
+                    "DBInterface.processData SQL = %s" % sqlstmt)
+                WMCore.WMLogging.sqldebug('DBInterface.processData  sql is %s items long' % len(sqlstmt))
+                WMCore.WMLogging.sqldebug('DBInterface.processData  binds are %s items long' % len(binds))
+                assert_value = False
+                if len(binds) == len(sqlstmt):
+                    assert_value = True
+                WMCore.WMLogging.sqldebug('DBInterface.processData are binds and sql same length? : %s' % (assert_value))
+                WMCore.WMLogging.sqldebug('sql: %s\n binds: %s\n, connection:%s\n, transaction:%s\n' %
+                                           (sqlstmt, binds, connection, transaction))
+                WMCore.WMLogging.sqldebug('type check:\nsql: %s\n binds: %s\n, connection:%s\n, transaction:%s\n' %
+                                           (type(sqlstmt), type(binds), type(connection), type(transaction)))
+                raise Exception("""DBInterface.processData Nothing executed, problem with your arguments
+                Probably mismatched sizes for sql (%i) and binds (%i)""" % (len(sqlstmt), len(binds)))
+        finally:
+            if not conn and connection != None:
+                connection.close() # Return connection to the pool
+        return result

WMCore/Database/DBCreator.py

@@ -0,0 +1,113 @@
+#!/usr/bin/python
+
+"""
+_DBCreator_
+
+Base class for formatters that create tables.
+
+"""
+
+from builtins import str
+
+from WMCore.Database.DBFormatter import DBFormatter
+from WMCore.WMException import WMException
+from WMCore.WMExceptions import WMEXCEPTION
+
+
+class DBCreator(DBFormatter):
+
+    """
+    _DBCreator_
+
+    Generic class for creating database tables.
+
+    """
+
+    def __init__(self, logger, dbinterface):
+        """
+        _init_
+
+        Call the constructor of the parent class and create empty dictionaries
+        to hold table create statements, constraint statements and insert
+        statements.
+        """
+        DBFormatter.__init__(self, logger, dbinterface)
+        self.create = {}
+        self.constraints = {}
+        self.inserts = {}
+        self.indexes = {}
+
+    def execute(self, conn = None, transaction = False):
+        """
+        _execute_
+
+        Generic method to create tables and constraints by execute
+        sql statements in the create, and constraints dictionaries.
+
+        Before execution the keys assigned to the tables in the self.create
+        dictionary are sorted, to offer the possibilitiy of executing
+        table creation in a certain order.
+
+        """
+        # create tables
+        for i in sorted(self.create.keys()):
+            try:
+                self.dbi.processData(self.create[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.create[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # create indexes
+        for i in self.indexes:
+            try:
+                self.dbi.processData(self.indexes[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.indexes[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # set constraints
+        for i in self.constraints:
+            try:
+                self.dbi.processData(self.constraints[i],
+                                 conn = conn,
+                                 transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.constraints[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        # insert permanent data
+        for i in self.inserts:
+            try:
+                self.dbi.processData(self.inserts[i],
+                                     conn = conn,
+                                     transaction = transaction)
+            except Exception as e:
+                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
+                                  str(self.inserts[i]) +'\n\n' +str(e)
+                self.logger.debug( msg )
+                raise WMException(msg,'WMCORE-2')
+
+        return True
+
+    def __str__(self):
+        """
+        _str_
+
+        Return a well formatted text representation of the schema held in the
+        self.create, self.constraints, self.inserts, self.indexes dictionaries.
+        """
+        string = ''
+        for i in self.create, self.constraints, self.inserts, self.indexes:
+            for j in i:
+                string = string + i[j].lstrip() + '\n'
+        return string

WMCore/Database/DBExceptionHandler.py

@@ -0,0 +1,59 @@
+from __future__ import division, print_function
+
+from builtins import str
+from functools import wraps
+import logging
+import threading
+
+### Don't crash the components on the following database exceptions
+# ORA-03113: end-of-file on communication channel
+# ORA-03114: Not Connected to Oracle
+# ORA-03135: connection lost contact
+# ORA-12545: Connect failed because target host or object does not exist
+# ORA-00060: deadlock detected while waiting for resource
+# ORA-01033: ORACLE initialization or shutdown in progress
+# (cx_Oracle.InterfaceError) not connected  # same as ORA-03114, in the new SQLAlchemy
+# (cx_Oracle.DatabaseError) ORA-25408: can not safely replay call
+# (cx_Oracle.DatabaseError) ORA-25401: can not continue fetches
+# (cx_Oracle.DatabaseError) DPI-1080: connection was closed by ORA-3113
+# and those two MySQL exceptions
+DB_CONNECTION_ERROR_STR = ["ORA-03113", "ORA-03114", "ORA-03135", "ORA-12545", "ORA-00060", "ORA-01033",
+                           "MySQL server has gone away", "Lock wait timeout exceeded",
+                           "(cx_Oracle.InterfaceError) not connected", "ORA-25408", "ORA-25401",
+                           "DPI-1080: connection was closed by ORA"]
+
+
+def db_exception_handler(f):
+    """
+    :param f: function
+    :return: wrapper fuction
+
+    Only need to handle DB connection problem other db problems which need to rollback the transaction,
+    shouldn't be included in this hanldler
+
+    Warning: This only used, when original function return values are not used, or wrapped around Utils.Timer.timeFunc
+    """
+
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        myThread = threading.currentThread()
+        if hasattr(myThread, "logdbClient") and myThread.logdbClient is not None:
+            myThread.logdbClient.delete("DBConnection_error_handler", "warning", this_thread=True)
+        try:
+            return f(*args, **kwargs)
+        except Exception as ex:
+            msg = str(ex)
+            for errStr in DB_CONNECTION_ERROR_STR:
+                if errStr in msg:
+                    logging.error("%s: Temp error will try later", msg)
+
+                    if hasattr(myThread, "logdbClient") and myThread.logdbClient is not None:
+                        myThread.logdbClient.post("DBConnection_error_handler", "warning")
+
+                    # returns tuples of 3 since timeFunc returns (time to spend, result, func.__name__)
+                    return 0, None, f.__name__
+
+            # for other case raise the same exception
+            raise
+
+    return wrapper

WMCore/Database/DBFactory.py

@@ -0,0 +1,117 @@
+#pylint: disable=W0102
+from builtins import object
+import threading
+import sys
+import oracledb
+
+# NOTE: We need to ensure SQLAlchemy would be able to find the oracledb python libraries.
+#       Further details at: https://cjones-oracle.medium.com/using-python-oracledb-1-0-with-sqlalchemy-pandas-django-and-flask-5d84e910cb19
+oracledb.version = "8.3.0"
+sys.modules["cx_Oracle"] = oracledb
+oracledb.init_oracle_client()
+
+
+from sqlalchemy import create_engine
+from sqlalchemy import __version__ as sqlalchemy_version
+from WMCore.Database.Dialects import MySQLDialect
+
+class DBFactory(object):
+
+    # class variable
+    _engineMap = {}
+    _defaultEngineParams = {"convert_unicode" : True,
+                            "pool_recycle": 7200}
+
+    def __init__(self, logger, dburl=None, options={}):
+        self.logger = logger
+        # get the engine parameter from option
+        if 'engine_parameters' in options:
+            self._defaultEngineParams.update(options['engine_parameters'])
+            del options['engine_parameters']
+
+        if dburl:
+            self.dburl = dburl
+        else:
+            # NOTE: This will be deprecated.
+            #
+            #       Need to make the dburl here. Possible formats are:
+
+            #       mysql://host/database
+            #       mysql://username@host/database
+            #       mysql://username:password@host:port/database
+
+            #       oracle://username:password@tnsName
+            #       oracle://username:password@host:port/sidname
+
+            hostjoin = ''
+            if 'dialect' in options:
+                self.dburl = '%s://' % options['dialect']
+                del options['dialect']
+            if 'user' in options:
+                self.dburl = '%s%s' % (self.dburl, options['user'])
+                hostjoin='@'
+                del options['user']
+            if 'username' in options:
+                self.dburl = '%s%s' % (self.dburl, options['username'])
+                hostjoin='@'
+                del options['username']
+            if 'passwd' in options:
+                self.dburl = '%s:%s' % (self.dburl, options['passwd'])
+                del options['passwd']
+            if 'password' in options:
+                self.dburl = '%s:%s' % (self.dburl, options['password'])
+                del options['password']
+            if 'tnsName' in options:
+                self.dburl = '%s%s%s' % (self.dburl, hostjoin, options['tnsName'])
+                del options['tnsName']
+            elif 'host' in options:
+                self.dburl = '%s%s%s' % (self.dburl, hostjoin, options['host'])
+                del options['host']
+                if 'port' in options:
+                    self.dburl = '%s:%s' % (self.dburl, options['port'])
+                    del options['port']
+            if 'host' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['host'])
+                del options['host']
+            if 'database' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['database'])
+                del options['database']
+            elif 'sid' in options:
+                self.dburl = '%s/%s' % (self.dburl, options['sid'])
+                del options['sid']
+
+        if self.dburl.split(':')[0].lower() == "http":
+
+            self.engine = None
+            self.dia = None
+
+        else:
+            self.engine = self._engineMap.setdefault(self.dburl,
+                                                     create_engine(self.dburl,
+                                                                   connect_args=options,
+                                                                   **self._defaultEngineParams)
+                                                     )
+            self.dia = self.engine.dialect
+
+        self.lock = threading.Condition()
+
+
+    def connect(self):
+        self.lock.acquire()
+
+        if self.engine:
+
+            self.logger.debug("Using SQLAlchemy v.%s" % sqlalchemy_version)
+
+            if isinstance(self.dia, MySQLDialect):
+                from WMCore.Database.MySQLCore import MySQLInterface as DBInterface
+            else:
+                from WMCore.Database.DBCore import DBInterface
+            # we instantiate within the lock so we can safely return the local instance.
+            dbInterface =  DBInterface(self.logger, self.engine)
+
+        else:
+            dbInterface =  None
+
+        self.lock.release()
+        return dbInterface

WMCore/Database/DBFormatter.py

@@ -0,0 +1,177 @@
+"""
+_DBFormatter_
+
+A class to define the standardised formatting of database results.
+Holds a bunch of helper methods to format input and output of sql
+interactions.
+"""
+
+from builtins import str, bytes, zip, range
+
+import datetime
+import time
+import types
+
+from Utils.Utilities import decodeBytesToUnicodeConditional
+from WMCore.DataStructs.WMObject import WMObject
+from Utils.PythonVersion import PY3
+
+
+class DBFormatter(WMObject):
+    def __init__(self, logger, dbinterface):
+        """
+        The class holds a connection to the database in self.dbi. This is a
+        DBCore.DBInterface object.
+        """
+        self.logger = logger
+        self.dbi = dbinterface
+
+    def truefalse(self, value):
+        if value in ('False', 'FALSE', 'n', 'N', 'NO', 'No'):
+            value = 0
+        return bool(value)
+
+    def convertdatetime(self, t):
+        return int(time.mktime(t.timetuple()))
+
+    def timestamp(self):
+        """
+        generate a timestamp
+        """
+        t = datetime.datetime.now()
+        return self.convertdatetime(t)
+
+    def format(self, result):
+        """
+        Some standard formatting, put all records into a list.
+        Returns a list of lists
+        """
+        out = []
+        for r in result:
+            for i in r.fetchall():
+                row = []
+                for j in i:
+                    row.append(j)
+                out.append(row)
+            r.close()
+        return out
+
+    def formatOne(self, result):
+        """
+        Return one record
+        """
+        out = []
+        for r in result:
+            for i in r.fetchone():
+                out.append(i)
+        return out
+
+    def formatDict(self, result):
+        """
+        Returns an array of dictionaries representing the results
+        """
+        dictOut = []
+        for r in result:
+            descriptions = r.keys
+            for i in r.fetchall():
+                # WARNING: this can generate errors for some stupid reason
+                # in both oracle and mysql.
+                entry = {}
+                for index in range(0, len(descriptions)):
+                    # WARNING: Oracle returns table names in CAP!
+                    if isinstance(descriptions[index], (str, bytes)):
+                        keyName = decodeBytesToUnicodeConditional(descriptions[index], condition=PY3)
+                    else:
+                        keyName = descriptions[index]
+                    if isinstance(i[index], (str, bytes)):
+                        entry[keyName.lower()] = decodeBytesToUnicodeConditional(i[index], condition=PY3)
+                    else:
+                        entry[keyName.lower()] = i[index]
+
+                dictOut.append(entry)
+
+            r.close()
+
+        return dictOut
+
+    def formatList(self, result):
+        """
+        Returns a flat array with the results.
+        Ideally used for single column queries
+        """
+        listOut = []
+        for r in result:
+            descriptions = r.keys
+            for i in r.fetchall():
+                for index in range(0, len(descriptions)):
+                    if isinstance(i[index], (str, bytes)):
+                        listOut.append(decodeBytesToUnicodeConditional(i[index], condition=PY3))
+                    else:
+                        listOut.append(i[index])
+            r.close()
+        return listOut
+
+    def formatOneDict(self, result):
+        """
+        Return a dictionary representing the first record
+        """
+        if (len(result) == 0):
+            return {}
+
+        r = result[0]
+        description = [str(x).lower() for x in r.keys]
+        if len(r.data) < 1:
+            return {}
+
+        return dict(list(zip(description, r.fetchone())))
+
+    def formatCursor(self, cursor, size=10):
+        """
+        Fetch the driver cursor directly.
+        Tested only with cx_Oracle.
+        Cursor must be already executed.
+        Use fetchmany(size = default arraysize = 50)
+
+        """
+        if isinstance(cursor.keys, types.MethodType):
+            keys = [x.lower() for x in cursor.keys()]  # warning: do not modernize this line.
+        else:
+            keys = [x.lower() for x in cursor.keys]
+        result = []
+        while True:
+            if not cursor.closed:
+                rows = cursor.fetchmany(size=size)
+                if not rows:
+                    cursor.close()
+                    break
+                for r in rows:
+                    result.append(dict(list(zip(keys, r))))
+            else:
+                break
+        if not cursor.closed:
+            cursor.close()
+        return result
+
+    def getBinds(self, **kwargs):
+        binds = {}
+        for i in kwargs:
+            binds = self.dbi.buildbinds(self.dbi.makelist(kwargs[i]), i, binds)
+        return binds
+
+    def execute(self, conn=None, transaction=False, returnCursor=False):
+        """
+        A simple select with no binds/arguments is the default
+        """
+        result = self.dbi.processData(self.sql, self.getBinds(),
+                                      conn=conn, transaction=transaction,
+                                      returnCursor=returnCursor)
+        return self.format(result)
+
+    def executeOne(self, conn=None, transaction=False, returnCursor=False):
+        """
+        A simple select with no binds/arguments is the default
+        """
+        result = self.dbi.processData(self.sql, self.getBinds(),
+                                      conn=conn, transaction=transaction,
+                                      returnCursor=returnCursor)
+        return self.formatOne(result)

WMCore/Database/Dialects.py

@@ -0,0 +1,13 @@
+
+"""
+sqlalchemy breaks this because they reorganized things. Fail.
+see: http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg18392.html
+  -AMM 6/15/10
+"""
+
+try:
+    from sqlalchemy.databases.mysql import MySQLDialect
+    from sqlalchemy.databases.oracle import OracleDialect
+except:
+    from sqlalchemy.dialects.mysql.base import MySQLDialect
+    from sqlalchemy.dialects.oracle.base import OracleDialect