wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/Database/MongoDB.py

@@ -0,0 +1,241 @@
+"""
+File       : MongoDB.py
+Description: Provides a wrapper class for MongoDB
+"""
+
+# futures
+from __future__ import division, print_function
+from builtins import str, object
+
+try:
+    import mongomock
+except ImportError:
+    # this library should only be required by unit tests
+    mongomock = None
+
+from pymongo import MongoClient, errors, IndexModel
+from pymongo.errors import ConnectionFailure
+
+
+class MongoDB(object):
+    """
+    A simple wrapper class for creating a connection to a MongoDB instance
+    """
+    def __init__(self, database=None, server=None,
+                 create=False, collections=None, testIndexes=False,
+                 logger=None, mockMongoDB=False, **kwargs):
+        """
+        :databases:   A database Name to connect to
+        :server:      The server url or a list of (server:port) pairs (see https://docs.mongodb.com/manual/reference/connection-string/)
+        :create:      A flag to trigger a database creation (if missing) during
+                      object construction, together with collections if present.
+        :collections: A list of tuples describing collections with indexes -
+                      the first element is considered the collection name, all
+                      the rest elements are considered as indexes
+        :testIndexes: A flag to trigger index test and eventually to create them
+                      if missing (TODO)
+        :mockMongoDB: A flag to trigger a database simulation instead of trying
+                      to connect to a real database server.
+        :logger:      Logger
+
+        Here follows a short list of usefull optional parameters accepted by the
+        MongoClient which may be passed as keyword arguments to the current module:
+
+        :replicaSet:       The name of the replica set to connect to. The driver will verify
+                           that all servers it connects to match this name. Implies that the
+                           hosts specified are a seed list and the driver should attempt to
+                           find all members of the set. Defaults to None.
+        :port:             The port number on which to connect. It is overwritten by the ports
+                           defined in the Url string or from the tuples listed in the server list
+        :connect:          If True, immediately begin connecting to MongoDB in the background.
+                           Otherwise connect on the first operation.
+        :directConnection: If True, forces the client to connect directly to the specified MongoDB
+                           host as a standalone. If False, the client connects to the entire
+                           replica set of which the given MongoDB host(s) is a part.
+                           If this is True and a mongodb+srv:// URI or a URI containing multiple
+                           seeds is provided, an exception will be raised.
+        :username:         A string
+        :password:         A string
+                           Although username and password must be percent-escaped in a MongoDB URI,
+                           they must not be percent-escaped when passed as parameters. In this example,
+                           both the space and slash special characters are passed as-is:
+                           MongoClient(username="user name", password="pass/word")
+        """
+        self.server = server
+        self.logger = logger
+        self.mockMongoDB = mockMongoDB
+        if mockMongoDB and mongomock is None:
+            msg = "You are trying to mock MongoDB, but you do not have mongomock in the python path."
+            self.logger.critical(msg)
+            raise ImportError(msg)
+
+        # NOTE: We need to explicitely check for server availiability.
+        #       From pymongo Documentation: https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html
+        #       """
+        #           ...
+        #           Starting with version 3.0 the :class:`MongoClient`
+        #           constructor no longer blocks while connecting to the server or
+        #           servers, and it no longer raises
+        #           :class:`~pymongo.errors.ConnectionFailure` if they are
+        #           unavailable, nor :class:`~pymongo.errors.ConfigurationError`
+        #           if the user's credentials are wrong. Instead, the constructor
+        #           returns immediately and launches the connection process on
+        #           background threads.
+        #           ...
+        #       """
+        try:
+            if mockMongoDB:
+                self.client = mongomock.MongoClient()
+                self.logger.info("NOTICE: MongoDB is set to use mongomock, instead of real database.")
+            else:
+                self.client = MongoClient(host=self.server, **kwargs)
+            self.client.server_info()
+            self.client.admin.command('ping')
+        except ConnectionFailure as ex:
+            msg = "Could not connect to MongoDB server: %s. Server not available. \n"
+            msg += "Giving up Now."
+            self.logger.error(msg, self.server)
+            raise ex from None
+        except Exception as ex:
+            msg = "Could not connect to MongoDB server: %s. Due to unknown reason: %s\n"
+            msg += "Giving up Now."
+            self.logger.error(msg, self.server, str(ex))
+            raise ex from None
+        self.create = create
+        self.testIndexes = testIndexes
+        self.dbName = database
+        self.collections = collections or []
+
+        self._dbConnect(database)
+
+        if self.create and self.collections:
+            for collection in self.collections:
+                self._collCreate(collection, database)
+
+        if self.testIndexes and self.collections:
+            for collection in self.collections:
+                self._indexTest(collection[0], collection[1])
+
+    def _indexTest(self, collection, index):
+        pass
+
+    def _collTest(self, coll, db):
+        # self[db].list_collection_names()
+        pass
+
+    def collCreate(self, coll):
+        """
+        A public method for _collCreate
+        """
+        self._collCreate(coll, self.database)
+
+    def _collCreate(self, coll, db):
+        """
+        A function used to explicitly create a collection with the relevant
+        indexes - used to avoid the Lazy Creating from MongoDB and eventual issues
+        in case we end up with no indexed collection, especially ones missing
+        the (`unique` index parameter)
+        :coll: A tuple describing one collection with indexes -
+               The first element is considered to be the collection name, and all
+               the rest of the elements are considered to be indexes.
+               The indexes must be of type IndexModel. See pymongo documentation:
+
+               https://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.create_index
+
+        :db:   The database name for the collection
+        """
+
+        collName = coll[0]
+        collIndexes = list(coll[1:])
+        try:
+            self.client[db].create_collection(collName)
+        except errors.CollectionInvalid:
+            # this error is thrown in case of an already existing collection
+            msg = "Collection '{}' Already exists in database '{}'".format(coll, db)
+            self.logger.warning(msg)
+
+        if collIndexes:
+            for index in collIndexes:
+                if not isinstance(index, IndexModel):
+                    msg = "ERR: Bad Index type for collection %s" % collName
+                    raise errors.InvalidName
+            try:
+                self.client[db][collName].create_indexes(collIndexes)
+            except Exception as ex:
+                msg = "Failed to create indexes on collection: %s\n%s" % (collName, str(ex))
+                self.logger.error(msg)
+                raise ex
+
+    def _dbTest(self, db):
+        """
+        Tests database connection.
+        """
+        # Test connection (from mongoDB documentation):
+        # https://api.mongodb.com/python/3.4.0/api/pymongo/mongo_client.html
+        try:
+            # The 'ismaster' command is cheap and does not require auth.
+            self.client.admin.command('ismaster')
+        except errors.ConnectionFailure as ex:
+            msg = "Server not available: %s" % str(ex)
+            self.logger.error(msg)
+            raise ex
+
+        # Test for database existence
+        if db not in self.client.list_database_names():
+            msg = "Missing MongoDB databases: %s" % db
+            self.logger.error(msg)
+            raise errors.InvalidName
+
+    def _dbCreate(self, db):
+        # creating an empty collection in order to create the database
+        _initColl = self.client[db].create_collection('_initCollection')
+        _initColl.insert_one({})
+        # NOTE: never delete the _initCollection if you want the database to persist
+        # self.client[db].drop_collection('_initCollection')
+
+    def dbConnect(self):
+        """
+        A public method for _dbConnect
+        """
+        self._dbConnect(self.database)
+
+    def _dbConnect(self, db):
+        """
+        The function to be used for the initial database connection creation and testing
+        """
+        try:
+            setattr(self, db, self.client[db])
+            if not self.mockMongoDB:
+                self._dbTest(db)
+        except errors.ConnectionFailure as ex:
+            msg = "Could not connect to MongoDB server for database: %s\n%s\n" % (db, str(ex))
+            msg += "Giving up Now."
+            self.logger.error(msg)
+            raise ex
+        except errors.InvalidName as ex:
+            msg = "Could not connect to a missing MongoDB databases: %s\n%s" % (db, str(ex))
+            self.logger.error(msg)
+            if self.create:
+                msg = "Trying to create: %s" % db
+                self.logger.error(msg)
+                try:
+                    # self._dbCreate(getattr(self, db))
+                    self._dbCreate(db)
+                except Exception as exc:
+                    msg = "Could not create MongoDB databases: %s\n%s\n" % (db, str(exc))
+                    msg += "Giving up Now."
+                    self.logger.error(msg)
+                    raise exc
+                try:
+                    self._dbTest(db)
+                except Exception as exc:
+                    msg = "Second failure while testing %s\n%s\n" % (db, str(exc))
+                    msg += "Giving up Now."
+                    self.logger.error(msg)
+                    raise exc
+                msg = "Database %s successfully created" % db
+                self.logger.error(msg)
+        except Exception as ex:
+            msg = "General Exception while trying to connect to : %s\n%s" % (db, str(ex))
+            self.logger.error(msg)
+            raise ex

WMCore/Database/MySQL/Destroy.py

@@ -0,0 +1,42 @@
+"""
+_Destroy_
+
+Implementation of Destroy for MySQL
+
+"""
+
+from WMCore.Database.DBFormatter import DBFormatter
+
+class Destroy(DBFormatter):
+
+    def execute(self, subscription = None, conn = None, transaction = False):
+
+        sql = """SELECT DATABASE() AS dbname"""
+
+        results = self.dbi.processData(sql, {}, conn = conn,
+                                       transaction = transaction)
+
+        dbName = self.formatDict(results)[0]['dbname']
+
+        if dbName == None or dbName == 'None':
+            # Then we have no database.
+            # This presents us with a problem.  We've been asked to clear a
+            # non-existant DB.
+            # Obviously we can't drop it, so what we have to do is try
+            # to pull the URL from the threaded dbi and use that
+            # to create the proper db
+            dbName = self.dbi.engine.url.database
+        else:
+            sql = """DROP DATABASE %s""" % dbName
+            self.dbi.processData(sql, {}, conn = conn,
+                                 transaction = transaction)
+
+        sql = """CREATE DATABASE %s""" % dbName
+        self.dbi.processData(sql, {}, conn = conn,
+                             transaction = transaction)
+
+        sql = """USE %s""" % dbName
+        self.dbi.processData(sql, {}, conn = conn,
+                             transaction = transaction)
+
+        return

WMCore/Database/MySQL/ListUserContent.py

@@ -0,0 +1,20 @@
+"""
+_ListUserContent_
+
+Implementation of ListUserContent for MySQL
+
+"""
+
+from WMCore.Database.DBFormatter import DBFormatter
+
+class ListUserContent(DBFormatter):
+
+    def execute(self, subscription = None, conn = None, transaction = False):
+
+        sql = """SELECT table_name FROM information_schema.tables
+                 WHERE table_schema = (SELECT DATABASE())"""
+
+        result = self.dbi.processData(sql, {}, conn = conn,
+                                      transaction = transaction)
+
+        return self.formatDict(result)

WMCore/Database/MySQL/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+"""
+_Database_
+
+Core Database interface libraries for workload management
+
+
+"""
+__all__ = []

WMCore/Database/MySQLCore.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+"""
+_MySQLCore_
+
+Handle bind variable parsing for MySQL.
+"""
+
+import copy
+
+from WMCore.Database.DBCore import DBInterface
+from WMCore.Database.ResultSet import ResultSet
+
+def bindVarCompare(a):
+    """
+    _bindVarCompare_
+
+    Bind variables are represented as a tuple with the first element being the
+    variable name and the second being it's position in the query.  We sort on
+    the position in the query.
+    """
+    return a[1]
+
+def stringLengthCompare(a):
+    """
+    _stringLengthCompare_
+
+    Sort comparison function to sort strings by length.
+    Since we want to sort from longest to shortest, this must be reversed when used
+    """
+    return len(a)
+
+
+class MySQLInterface(DBInterface):
+    def substitute(self, origSQL, origBindsList):
+        """
+        _substitute_
+
+        Transform as set of bind variables from a list of dictionaries to a list
+        of tuples:
+
+        b = [ {'bind1':'value1a', 'bind2': 'value2a'},
+        {'bind1':'value1b', 'bind2': 'value2b'} ]
+
+        Will be transformed into:
+
+        b = [ ('value1a', 'value2a'), ('value1b', 'value2b')]
+
+        Don't need to substitute in the binds as executemany does that
+        internally. But the sql will also need to be reformatted, such that
+        :bind_name becomes %s.
+
+        See: http://www.devshed.com/c/a/Python/MySQL-Connectivity-With-Python/5/
+        """
+        if origBindsList == None:
+            return origSQL, None
+
+        origBindsList = self.makelist(origBindsList)
+        origBind = origBindsList[0]
+
+        bindVarPositionList = []
+        updatedSQL = copy.copy(origSQL)
+
+        # We process bind variables from longest to shortest to avoid a shorter
+        # bind variable matching a longer one.  For example if we have two bind
+        # variables: RELEASE_VERSION and RELEASE_VERSION_ID the former will
+        # match against the latter, causing problems.  We'll sort the variable
+        # names by length to guard against this.
+        bindVarNames = list(origBind)
+        bindVarNames.sort(key=stringLengthCompare, reverse=True)
+
+        bindPositions = {}
+        for bindName in bindVarNames:
+            searchPosition = 0
+
+            while True:
+                bindPosition = origSQL.lower().find(":%s" % bindName.lower(),
+                                                    searchPosition)
+                if bindPosition == -1:
+                    break
+
+                if bindPosition not in bindPositions:
+                    bindPositions[bindPosition] = 0
+                    bindVarPositionList.append((bindName, bindPosition))
+                searchPosition = bindPosition + 1
+
+            searchPosition = 0
+            while True:
+                bindPosition = updatedSQL.lower().find(":%s" % bindName.lower(),
+                                                       searchPosition)
+
+                if bindPosition == -1:
+                    break
+
+                left = updatedSQL[0:bindPosition]
+                right = updatedSQL[bindPosition + len(bindName) + 1:]
+                updatedSQL = left + "%s" + right
+
+        bindVarPositionList.sort(key=bindVarCompare)
+
+        mySQLBindVarsList = []
+        for origBind in origBindsList:
+            mySQLBindVars = []
+            for bindVarPosition in bindVarPositionList:
+                mySQLBindVars.append(origBind[bindVarPosition[0]])
+
+            mySQLBindVarsList.append(tuple(mySQLBindVars))
+
+        return (updatedSQL, mySQLBindVarsList)
+
+    def executebinds(self, s = None, b = None, connection = None,
+                     returnCursor = False):
+        """
+        _executebinds_
+
+        Execute a SQL statement that has a single set of bind variables.
+        Transform the bind variables into the format that MySQL expects.
+        """
+        s, b = self.substitute(s, b)
+        return DBInterface.executebinds(self, s, b, connection, returnCursor)
+
+    def executemanybinds(self, s = None, b = None, connection = None,
+                         returnCursor = False):
+        """
+        _executemanybinds_
+
+        Execute a SQL statement that has multiple sets of bind variables.
+        Transform the bind variables into the format that MySQL expects.
+        """
+        newsql, binds = self.substitute(s, b)
+
+        return DBInterface.executemanybinds(self, newsql, binds, connection,
+                                            returnCursor)

WMCore/Database/Oracle/Destroy.py

@@ -0,0 +1,56 @@
+"""
+_Destroy_
+
+Implementation of Destroy for Oracle
+
+"""
+
+from WMCore.Database.DBFormatter import DBFormatter
+
+class Destroy(DBFormatter):
+
+    def execute(self, subscription = None, conn = None, transaction = False):
+
+        sql = """DECLARE
+                 BEGIN
+
+                   execute immediate 'purge recyclebin';
+
+                   -- Tables
+                   FOR o IN (SELECT table_name name FROM user_tables) LOOP
+                     execute immediate 'drop table ' || o.name || ' cascade constraints';
+                   END LOOP;
+
+                   -- Sequences
+                   FOR o IN (SELECT sequence_name name FROM user_sequences) LOOP
+                     execute immediate 'drop sequence ' || o.name;
+                   END LOOP;
+
+                   -- Triggers
+                   FOR o IN (SELECT trigger_name name FROM user_triggers) LOOP
+                     execute immediate 'drop trigger ' || o.name;
+                   END LOOP;
+
+                   -- Synonyms
+                   FOR o IN (SELECT synonym_name name FROM user_synonyms) LOOP
+                     execute immediate 'drop synonym ' || o.name;
+                   END LOOP;
+
+                   -- Functions
+                   FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'FUNCTION') LOOP
+                     execute immediate 'drop function ' || o.name;
+                   END LOOP;
+
+                   -- Procedures
+                   FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'PROCEDURE') LOOP
+                     execute immediate 'drop procedure ' || o.name;
+                   END LOOP;
+
+                   execute immediate 'purge recyclebin';
+
+                 END;"""
+
+        self.dbi.processData(sql, {}, conn = conn,
+                             transaction = transaction)
+
+        return

WMCore/Database/Oracle/ListUserContent.py

@@ -0,0 +1,19 @@
+"""
+_ListUserContent_
+
+Implementation of ListUserContent for Oracle
+
+"""
+
+from WMCore.Database.DBFormatter import DBFormatter
+
+class ListUserContent(DBFormatter):
+
+    def execute(self, subscription = None, conn = None, transaction = False):
+
+        sql = """SELECT object_name FROM user_objects"""
+
+        result = self.dbi.processData(sql, {}, conn = conn,
+                                      transaction = transaction)
+
+        return self.formatDict(result)

WMCore/Database/Oracle/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+"""
+_Database_
+
+Core Database interface libraries for workload management
+
+
+"""
+__all__ = []

WMCore/Database/ResultSet.py

@@ -0,0 +1,44 @@
+"""
+_ResultSet_
+
+A class to read in a SQLAlchemy result proxy and hold the data, such that the
+SQLAlchemy result sets (aka cursors) can be closed. Make this class look as much
+like the SQLAlchemy class to minimise the impact of adding this class.
+"""
+
+from builtins import object
+import threading
+
+
+class ResultSet(object):
+    def __init__(self):
+        self.data = []
+        self.keys = []
+
+    def close(self):
+        return
+
+    def fetchone(self):
+        if len(self.data) > 0:
+            return self.data[0]
+        else:
+            return []
+
+    def fetchall(self):
+        return self.data
+
+    def add(self, resultproxy):
+
+        myThread = threading.currentThread()
+
+        if resultproxy.closed:
+            return
+        elif resultproxy.returns_rows:
+            for r in resultproxy:
+                if len(self.keys) == 0:
+                    # do not modernize next line. 
+                    # r is a `sqlalchemy.engine.result.RowProxy`, not a `dict`
+                    self.keys.extend(r.keys()) 
+                self.data.append(r)
+
+        return

WMCore/Database/Transaction.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+"""
+_Transaction_
+
+A simple wrapper around DBInterface to make working with transactions simpler
+
+On MySQL transactions only work for innodb tables.
+"""
+
+
+
+import logging
+import time
+
+from WMCore.DataStructs.WMObject import WMObject
+from WMCore.WMException import WMException
+from WMCore.WMExceptions import WMEXCEPTION
+
+class Transaction(WMObject):
+    dbi = None
+
+    def __init__(self, dbinterface = None):
+        """
+        Get the connection from the DBInterface and open a new transaction on it
+        """
+        self.dbi = dbinterface
+        self.conn = None
+        self.transaction = None
+
+    def begin(self):
+        if self.conn == None:
+            self.conn = self.dbi.connection()
+        if self.conn.closed:
+            self.conn = self.dbi.connection()
+
+        if self.transaction == None:
+            self.transaction = self.conn.begin()
+
+        return
+
+    def processData(self, sql, binds={}):
+        """
+        Propagates the request to the proper dbcore backend,
+        and performs checks for lost (or closed) connection.
+        """
+        result = self.dbi.processData(sql, binds, conn = self.conn,
+                                      transaction = True)
+        return result
+
+    def commit(self):
+        """
+        Commit the transaction and return the connection to the pool
+        """
+        if not self.transaction == None:
+            self.transaction.commit()
+
+        if not self.conn == None:
+            self.conn.close()
+        self.conn = None
+        self.transaction = None
+
+    def rollback(self):
+        """
+        To be called if there is an exception and you want to roll back the
+        transaction and return the connection to the pool
+        """
+        if self.transaction:
+            self.transaction.rollback()
+
+        if self.conn:
+            self.conn.close()
+
+        self.conn = None
+        self.transaction = None
+        return
+
+    def rollbackForError(self):
+        """
+        This is called when handling a major exception.  This is because sometimes
+        you can end up in a situation where the transaction appears open, but is not.  In
+        this case, calling a rollback on the transaction will cause an exception, which
+        then destroys all logging and shutdown of the actual code.
+
+        Use only in components.
+        """
+
+        try:
+            self.rollback()
+        except:
+            pass
+        return

WMCore/Database/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+"""
+_Database_
+
+Core Database interface libraries for workload management
+
+
+"""
+__all__ = []