wmglobalqueue 2.4.0.2__py3-none-any.whl → 2.4.2__py3-none-any.whl

WMCore/Services/RucioConMon/RucioConMon.py

@@ -11,7 +11,6 @@ from future import standard_library
 
 from urllib.parse import urlencode
 
-import gzip
 import json
 import logging
 
@@ -40,14 +39,13 @@ class RucioConMon(Service):
         super(RucioConMon, self).__init__(configDict)
         self['logger'].debug("Initializing RucioConMon with url: %s", self['endpoint'])
 
-    def _getResult(self, uri, callname="", clearCache=False, args=None, binary=False):
+    def _getResult(self, uri, callname="", clearCache=False, args=None):
         """
         Either fetch data from the cache file or query the data-service
         :param uri: The endpoint uri
         :param callname: alias for caller function
         :param clearCache: parameter to control the cache behavior
         :param args: additional parameters to HTTP request call
-        :param binary: specifies request for binary object from HTTP requests (e.g. zipped content)
         :return:    A dictionary
         """
 
@@ -67,32 +65,25 @@ class RucioConMon(Service):
 
         if clearCache:
             self.clearCache(cachedApi, args)
-        results = '{}' # explicitly define results which will be loaded by json.loads below
-        if binary:
-            with self.refreshCache(cachedApi, apiUrl, decoder=False, binary=True) as istream:
-                results = gzip.decompress(istream.read())
-            return results
-        else:
-            with self.refreshCache(cachedApi, apiUrl) as istream:
-                results = istream.read()
-
-        results = json.loads(results)
-        return results
+        with self.refreshCache(cachedApi, apiUrl, decoder=True, binary=False) as istream:
+            return json.load(istream)
 
-    def _getResultZipped(self, uri, callname="", clearCache=True, args=None):
+    def _getResultZipped(self, uri, callname="", clearCache=True):
         """
-        This method is retrieving a zipped file from the uri privided, instead
-        of the normal json
+        This method retrieves gzipped content, instead of the standard json format.
         :param uri: The endpoint uri
         :param callname: alias for caller function
         :param clearCache: parameter to control the cache behavior
-        :param args: additional parameters to HTTP request call
-        :return:    a list of LFNs
+        :return: yields a single record from the data retrieved
         """
-        data = self._getResult(uri, callname, clearCache, args, binary=True)
-        # convert bytes which we received upstream to string
-        data = decodeBytesToUnicode(data)
-        return [f for f in data.split('\n') if f]
+        cachedApi = callname
+        if clearCache:
+            self.clearCache(cachedApi)
+
+        with self.refreshCache(cachedApi, uri, decoder=False, binary=True) as istream:
+            for line in istream:
+                line = decodeBytesToUnicode(line).replace("\n", "")
+                yield line
 
     def getRSEStats(self):
         """
@@ -109,7 +100,7 @@ class RucioConMon(Service):
         Gets the list of all unmerged files in an RSE
         :param rseName: The RSE whose list of unmerged files to be retrieved
         :param zipped:  If True the interface providing the zipped lists will be called
-        :return:        A list of unmerged files for the RSE in question
+        :return: a generator of unmerged files for the RSE in question
         """
         # NOTE: The default API provided by Rucio Consistency Monitor is in a form of a
         #       zipped file/stream. Currently we are using the newly provided json API
@@ -117,12 +108,14 @@ class RucioConMon(Service):
         #       implement the method with the zipped API and use disc cache for
         #       reading/streaming from file. This will prevent any set arithmetic
         #       in the future.
-        if not zipped:
-            uri = "files?rse=%s&format=json" % rseName
-            rseUnmerged = self._getResult(uri, callname=rseName)
-            return rseUnmerged
-        else:
+        if zipped:
             uri = "files?rse=%s&format=raw" % rseName
             callname = '{}.zipped'.format(rseName)
-            rseUnmerged = self._getResultZipped(uri,  callname=callname, clearCache=True)
-            return rseUnmerged
+            rseUnmerged = self._getResultZipped(uri, callname=callname, clearCache=True)
+        else:
+            uri = "files?rse=%s&format=json" % rseName
+            callname = '{}.json'.format(rseName)
+            rseUnmerged = self._getResult(uri, callname=callname)
+        # now lazily return items
+        for item in rseUnmerged:
+            yield item

WMCore/Services/WMStatsServer/WMStatsServer.py

@@ -128,7 +128,7 @@ class WMStatsServer(Service):
     def getProtectedLFNs(self):
         """
         A method to be used for fetching a list of all protected lfns from WMStatServer
-        :returns: A list of lfns
+        :returns: a unique list of protected LFNs
         """
         callname = 'protectedlfns'
-        return self._getResult(callname, verb="GET")
+        return list(set(self._getResult(callname, verb="GET")))

WMCore/WMInit.py

@@ -13,6 +13,8 @@ import os.path
 import sys
 import threading
 import traceback
+import warnings
+import wmcoredb
 
 from WMCore.Configuration import loadConfigurationFile
 from WMCore.DAOFactory import DAOFactory
@@ -20,7 +22,6 @@ from WMCore.Database.DBFactory import DBFactory
 from WMCore.Database.Transaction import Transaction
 from WMCore.WMBase import getWMBASE
 from WMCore.WMException import WMException
-from WMCore.WMFactory import WMFactory
 
 
 class WMInitException(WMException):
@@ -54,11 +55,12 @@ def connectToDB():
 
     socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
     connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
-    (dialect, junk) = connectUrl.split(":", 1)
+    (dialect, _) = connectUrl.split(":", 1)
 
     myWMInit = WMInit()
     myWMInit.setDatabaseConnection(dbConfig=connectUrl, dialect=dialect,
                                    socketLoc=socketLoc)
+
     return
 
 
@@ -113,12 +115,12 @@ class WMInit(object):
             return
 
         options = {}
-        if dialect.lower() == 'mysql':
-            dialect = 'MySQL'
+        if dialect.lower() in ['mysql', 'mariadb']:
+            dialect = 'mariadb'  # Both MySQL and MariaDB use the mariadb directory
             if socketLoc != None:
                 options['unix_socket'] = socketLoc
         elif dialect.lower() == 'oracle':
-            dialect = 'Oracle'
+            dialect = 'oracle'  # Keep lowercase for consistency
         elif dialect.lower() == 'http':
             dialect = 'CouchDB'
         else:
@@ -146,31 +148,178 @@ class WMInit(object):
 
         This method needs to have been preceded by the
         setDatabaseConnection.
+
+        @deprecated: Use setSchemaFromModules instead
+        """
+        warnings.warn("setSchema is deprecated. Use setSchemaFromModules instead.", DeprecationWarning)
+
+        # create a map of old to new SQL module names
+        moduleMap = {
+            'WMCore.WMBS': 'wmbs',
+            'WMCore.ResourceControl': 'resourcecontrol',
+            'WMCore.BossAir': 'bossair',
+            'WMCore.Agent.Database': 'agent',
+            'WMComponent.DBS3Buffer': 'dbs3buffer',
+            'T0.WMBS': 'tier0',
+            'WMQuality.TestDB': 'testdb'
+        }
+
+        # convert old module names to new format
+        if modules:
+            modules = [moduleMap.get(module, module) for module in modules]
+
+        self.setSchemaFromModules(modules)
+
+    def setSchemaFromModules(self, sqlModules):
+        """
+        Initialize database schema for one or more SQL packages.
+        It finds out which dialect is being used and then looks for the
+        appropriate SQL files in the sql directory.
+
+        :param sqlModules: List of SQL database modules to be initialized.
+            Current supported values are (default is 'wmbs'):
+            - 'wmbs'
+            - 'resourcecontrol'
+            - 'bossair'
+            - 'agent'
+            - 'dbs3buffer'
+            - 'tier0'
         """
-        modules = modules or []
         myThread = threading.currentThread()
+        if not hasattr(myThread, 'dbi'):
+            raise WMInitException("Database connection not initialized. Call setDatabaseConnection first.")
+
+        # Get the database dialect
+        dialect = myThread.dialect.lower()
+        if dialect not in ['mariadb', 'oracle']:
+            raise WMInitException(f"Unsupported database dialect: {dialect}")
+
+        # Get the base directory (WMCore root)
+        if os.environ.get('WMCORE_ROOT'):
+            baseDir = os.environ['WMCORE_ROOT']
+            logging.info("SQL base directory based on WMCORE_ROOT: %s", baseDir)
+        else:
+            baseDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+            logging.info("SQL base directory based on WMInit-relative path: %s", baseDir)
+
+        # Define the SQL files needed for each module and their dependencies
+        # The order matters - modules that depend on others should come later
+        moduleSQLFiles = {
+            'wmbs': {
+                'files': ['create_wmbs_tables.sql', 'create_wmbs_indexes.sql', 'initial_wmbs_data.sql'],
+                'dependencies': []
+            },
+            'resourcecontrol': {
+                'files': ['create_resourcecontrol.sql'],
+                'dependencies': ['wmbs']
+            },
+            'bossair': {
+                'files': ['create_bossair.sql'],
+                'dependencies': ['wmbs']
+            },
+            'agent': {
+                'files': ['create_agent.sql'],
+                'dependencies': []
+            },
+            'dbs3buffer': {
+                'files': ['create_dbs3buffer.sql'],
+                'dependencies': ['wmbs']
+            },
+            'tier0': {
+                'files': ['create_tier0_tables.sql', 'create_tier0_indexes.sql', 'create_tier0_functions.sql', 'initial_tier0_data.sql'],
+                'dependencies': ['wmbs', 'dbs3buffer']
+            },
+            'testdb': {
+                'files': ['create_testdb.sql'],
+                'dependencies': []
+            },
+        }
+
+        # Validate all requested modules exist
+        for module in sqlModules:
+            if module not in moduleSQLFiles:
+                raise WMInitException(f"Unknown module: {module}")
+
+        # Sort modules based on dependencies
+        sorted_modules = []
+        remaining_modules = set(sqlModules)
+
+        while remaining_modules:
+            # Find modules with no remaining dependencies
+            ready_modules = [
+                mod for mod in remaining_modules
+                if all(dep in sorted_modules for dep in moduleSQLFiles[mod]['dependencies'])
+            ]
+
+            if not ready_modules:
+                # Circular dependency detected
+                raise WMInitException("Circular dependency detected in module dependencies")
+
+            sorted_modules.extend(ready_modules)
+            remaining_modules -= set(ready_modules)
+
+        # Execute SQL files in dependency order
+        for module in sorted_modules:
+            logging.info("Executing SQL files for: %s", module)
+            for sql_file in moduleSQLFiles[module]['files']:
+                dialect_sql_file = wmcoredb.get_sql_file(module_name=module, file_name=sql_file, backend=dialect)
+
+                # now execute each SQL statement
+                for stmt in self._getSQLStatements(dialect_sql_file, dialect):
+                    try:
+                        myThread.dbi.processData(stmt)
+                    except Exception as ex:
+                        msg = f"Error executing SQL file {dialect_sql_file}. "
+                        msg += f"Statement: {stmt}"
+                        msg += str(ex)
+                        raise WMInitException(msg)
 
-        parameters = None
-        flag = False
-        # Set up for typical DBCreator format: logger, dbi, params
-        if params != None:
-            parameters = [None, None, params]
-            flag = True
-
-        myThread.transaction.begin()
-        for factoryName in modules:
-            # need to create these tables for testing.
-            # notice the default structure: <dialect>/Create
-            factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
-
-            create = factory.loadObject("Create", args=parameters, listFlag=flag)
-            createworked = create.execute(conn=myThread.transaction.conn,
-                                          transaction=myThread.transaction)
-            if createworked:
-                logging.debug("Tables for " + factoryName + " created")
-            else:
-                logging.debug("Tables " + factoryName + " could not be created.")
-        myThread.transaction.commit()
+        return
+
+    def _getSQLStatements(self, sqlFile, dialect):
+        """
+        Return the SQL statements from the file.
+        For MariaDB, it accepts the whole SQL file content in a single statement.
+        For Oracle, it splits the SQL file content into statements using the slash (/) terminator
+        when it appears as the first character in a line.
+        """
+        if not os.path.exists(sqlFile):
+            raise WMInitException(f"SQL file not found: {sqlFile}")
+
+        with open(sqlFile, 'r', encoding='utf-8') as f:
+            sql = f.read()
+
+        if dialect == 'mariadb':
+            return [sql]
+        elif dialect == 'oracle':
+            statements = []
+            current_statement = []
+
+            for line in sql.split('\n'):
+                line = line.strip()
+                # Skip empty lines and comments
+                if not line or line.startswith('--'):
+                    continue
+
+                # If we find a slash terminator at the start of a line
+                if line == '/':
+                    # Join all lines collected so far into a statement
+                    stmt = '\n'.join(current_statement)
+                    if stmt.strip():
+                        statements.append(stmt)
+                    current_statement = []
+                else:
+                    current_statement.append(line)
+
+            # Add any remaining statement
+            if current_statement:
+                stmt = '\n'.join(current_statement)
+                if stmt.strip():
+                    statements.append(stmt)
+
+            return statements
+        else:
+            raise WMInitException(f"Unsupported database dialect: {dialect}")
 
     def clearDatabase(self, modules=None):
         """

WMCore/WMSpec/WMWorkloadTools.py

@@ -61,7 +61,7 @@ def makeLumiList(lumiDict):
             lumiDict = json.loads(lumiDict)
         ll = LumiList(compactList=lumiDict)
         return ll.getCompactList()
-    except:
+    except Exception:
         raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
 
 
@@ -91,12 +91,24 @@ def _validateArgument(argument, value, argumentDefinition):
     elif value is None:
         return value
 
+    # is it a built-in type (int, str, bool, etc.) or a custom function
+    expected_type = argumentDefinition["type"]
     try:
-        value = argumentDefinition["type"](value)
-    except Exception:
+        if expected_type == str and not isinstance(value, str):
+            raise TypeError(f"Argument '{argument}' with value {value} is not a string")
+        else:
+            # for other data types (or custom functions), just try to cast it to the expected type
+            value = expected_type(value)
+    except (ValueError, TypeError) as e:
         msg = "Argument '%s' with value %r, has an incorrect data type: " % (argument, value)
-        msg += "%s. It must be %s" % (type(value), argumentDefinition["type"])
-        raise WMSpecFactoryException(msg)
+        msg += "%s. It must be convertible by %s" % (type(value), expected_type.__name__)
+        if str(e):
+            msg += f" (Error: {e})"
+        raise WMSpecFactoryException(msg) from None
+    except Exception as e:
+        msg = f"Generic exception raised during argument validation for argument: {argument} "
+        msg += "with value: %r. Error details: %s" % (value, str(e))
+        raise WMSpecFactoryException(msg) from None
 
     _validateArgFunction(argument, value, argumentDefinition["validate"])
     return value

WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.4.0.2'
+__version__ = '2.4.2'
 __all__ = []

wmglobalqueue-2.4.2.dist-info/METADATA

@@ -0,0 +1,26 @@
+Metadata-Version: 2.4
+Name: wmglobalqueue
+Version: 2.4.2
+Home-page: https://github.com/dmwm/WMCore
+Maintainer: CMS DMWM Group
+Maintainer-email: hn-cms-wmdevelopment@cern.ch
+License: Apache License, Version 2.0
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: CherryPy~=18.10.0
+Requires-Dist: CMSMonitoring~=0.6.13
+Requires-Dist: dbs3-client==4.0.19
+Requires-Dist: future~=1.0.0
+Requires-Dist: httplib2~=0.22.0
+Requires-Dist: psutil~=7.0.0
+Requires-Dist: pycurl~=7.45.6
+Requires-Dist: retry~=0.9.2
+Requires-Dist: rucio-clients~=36.5.0
+Requires-Dist: Sphinx~=5.3.0
+Requires-Dist: PyJWT~=2.9.0
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: maintainer
+Dynamic: maintainer-email
+Dynamic: requires-dist