reqmgr2ms-monitor 2.3.6rc3__tar.gz → 2.3.6rc5__tar.gz

{reqmgr2ms-monitor-2.3.6rc3/src/python/reqmgr2ms_monitor.egg-info → reqmgr2ms-monitor-2.3.6rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.6rc3
+Version: 2.3.6rc5
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-monitor-2.3.6rc5/bin/wmagent-component-standalone

@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+"""
+wmagent-component-standalone
+
+Utility script for running a WMAgent component standalone.
+
+Example usage:
+
+interactive mode:
+ipython -i $WMA_DEPLOY_DIR/bin/wmagent-component-standalone -- -c $WMA_CONFIG_FILE  -p JobAccountant -e $WMA_ENV_FILE
+
+daemon mode:
+wmagent-component-standalone -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE
+"""
+
+import os
+import sys
+import logging
+import importlib
+import runpy
+import pkgutil
+import inspect
+import subprocess
+from pprint import pprint,pformat
+from distutils.sysconfig import get_python_lib
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from WMCore.Configuration import loadConfigurationFile
+from WMCore.ResourceControl.ResourceControl import ResourceControl
+from WMCore.Services.CRIC.CRIC import CRIC
+from WMCore.WMInit import connectToDB
+from Utils.FileTools import loadEnvFile
+
+def createOptionParser():
+    """
+    _createOptionParser_
+
+    Creates an option parser to setup the component run parameters
+    """
+    exampleStr = """
+    Examples:
+
+    * interactive mode:
+    ipython -i $WMA_DEPLOY_DIR/bin/wmagent-component-standalone -- -c $WMA_CONFIG_FILE  -p JobAccountant -e $WMA_ENV_FILE
+
+    * daemon mode:
+    wmagent-component-standalone -c $WMA_CONFIG_FILE -p JobAccountant -e $WMA_ENV_FILE
+    """
+    helpStr = """
+    Utility script for running a WMAgent component standalone. It supports two modes:
+    * interactive - it loads all possible sub modules defined within the component's source area
+                    and tries to create a proper set of object instances for each of them
+                    in the global scope of the script, such that they could be used interactively
+    * daemon      - it mimics the normal behavior of a regular component run in the background,
+                    while at the same time giving the ability to overwrite some of the component's configuration parameters
+                    e.g. loglevel or provide an alternative WMA_CONFIG_FILE and/or WMA_ENV_FILE.
+    """
+    optParser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
+                               description=helpStr,
+                               epilog=exampleStr)
+    optParser.add_argument("-c", "--config", dest="wmaConfigPath", help="WMAgent configuration file.",
+                           default=os.environ.get("WMA_CONFIG_FILE", None))
+    optParser.add_argument("-e", "--envfile", dest="wmaEnvFilePath", help="WMAgent environment file.",
+                           default=os.environ.get("WMA_ENV_FILE", None))
+    optParser.add_argument("-d", "--daemon", dest="daemon",
+                           default=False, action="store_true",
+                           help="The component would be run as a daemon, similar to how it is run through the standard agent manage scripts.")
+    optParser.add_argument("-p", "--component", dest="wmaComponentName",
+                           default=None,
+                           help="The component/package to be loaded.")
+    optParser.add_argument("-l", "--loglevel", dest="logLevel",
+                           default='INFO',
+                           help="The loglevel at which the component should be run,")
+
+    return optParser
+
+
+def main():
+    """
+    _main_
+    The main function to be used for starting the chosen component as a daemon
+    """
+    # NOTE: There are two methods for running the wmcore daemon
+    #       * The first method forks and assosiates a new shell to the running process
+    #         os.system(f"wmcoreD --start --component {wmaComponentName}")
+    #       * The second method executes the daemon in the very same interpretter:
+    #         runpy("wmcoreD")
+
+    wmaDaemonPath = os.environ.get("WMA_DEPLOY_DIR", None) + "/bin/wmcoreD"
+
+    # Run the daemon:
+    logger.info("Starting the component daemon")
+    sys.argv = ["--", "--restart", "--component", wmaComponentName]
+    runpy.run_path(wmaDaemonPath)
+
+    return
+
+if __name__ == '__main__':
+    optParser = createOptionParser()
+    (options, args) = optParser.parse_known_args()
+    FORMAT = "%(asctime)s:%(levelname)s:%(module)s:%(funcName)s(): %(message)s"
+    LOGLEVEL = logging.INFO
+    # add logfile handler as well:
+    logFilePath = os.path.join(os.getenv('WMA_INSTALL_DIR', '/tmp'), f'{options.wmaComponentName}/ComponentLogStandalone')
+    logFileHandler = logging.FileHandler(logFilePath)
+    logStdoutHandler = logging.StreamHandler(sys.stdout)
+
+    logging.basicConfig(handlers=[logStdoutHandler, logFileHandler], format=FORMAT, level=LOGLEVEL)
+    logger = logging.getLogger(__name__)
+    # logger.setLevel(LOGLEVEL)
+
+    # sourcing $WMA_ENV_FILE explicitely
+    if not options.wmaEnvFilePath or not os.path.exists(options.wmaEnvFilePath):
+        msg = "Missing WMAgent environment file! One may expect component misbehaviour!"
+        logger.warning(msg)
+    else:
+        msg = "Trying to source explicitely the WMAgent environment file: %s"
+        logger.info(msg, options.wmaEnvFilePath)
+        try:
+            loadEnvFile(options.wmaEnvFilePath)
+        except Exception as ex:
+            logger.error("Failed to load wmaEnvFile: %s", options.wmaEnvFilePath)
+            raise
+
+    # checking the existence of wmaConfig file
+    if not options.wmaConfigPath or not os.path.exists(options.wmaConfigPath):
+        msg = "Missing WMAgent config file! One may expect component failure"
+        logger.warning(msg)
+    else:
+        # resetting the configuration in the env (if the default is overwritten through args)
+        os.environ['WMAGENT_CONFIG'] = options.wmaConfigPath
+        os.environ['WMA_CONFIG_FILE'] = options.wmaConfigPath
+
+    wmaConfig = loadConfigurationFile(options.wmaConfigPath)
+    logger.info(f"wmaEnvFilePath: {options.wmaEnvFilePath}")
+    logger.info(f"wmaConfigPath: {options.wmaConfigPath}")
+    logger.info(f"wmaComponent: {options.wmaComponentName}")
+    logger.info(f"logLevel: {options.logLevel}")
+    logger.info(f"daemon: {options.daemon}")
+
+    connectToDB()
+
+    logger.info(f"Creating default component objects.")
+    resourceControl = ResourceControl(config=wmaConfig)
+    wmaComponentName = options.wmaComponentName
+    wmaComponentModule = f"WMComponent.{wmaComponentName}"
+    # wmaComponent = importlib.import_module(wmaComponentModule + "." + wmaComponentName)
+
+    logger.info("Importing all possible modules found for this component")
+
+    # First find all possible locations for the component source
+    # NOTE: We always consider PYTHONPATH first
+    pythonLibPaths = os.getenv('PYTHONPATH', '').split(':')
+    pythonLibPaths.append(get_python_lib())
+    # Normalize paths and remove empty ones
+    pythonLibPaths = [x for x in pythonLibPaths if x]
+    for index, libPath in enumerate(pythonLibPaths):
+        pythonLibPaths[index] = os.path.normpath(libPath)
+
+    wmaComponentPaths = []
+    for comPath in pythonLibPaths:
+        comPath = f"{comPath}/WMComponent/{wmaComponentName}"
+        comPath = os.path.normpath(comPath)
+        if os.path.exists(comPath):
+            wmaComponentPaths.append(comPath)
+
+    modules = {}
+    classDefs = {}
+    # Then try to load all possible modules and submodules under the component's source path
+    # for pkgSource, pkgName, _ in pkgutil.iter_modules(wmaComponentPaths):
+    for pkgSource, pkgName, _ in pkgutil.walk_packages(wmaComponentPaths):
+        fullPkgName =  f"{wmaComponentModule}.{pkgName}"
+        if pkgName == "DefaultConfig":
+            continue
+        logger.info("Loading package: %s", fullPkgName)
+        modSpec = pkgSource.find_spec(fullPkgName)
+        module = importlib.util.module_from_spec(modSpec)
+        modSpec.loader.exec_module(module)
+        modules[pkgName] = module
+
+        # Emulating `from module import *`"
+        logger.info(f"Populating the namespace with all definitions from {pkgName}")
+        if "__all__" in module.__dict__:
+            names = module.__dict__["__all__"]
+        else:
+            names = [x for x in module.__dict__ if not x.startswith("_")]
+        globals().update({k: getattr(module, k) for k in names})
+
+        # Creating instances only for class definitions local to pkgName
+
+        # Note: The method bellow for separating local definitions from imported ones
+        #       won't work for decorated  classes, since the module name of the
+        #       decorated class defintion is considered to be the fully qulified
+        #       module name/path of the decorator rather than the package where the
+        #       the class definition exists:
+        #       e.g.:
+        #       In [10]: modules['DataCollectAPI'].__name__
+        #       Out[10]: 'WMComponent.AnalyticsDataCollector.DataCollectAPI'
+        #
+        #       In [11]: classDefs['DataCollectAPI']['LocalCouchDBData']
+        #       Out[11]: WMComponent.AnalyticsDataCollector.DataCollectorEmulatorSwitch.emulatorHook.<locals>.EmulatorWrapper
+        #
+        #       In [12]: classDefs['DataCollectAPI']['LocalCouchDBData'].__name__
+        #       Out[12]: 'EmulatorWrapper'
+        #
+        #       In [13]: classDefs['DataCollectAPI']['LocalCouchDBData'].__module__
+        #       Out[13]: 'WMComponent.AnalyticsDataCollector.DataCollectorEmulatorSwitch'
+        #
+        #       In [14]: modules['DataCollectAPI'].__name__
+        #       Out[14]: 'WMComponent.AnalyticsDataCollector.DataCollectAPI'
+
+        logger.info("Trying to create an instance of all class definitions inside: %s", pkgName)
+        classDefs[pkgName] = {}
+        for objName, obj in inspect.getmembers(module):
+            if inspect.isclass(obj) and obj.__module__ == modules[pkgName].__name__:
+                classDefs[pkgName][objName]=obj
+
+        for className in classDefs[pkgName]:
+            logger.info(f"{fullPkgName}:{className}")
+            try:
+                exec(f"{className.lower()} = {className}()")
+            except TypeError:
+                try:
+                    exec(f"{className.lower()} = {className}(wmaConfig)")
+                except TypeError:
+                    try:
+                        exec(f"{className.lower()} = {className}(wmaConfig, logger=logger)")
+                    except TypeError:
+                        logger.warning(f"We did our best to create an instance of: {pkgName}. Giving up now!")
+
+    def modReload():
+        for module in modules:
+            modName = modules[module].__name__
+            modInst = sys.modules.get(modName)
+            if modInst:
+                logger.info(f"Reloading module:{modName}")
+                importlib.reload(modInst)
+            else:
+                logger.warning(f"Cannot find module: {modName} in sys.modules")
+
+    if options.daemon:
+        main()

{reqmgr2ms-monitor-2.3.6rc3 → reqmgr2ms-monitor-2.3.6rc5}/src/python/Utils/FileTools.py

@@ -11,6 +11,7 @@ import stat
 import subprocess
 import time
 import zlib
+import logging
 
 from Utils.Utilities import decodeBytesToUnicode
 
@@ -152,3 +153,30 @@ def getFullPath(name, envPath="PATH"):
         if os.path.exists(fullPath):
             return fullPath
     return None
+
+
+def loadEnvFile(wmaEnvFilePath, logger=None):
+    """
+    _loadEnvFile_
+    A simple function to load an additional bash env file into the current script
+    runtime environment
+    :param wmaEnvFilePath: The path to the environment file to be loaded
+    :return:               True if the script has loaded successfully, False otherwise.
+    """
+    if not logger:
+        logger = logging.getLogger()
+    subProc = subprocess.run(['bash', '-c', f'source {wmaEnvFilePath} && python -c "import os; print(repr(os.environ.copy()))" '],
+                                   capture_output=True, check=False)
+    if subProc.returncode == 0:
+        newEnv = eval(subProc.stdout)
+        os.environ.update(newEnv)
+        if subProc.stderr:
+            logger.warning("Environment file: %s loaded with errors:", wmaEnvFilePath)
+            logger.warning(subProc.stderr.decode())
+        else:
+            logger.info("Environment file: %s loaded successfully", wmaEnvFilePath)
+        return True
+    else:
+        logger.error("Failed to load environment file: %s", wmaEnvFilePath)
+        logger.error(subProc.stderr.decode())
+        return False

{reqmgr2ms-monitor-2.3.6rc3 → reqmgr2ms-monitor-2.3.6rc5}/src/python/WMCore/Database/ExecuteDAO.py

@@ -24,16 +24,17 @@ Usage:
 import sys
 import os
 import re
-import ast
+
 import threading
 import logging
 import argparse
+import pickle
 from pprint import pformat
 
 from WMCore.DAOFactory import DAOFactory
 from WMCore.WMInit import WMInit
 from WMCore.Agent.Configuration import Configuration, loadConfigurationFile
-
+from Utils.FileTools import loadEnvFile
 
 def parseArgs():
     """
@@ -44,11 +45,18 @@ def parseArgs():
         formatter_class=argparse.RawTextHelpFormatter,
         description=__doc__)
 
-    parser.add_argument('-c', '--config', required=True,
+    parser.add_argument('-c', '--config', required=False,
+                        default=os.environ.get("WMA_CONFIG_FILE", None),
                         help="""\
-                        The path to WMAGENT_CONFIG to be used, e.g.
-                        for production: /data/srv/wmagent/current/config/wmagent/config.py
-                        for tier0:      /data/tier0/srv/wmagent/current/config/tier0/config.py""")
+                        The WMAgent config file to be used for the this execution. Default is taken from
+                        the current's shell environment variable $WMA_CONFIG_FILE
+                        """)
+    parser.add_argument('-e', '--envFile', required=False,
+                        default=os.environ.get("WMA_ENV_FILE", None),
+                        help="""
+                        The WMAgent environment file to be used for the this execution. Default is taken from
+                        the current's shell environment variable $WMA_ENV_FILE
+                        """)
     parser.add_argument('-p', '--package', required=True,
                         help="""\
                         The package from which the DAO factory to be created for this execution, e.g. WMCore.WMBS or WMComponent.DBS3Buffer""")
@@ -70,25 +78,32 @@ def parseArgs():
     parser.add_argument('sqlArgs', nargs=argparse.REMAINDER, default=(),
                         help="""\
                         -- Positional parameters to be forwarded to the DAO execute method and used as SQL arguments in the query.""")
+    parser.add_argument('-f', '--pklFile', default=None,
+                        help="""\
+                        An extra *.pkl file containing any additional python objects needed for the given dao
+                        e.g. WMCore.WMBS.Files.AddRunLumi.
+                        The object is always loaded under the name `pklFile`. One can access the contents of the so loaded pkl file
+                        during the dao execution trough the -s arguent e.g.:
+                        ExecuteDAO.py -p  WMCore.WMBS -m Files.AddRunLumi -c $WMA_CONFIG_FILE -f runLumiBinds_2035-4016.pkl -s "{'file': pklFile['data']}
+                        """)
+    currArgs = parser.parse_args()
 
-    args = parser.parse_args()
-
-    return args
+    return currArgs
 
 
 def loggerSetup(logLevel=logging.INFO):
     """
     Return a logger which writes everything to stdout.
     """
-    logger = logging.getLogger()
+    currLogger = logging.getLogger()
     outHandler = logging.StreamHandler(sys.stdout)
     outHandler.setFormatter(logging.Formatter("%(asctime)s:%(levelname)s:%(module)s: %(message)s"))
     outHandler.setLevel(logLevel)
-    if logger.handlers:
-        logger.handlers.clear()
-    logger.addHandler(outHandler)
-    logger.setLevel(logLevel)
-    return logger
+    if currLogger.handlers:
+        currLogger.handlers.clear()
+    currLogger.addHandler(outHandler)
+    currLogger.setLevel(logLevel)
+    return currLogger
 
 
 def getBackendFromDbURL(dburl):
@@ -110,15 +125,12 @@ class ExecuteDAO():
     """
     A generic class to create the DAO Factory and execute the DAO module.
     """
-    def __init__(self, logger=None, configFile=None,
-                 connectUrl=None, socket=None,
-                 package=None, daoModule=None):
+    def __init__(self, connectUrl=None, socket=None, configFile=None,
+                 package=None, daoModule=None, logger=None):
         """
         __init__
         The ExecuteDAO constructor method.
-        :param logger: The logger instance.
         :param package: The Package from which the DAO factory to be initialised.
-        :param configFile: Path to WMAgent configuration file.
         :param connectUrl: Database connection URL (overwrites the connectUrl param from configFile if both present)
         :param socket: Database connection URL (overwrites the socket param from configFile if both present)
         :param module: The DAO module to be executed.
@@ -215,7 +227,13 @@ class ExecuteDAO():
             self.logger.info("DAO SQL arguments provided:\n%s, %s", pformat(sqlArgs), pformat(sqlKwArgs))
         else:
             results = self.dao.execute(*sqlArgs, **sqlKwArgs)
-            self.logger.info("DAO Results:\n%s", pformat(results if isinstance(results, dict) else list(results)))
+            # self.logger.info("DAO Results:\n%s", pformat(results if isinstance(results, dict) else list(results)))
+            if isinstance(results, dict):
+                self.logger.info("DAO Results:\n%s", pformat(results))
+            elif isinstance(results, bool):
+                self.logger.info("DAO Results:\n%s", results)
+            else:
+                self.logger.info("DAO Results:\n%s", list(results))
         return results
 
     def getSqlQuery(self):
@@ -244,21 +262,19 @@ def strToDict(dString, logger=None):
     :param dString: The dictionary string to be parsed. Possible formats are either a string
                     of multiple space separated named values of the form 'name=value':
                     or a srting fully defining the dictionary itself.
-    :param logger:  A logger object to be used for Error message printout
     :return:        The constructed dictionary
     """
-    logger = logger or logging.getLogger()
-    result = ast.literal_eval(dString)
+    if not logger:
+        logger = logging.getLogger()
+    # result = ast.literal_eval(dString)
+    result = eval(dString)
     if not isinstance(result, dict):
         logger.error("The Query named arguments need to be provided as a dictionary. WRONG option: %s", pformat(dString))
         raise TypeError(pformat(dString))
     return result
 
 
-def main():
-    """
-    An Utility to construct a DAO Factory and execute the DAO requested.
-    """
+if __name__ == '__main__':
     args = parseArgs()
 
     if args.debug:
@@ -266,6 +282,17 @@ def main():
     else:
         logger = loggerSetup()
 
+    # Create an instance of the *.pkl file provided with the dao call, if any.
+    if args.pklFile:
+        pklFilePath = os.path.normpath(args.pklFile)
+        if not os.path.exists(pklFilePath):
+            logger.error("Cannot find the pkl file: %s. Exit!", pklFilePath)
+            sys.exit(1)
+        with open(pklFilePath, 'rb') as fd:
+            pklFile = pickle.load(fd)
+            logger.info('PklFile: %s loaded as: `pklFile`. You can refer to its content through the -s argument.', pklFilePath)
+    # logger.info(pformat(pklFile))
+
     # Remove leading double slash if present:
     if args.sqlArgs and args.sqlArgs[0] == '--':
         args.sqlArgs = args.sqlArgs[1:]
@@ -276,17 +303,25 @@ def main():
 
     # Parse named arguments to a proper dictionary:
     if not isinstance(args.sqlKwArgs, dict):
-        args.sqlKwArgs = strToDict(args.sqlKwArgs, logger=logger)
+        args.sqlKwArgs = strToDict(args.sqlKwArgs)
 
-    # Try to set the wmagent configuration in the environment
-    if 'WMAGENT_CONFIG' not in os.environ:
+    # Trying to load WMA_ENV_FILE
+    if not args.envFile or not os.path.exists(args.envFile):
+        logger.warning("Missing WMAgent environment file! One may expect DAO misbehavior!")
+    else:
+        logger.info("Trying to source explicitely the WMAgent environment file: %s", args.envFile)
+        try:
+            loadEnvFile(args.envFile)
+        except Exception as ex:
+            logger.error("Failed to load wmaEnvFile: %s", args.envFile)
+            raise
+
+    if not args.config or not os.path.exists(args.config):
+        logger.warning("Missing WMAgent config file! One may expect DAO failure")
+    else:
+        # resetting the configuration file in the env (if the default is overwritten through args)
         os.environ['WMAGENT_CONFIG'] = args.config
-    configFile = os.environ['WMAGENT_CONFIG']
+        os.environ['WMA_CONFIG_FILE'] = args.config
 
-    daoObject = ExecuteDAO(logger=logger, configFile=configFile,
-                           package=args.package, daoModule=args.module)
+    daoObject = ExecuteDAO(package=args.package, daoModule=args.module, configFile=args.config)
     daoObject(*args.sqlArgs, dryRun=args.dryRun, daoHelp=True, **args.sqlKwArgs)
-
-
-if __name__ == '__main__':
-    main()

{reqmgr2ms-monitor-2.3.6rc3 → reqmgr2ms-monitor-2.3.6rc5}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.6rc3'
+__version__ = '2.3.6rc5'
 __all__ = []

{reqmgr2ms-monitor-2.3.6rc3 → reqmgr2ms-monitor-2.3.6rc5/src/python/reqmgr2ms_monitor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.6rc3
+Version: 2.3.6rc5
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.6rc3 → reqmgr2ms-monitor-2.3.6rc5}/src/python/reqmgr2ms_monitor.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ bin/purgeDeletedCouchDoc.py
 bin/reqmgr-put-default-config
 bin/reqmgr-sw-update
 bin/vaildateCMSSWMergeVersion
+bin/wmagent-component-standalone
 bin/wmagent-couchapp-init
 bin/wmagent-delete-couchdb-workflow
 bin/wmagent-mod-config