reqmgr2ms-output 2.3.2rc9__tar.gz → 2.3.4__tar.gz

{reqmgr2ms-output-2.3.2rc9/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc9
+Version: 2.3.4
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/adhoc-scripts/checkStuckLQE.py

@@ -72,10 +72,8 @@ def commonDataLocation(element):
     return commonLoc
 
 def main():
-    if 'WMAGENT_CONFIG' not in os.environ:
-        os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
-    config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
-
+    config = loadConfigurationFile(os.environ.get('WMA_CONFIG_FILE', '/data/srv/wmagent/current/config/config.py')) 
+    
     if len(sys.argv) != 2:
         print("You must provide a request name")
         sys.exit(1)

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/adhoc-scripts/drainAgent.py

@@ -366,8 +366,7 @@ def main():
     twiki = ('<pre>', '</pre>') if args.twiki else ('', '')
     print(twiki[0])
 
-    os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
-    config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
+    config = loadConfigurationFile(os.environ.get('WMA_CONFIG_FILE', '/data/srv/wmagent/current/config/config.py')) 
 
     gmtTimeNow = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
     print("Drain check time: %s\n" % gmtTimeNow)

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/adhoc-scripts/getWQStatusByWorkflow.py

@@ -37,9 +37,7 @@ def createElementsSummary(reqName, elements, queueUrl):
 
 
 def main():
-    if 'WMAGENT_CONFIG' not in os.environ:
-        os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
-    config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
+    config = loadConfigurationFile(os.environ.get('WMA_CONFIG_FILE', '/data/srv/wmagent/current/config/config.py'))
 
     if len(sys.argv) != 2:
         print("You must provide a request name")

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/dbsbuffer-file-fix.py

@@ -9,7 +9,7 @@ from WMCore.Database.DBFormatter import DBFormatter
 from WMCore.WMInit import connectToDB
 
 def fixDBSmissingFileAssoc():
-    os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
+    os.environ['WMA_CONFIG_FILE'] = '/data/srv/wmagent/current/config/config.py'
     connectToDB()
     myThread = threading.currentThread()
     formatter = DBFormatter(logging, myThread.dbi)

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/kill-workflow-in-agent

@@ -16,12 +16,8 @@ def killWorkflowAgent(WorkflowName):
     """
     Cancel work for a given workflow - delete in wmbs, delete from workqueue db, set canceled in inbox
     """    
-    # get configuration file path
-    if "WMAGENT_CONFIG" not in os.environ:
-        os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
-    
-    # load config
-    wmConfig = loadConfigurationFile(os.environ['WMAGENT_CONFIG'])
+    # get configuration file path and load it
+    wmConfig = loadConfigurationFile(os.environ.get('WMA_CONFIG_FILE', '/data/srv/wmagent/current/config/config.py'))
     wqManager = wmConfig.section_('WorkQueueManager')
     
     couchUrl = wqManager.couchurl

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/kill-workflow-in-global

@@ -25,12 +25,8 @@ def main():
         sys.exit(0)
     wflowName = args[0]
 
-    # get configuration file path
-    if "WMAGENT_CONFIG" not in os.environ:
-        os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
-
-    # load config
-    wmConfig = loadConfigurationFile(os.environ['WMAGENT_CONFIG'])
+    # get configuration file path and load it
+    wmConfig = loadConfigurationFile(os.environ.get('WMA_CONFIG_FILE', '/data/srv/wmagent/current/config/config.py'))
 
     gqService = WorkQueue(wmConfig.WorkloadSummary.couchurl,
                           wmConfig.WorkQueueManager.dbname)

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/bin/wmagent-couchapp-init

@@ -18,6 +18,19 @@ from WMCore.Configuration import loadConfigurationFile
 from WMCore.Lexicon import splitCouchServiceURL
 from WMCore.WMBase import getWMBASE
 
+parser = argparse.ArgumentParser()
+parser.add_argument("--skip-cron", dest = "cron",
+                    default = True, action = "store_false",
+                    help = "Do not install maintenance cron jobs")
+parser.add_argument("--only-cron", dest = "pushApps",
+                    default = True, action = "store_false",
+                    help = "Install only maintenance cron jobs")
+options = parser.parse_args()
+
+if not options.cron and not options.pushApps:
+    print("Mutually excluding options: --skip-cron && --only-cron")
+    print("Chose only one of them.")
+    sys.exit(1)
 
 def couchAppRoot(couchapp):
     """Return parent path containing couchapp"""
@@ -31,7 +44,32 @@ def couchAppRoot(couchapp):
         return os.path.join(wmcoreroot, 'data', 'couchapps')
     raise OSError('Cannot find couchapp: %s' % couchapp)
 
-
+# Defining a minimal decorator with parameter
+def pushApps(pushAppsFlag):
+    """
+    Decorator with a single Bool parameter
+    :param pushApps: Bool parameter to check if the callFunc is to be executed or not
+    :return:         The decorated function with call arguments if pushApps=True
+    """
+    def pushAppsDecorator(callFunc):
+        """
+        An Auxiliary simple decorator
+        """
+        def installCouchAppWrapper(*args, **kwargs):
+            """
+            The callFunc wrapper. To decide whether to return the call to the
+            wrapped function or not based on the decorator's pushAppsFlag
+            :*args:    All parameters passed to the wrapped function call
+            :**kwargs: All keyword arguments passed to the wrapped function call
+            """
+            if pushAppsFlag:
+                return callFunc(*args, **kwargs)
+            else:
+                return
+        return installCouchAppWrapper
+    return pushAppsDecorator
+
+@pushApps(options.pushApps)
 def installCouchApp(couchUrl, couchDBName, couchAppName, basePath=None):
     """
     _installCouchApp_
@@ -273,11 +311,6 @@ files_needed_from_yui = [
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--skip-cron", dest = "cron",
-                        default = True, action = "store_false",
-                        help = "Do not install maintenance cron jobs")
-    options = parser.parse_args()
 
     if "WMAGENT_CONFIG" not in os.environ:
         print("The WMAGENT_CONFIG environment variable needs to be set before")
@@ -285,7 +318,7 @@ if __name__ == "__main__":
         sys.exit(1)
 
     wmagentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
-            
+
     if hasattr(wmagentConfig, "JobStateMachine") and hasattr(wmagentConfig.JobStateMachine, "couchDBName"):
         fwjrDumpName = urllib.parse.quote_plus("%s/fwjrs" % wmagentConfig.JobStateMachine.couchDBName)
         jobDumpName = urllib.parse.quote_plus("%s/jobs" % wmagentConfig.JobStateMachine.couchDBName)
@@ -355,7 +388,7 @@ if __name__ == "__main__":
            urlsplit(wmagentConfig.WorkloadSummary.couchurl).scheme == 'http':
             installCouchApp(wmagentConfig.ACDC.couchurl, wmagentConfig.ACDC.database, "ACDC")
             installCouchApp(wmagentConfig.ACDC.couchurl, wmagentConfig.ACDC.database, "GroupUser")
-    
+
     if hasattr(wmagentConfig, "Tier0Feeder"):
         installCouchApp(wmagentConfig.JobStateMachine.couchurl, wmagentConfig.Tier0Feeder.requestDBName, "T0Request")
         centralWMStatsURL = wmagentConfig.General.centralWMStatsURL

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/setup_dependencies.py

@@ -186,6 +186,7 @@ dependencies = {
                      'WMCore.BossAir+', 'WMCore.Credential',
                      'WMCore.JobSplitting+', 'WMCore.ProcessPool',
                      'WMCore.Services+', 'WMCore.WMSpec+',
+                     'WMCore.ReqMgr.DataStructs+', 'WMCore.ReqMgr.Tools+',
                      'WMCore.WMBS+', 'WMCore.ResourceControl+'],
         'systems': ['wmc-web', 'wmc-database', 'global-workqueue', 'wmc-runtime'],
         'statics': ['bin+',
@@ -196,6 +197,10 @@ dependencies = {
                     'src/couchapps/JobDump+',
                     'src/couchapps/WMStatsAgent+',
                     'src/couchapps/WorkQueue+',
+                    'src/couchapps/T0Request+',
+                    'src/couchapps/WMStats+',
+                    'src/couchapps/LogDB+',
+                    'src/couchapps/UserMonitoring+',
                     'src/javascript/WMCore/WebTools/Agent',
                     'src/javascript/WMCore/WebTools/WMBS',
                     'src/javascript/external/graphael',

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/src/python/Utils/IteratorTools.py

@@ -18,6 +18,18 @@ def grouper(iterable, n):
     return iter(lambda: list(islice(iterable, n)), [])
 
 
+def getChunk(arr, step):
+    """
+    Return chunk of entries from given array and step, it is similar in behavior to grouper
+    function but instead of returning new list it provides a generator iterable object.
+    :param arr: input array of data
+    :param step: step to iterate
+    :return: generator, set of slices with number of entries equal to step of iteration
+    """
+    for i in range(0, len(arr), step):
+        yield arr[i:i + step]
+
+
 def flattenList(doubleList):
     """
     Make flat a list of lists.

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/src/python/WMCore/Services/Rucio/Rucio.py

@@ -18,6 +18,7 @@ from rucio.common.exception import (AccountNotFound, DataIdentifierNotFound, Acc
                                     DataIdentifierAlreadyExists, DuplicateContent, InvalidRSEExpression,
                                     UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound)
 from Utils.MemoryCache import MemoryCache
+from Utils.IteratorTools import grouper
 from WMCore.Services.Rucio.RucioUtils import (validateMetaData, weightedChoice,
                                               isTapeRSE, dropTapeRSEs)
 from WMCore.WMException import WMException
@@ -346,7 +347,7 @@ class Rucio(object):
             response = self.attachDIDs(kwargs.get('rse'), container, name, scope)
         return response
 
-    def attachDIDs(self, rse, superDID, dids, scope='cms'):
+    def attachDIDs(self, rse, superDID, dids, scope='cms', chunkSize=1000):
         """
         _attachDIDs_
 
@@ -356,28 +357,38 @@ class Rucio(object):
              then it's a container name; if attaching files, then it's a block name)
         :param dids: either a string or a list of data identifiers (can be block or files)
         :param scope: string with the scope name
+        :param chunkSize: maximum number of dids to be attached in any single Rucio server call
         :return: a boolean to represent whether it succeeded or not
         """
         if not isinstance(dids, list):
             dids = [dids]
         # NOTE: the attaching dids do not create new container within a scope
         # and it is safe to use cms scope for it
-        dids = [{'scope': 'cms', 'name': did} for did in dids]
+        alldids = [{'scope': 'cms', 'name': did} for did in sorted(dids)]
+
+        # report if we use chunk size in rucio attach_dids API call
+        if len(alldids) > chunkSize:
+            self.logger.info("Attaching a total of %d DIDs in chunk size of: %d", len(alldids), chunkSize)
+
+        for dids in grouper(alldids, chunkSize):
+            response = False
+            try:
+                response = self.cli.attach_dids(scope, superDID, dids=dids, rse=rse)
+            except DuplicateContent:
+                self.logger.warning("Dids: %s already attached to: %s", dids, superDID)
+                response = True
+            except FileAlreadyExists:
+                self.logger.warning("Failed to attach files already existent on block: %s", superDID)
+                response = True
+            except DataIdentifierNotFound:
+                self.logger.error("Failed to attach dids: %s. Parent DID %s does not exist.", dids, superDID)
+            except Exception as ex:
+                self.logger.error("Exception attaching %s dids to: %s. Error: %s. First 10 dids: %s",
+                                  len(dids), superDID, str(ex), dids[:10])
+            if not response:
+                # if we had failure with specific chunk of dids we'll return immediately
+                return response
 
-        response = False
-        try:
-            response = self.cli.attach_dids(scope, superDID, dids=dids, rse=rse)
-        except DuplicateContent:
-            self.logger.warning("Dids: %s already attached to: %s", dids, superDID)
-            response = True
-        except FileAlreadyExists:
-            self.logger.warning("Failed to attach files already existent on block: %s", superDID)
-            response = True
-        except DataIdentifierNotFound:
-            self.logger.error("Failed to attach dids: %s. Parent DID %s does not exist.", dids, superDID)
-        except Exception as ex:
-            self.logger.error("Exception attaching dids: %s to: %s. Error: %s",
-                              dids, superDID, str(ex))
         return response
 
     def createReplicas(self, rse, files, block, scope='cms', ignoreAvailability=True):

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/src/python/WMCore/Services/TagCollector/XMLUtils.py

@@ -71,7 +71,7 @@ def get_children(elem, event, row, key):
     row for given key. The change of notations can be applied during
     parsing step by using provided notations dictionary.
     """
-    for child in elem.getchildren():
+    for child in list(elem):
         child_key = child.tag
         child_data = child.attrib
         if not child_data:
@@ -79,7 +79,7 @@ def get_children(elem, event, row, key):
         else:
             child_dict = child_data
 
-        if child.getchildren():  # we got grand-children
+        if list(child):  # we got grand-children
             if child_dict:
                 row[key][child_key] = child_dict
             else:

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.2rc9'
+__version__ = '2.3.4'
 __all__ = []

{reqmgr2ms-output-2.3.2rc9 → reqmgr2ms-output-2.3.4/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc9
+Version: 2.3.4
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group