reqmgr2ms-output 2.3.4rc7__tar.gz → 2.3.4rc9__tar.gz

{reqmgr2ms-output-2.3.4rc7/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.4rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.4rc7
+Version: 2.3.4rc9
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.3.4rc7 → reqmgr2ms-output-2.3.4rc9}/src/python/Utils/IteratorTools.py

@@ -18,6 +18,18 @@ def grouper(iterable, n):
     return iter(lambda: list(islice(iterable, n)), [])
 
 
+def getChunk(arr, step):
+    """
+    Return chunk of entries from given array and step, it is similar in behavior to grouper
+    function but instead of returning new list it provides a generator iterable object.
+    :param arr: input array of data
+    :param step: step to iterate
+    :return: generator, set of slices with number of entries equal to step of iteration
+    """
+    for i in range(0, len(arr), step):
+        yield arr[i:i + step]
+
+
 def flattenList(doubleList):
     """
     Make flat a list of lists.

{reqmgr2ms-output-2.3.4rc7 → reqmgr2ms-output-2.3.4rc9}/src/python/WMCore/Services/Rucio/Rucio.py

@@ -18,6 +18,7 @@ from rucio.common.exception import (AccountNotFound, DataIdentifierNotFound, Acc
                                     DataIdentifierAlreadyExists, DuplicateContent, InvalidRSEExpression,
                                     UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound)
 from Utils.MemoryCache import MemoryCache
+from Utils.IteratorTools import grouper
 from WMCore.Services.Rucio.RucioUtils import (validateMetaData, weightedChoice,
                                               isTapeRSE, dropTapeRSEs)
 from WMCore.WMException import WMException
@@ -346,7 +347,7 @@ class Rucio(object):
             response = self.attachDIDs(kwargs.get('rse'), container, name, scope)
         return response
 
-    def attachDIDs(self, rse, superDID, dids, scope='cms'):
+    def attachDIDs(self, rse, superDID, dids, scope='cms', chunkSize=1000):
         """
         _attachDIDs_
 
@@ -356,28 +357,38 @@ class Rucio(object):
              then it's a container name; if attaching files, then it's a block name)
         :param dids: either a string or a list of data identifiers (can be block or files)
         :param scope: string with the scope name
+        :param chunkSize: maximum number of dids to be attached in any single Rucio server call
         :return: a boolean to represent whether it succeeded or not
         """
         if not isinstance(dids, list):
             dids = [dids]
         # NOTE: the attaching dids do not create new container within a scope
         # and it is safe to use cms scope for it
-        dids = [{'scope': 'cms', 'name': did} for did in dids]
+        alldids = [{'scope': 'cms', 'name': did} for did in sorted(dids)]
+
+        # report if we use chunk size in rucio attach_dids API call
+        if len(alldids) > chunkSize:
+            self.logger.info("Attaching a total of %d DIDs in chunk size of: %d", len(alldids), chunkSize)
+
+        for dids in grouper(alldids, chunkSize):
+            response = False
+            try:
+                response = self.cli.attach_dids(scope, superDID, dids=dids, rse=rse)
+            except DuplicateContent:
+                self.logger.warning("Dids: %s already attached to: %s", dids, superDID)
+                response = True
+            except FileAlreadyExists:
+                self.logger.warning("Failed to attach files already existent on block: %s", superDID)
+                response = True
+            except DataIdentifierNotFound:
+                self.logger.error("Failed to attach dids: %s. Parent DID %s does not exist.", dids, superDID)
+            except Exception as ex:
+                self.logger.error("Exception attaching %s dids to: %s. Error: %s. First 10 dids: %s",
+                                  len(dids), superDID, str(ex), dids[:10])
+            if not response:
+                # if we had failure with specific chunk of dids we'll return immediately
+                return response
 
-        response = False
-        try:
-            response = self.cli.attach_dids(scope, superDID, dids=dids, rse=rse)
-        except DuplicateContent:
-            self.logger.warning("Dids: %s already attached to: %s", dids, superDID)
-            response = True
-        except FileAlreadyExists:
-            self.logger.warning("Failed to attach files already existent on block: %s", superDID)
-            response = True
-        except DataIdentifierNotFound:
-            self.logger.error("Failed to attach dids: %s. Parent DID %s does not exist.", dids, superDID)
-        except Exception as ex:
-            self.logger.error("Exception attaching dids: %s to: %s. Error: %s",
-                              dids, superDID, str(ex))
         return response
 
     def createReplicas(self, rse, files, block, scope='cms', ignoreAvailability=True):

{reqmgr2ms-output-2.3.4rc7 → reqmgr2ms-output-2.3.4rc9}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.4rc7'
+__version__ = '2.3.4rc9'
 __all__ = []

{reqmgr2ms-output-2.3.4rc7 → reqmgr2ms-output-2.3.4rc9/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.4rc7
+Version: 2.3.4rc9
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group