reqmgr2ms-output 2.2.4rc10__tar.gz → 2.2.4.2__tar.gz

{reqmgr2ms-output-2.2.4rc10/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.2.4rc10
+Version: 2.2.4.2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2}/src/python/Utils/IteratorTools.py

@@ -2,7 +2,7 @@
 
 from builtins import str, map
 import collections
-from itertools import islice, chain
+from itertools import islice, chain, groupby
 
 def grouper(iterable, n):
     """
@@ -24,6 +24,7 @@ def flattenList(doubleList):
     """
     return list(chain.from_iterable(doubleList))
 
+
 def nestedDictUpdate(d, u):
     """
     Code from Alex Matelli
@@ -50,3 +51,18 @@ def convertFromUnicodeToBytes(data):
         return type(data)(list(map(convertFromUnicodeToBytes, data)))
     else:
         return data
+
+
+def makeListElementsUnique(listObj):
+    """
+    Given a list of lists or a list of tuples, find all duplicate elements
+    and make them unique.
+    :param listObj: an unsorted list of lists or a list of tuples, e.g.:
+        [[1, 1], [1, 5], [1, 1]]; or
+        [(1, 1), (1, 5), (1, 1)]
+    :return: the same list object but with no duplicates
+
+    Source: https://stackoverflow.com/questions/2213923/removing-duplicates-from-a-list-of-lists
+    """
+    listObj.sort()
+    return list(k for k, _ in groupby(listObj))

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2}/src/python/WMCore/MicroService/MSCore/MSManager.py

@@ -11,7 +11,7 @@ used in service config.py as following
     # REST interface
     data = views.section_('data')
     data.object = 'WMCore.MicroService.Service.RestApiHub.RestApiHub'
-    data.manager = 'WMCore.MicroService.Unified.MSManager.MSManager'
+    data.manager = 'WMCore.MicroService.MSManager.MSManager'
     data.reqmgr2Url = "%s/reqmgr2" % BASE_URL
     data.limitRequestsPerCycle = 500
     data.enableStatusTransition = False
@@ -217,8 +217,9 @@ class MSManager(object):
     def transferor(self, reqStatus):
         """
         MSManager transferor function.
-        It performs Unified logic for data subscription and
-        transfers requests from assigned to staging/staged state of ReqMgr2.
+        It performs input data placement logic for workflows that have
+        been assigned in the system. Successful workflows will be moved
+        to the staging status in ReqMgr2.
         For references see
         https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Transferor
         """

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2}/src/python/WMCore/REST/Error.py

@@ -226,6 +226,27 @@ class ExecutionError(RESTError):
     app_code = 403
     message = "Execution error"
 
+
+class MissingBodyData(RESTError):
+    """Exception for requests missing body data."""
+    http_code = 400
+    app_code = 1201
+
+    def __init__(self):
+        RESTError.__init__(self)
+        self.message = "User did not pass any body data with the request"
+
+
+class InvalidUnifiedSchema(RESTError):
+    """Exception for incorrect user data schema."""
+    http_code = 400
+    app_code = 1202
+
+    def __init__(self, message):
+        RESTError.__init__(self)
+        self.message = message
+
+
 def report_error_header(header, val):
     """If `val` is non-empty, set CherryPy response `header` to `val`.
     Replaces all newlines with "; " characters. If the resulting value is

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2}/src/python/WMCore/Services/DBS/DBS3Reader.py

@@ -20,7 +20,7 @@ from dbs.apis.dbsClient import DbsApi
 from dbs.exceptions.dbsClientException import dbsClientException
 from retry import retry
 
-from Utils.IteratorTools import grouper
+from Utils.IteratorTools import grouper, makeListElementsUnique
 from Utils.PythonVersion import PY2
 from WMCore.Services.DBS.DBSErrors import DBSReaderError, formatEx3
 from WMCore.Services.DBS.DBSUtils import dbsListFileParents, dbsListFileLumis, \
@@ -848,18 +848,26 @@ class DBS3Reader(object):
                 result.append({"ParentDataset": parent['parent_dataset'], "ParentFiles": list(parentFiles[childLFN])})
         return result
 
-    def insertFileParents(self, childBlockName, childParentsIDPairs):
+    def insertFileParents(self, childBlockName, childParentsIDPairs, missingFiles=0):
         """
+        For a given block name, inject its child/parent file id relationship
         :param childBlockName: child block name
         :param childParentsIDPairs: list of list child and parent file ids, i.e. [[1,2], [3,4]...]
                 dbs validate child ids from the childBlockName
+        :param missingFiles: an integer with the number of children files missing parents
         :return: None
         """
-        return self.dbs.insertFileParents({"block_name": childBlockName, "child_parent_id_list": childParentsIDPairs})
+        self.logger.debug("Going to insert parentage for child_parent_id_list: %s",
+                          childParentsIDPairs)
+        return self.dbs.insertFileParents({"block_name": childBlockName,
+                                           "child_parent_id_list": childParentsIDPairs,
+                                           "missing_files": missingFiles})
 
     def listBlocksWithNoParents(self, childDataset):
         """
-        :param childDataset: child dataset for
+        Given a dataset name, list all its blocks, fetch their parentage
+        blocks and return a set of blocks without any parentage information.
+        :param childDataset: string with a dataset name
         :return: set of child blocks with no parentBlock
         """
         allBlocks = self.dbs.listBlocks(dataset=childDataset)
@@ -916,29 +924,11 @@ class DBS3Reader(object):
         self.logger.info("Found %d blocks without parentage information", len(blocks))
         for blockName in blocks:
             try:
-                self.logger.info("Fixing parentage for block: %s", blockName)
-                childFlatData = self.getChildBlockTrio(blockName)
-                listChildParent = []
-
-                # first resolve parentage for all common runLumi pairs between childBlock and parentDataset
-                for runLumi in childFlatData.keys() & parentFlatData.keys():
-                    childFileId = childFlatData[runLumi]
-                    parentFileId = parentFlatData[runLumi]
-                    listChildParent.append([childFileId, parentFileId])
-
-                # then add all run lumi pairs which are missing at the parent Dataset by appending None as parentage information
-                for runLumi in childFlatData.keys() - parentFlatData.keys():
-                    childFileId = childFlatData[runLumi]
-                    parentFileId = None
-                    listChildParent.append([childFileId, parentFileId])
-                    msg = "Child file id: %s, with run/lumi: %s, has no match in the parent dataset. "
-                    msg += "Adding it with null parentage information to DBS."
-                    self.logger.warning(msg, childFileId, runLumi)
-
+                listChildParent, countMissingFiles = self._compileParentageList(blockName, parentFlatData)
                 # insert block parentage information to DBS
                 if insertFlag and any(listChildParent):
-                    self.insertFileParents(blockName, listChildParent)
-                    self.logger.info("Parentage information successfuly added to DBS for block %s", blockName)
+                    self.insertFileParents(blockName, listChildParent, countMissingFiles)
+                    self.logger.info("Parentage information successfully added to DBS for block %s", blockName)
                 else:
                     self.logger.warning("No parentage information added to DBS for block %s", blockName)
             except Exception as ex:
@@ -948,6 +938,59 @@ class DBS3Reader(object):
 
         return failedBlocks
 
+    def _compileParentageList(self, blockName, parentRunLumi):
+        """
+        Method to find out child and parent file relationship based
+        on their run/lumi tuples.
+        :param blockName: string with the child block name
+        :param parentRunLumi: a set like {(1, 53): 3077147397, (1, 54): 3077147397, (1, 27): 3077147397
+        :return: a list of child/parent file id tuples and a set of children files
+            that are missing parent files, e.g.
+            [[3077147917, 3077148037], [3077147917, 3077148037], 123
+        """
+        self.logger.info("Compiling parentage list for block: %s", blockName)
+        # fetch run/lumi and file ids for the child block name
+        childFlatData = self.getChildBlockTrio(blockName)
+
+        self.logger.debug("Block name: %s has this run/lumi/file id information: %s",
+                          blockName, childFlatData)
+        listChildParent = []
+
+        # first resolve parentage for all common runLumi pairs between childBlock and parentDataset
+        withParents = set()
+        for runLumi in childFlatData.keys() & parentRunLumi.keys():
+            childFileId = childFlatData[runLumi]
+            withParents.add(childFileId)
+            parentFileId = parentRunLumi[runLumi]
+            listChildParent.append([childFileId, parentFileId])
+
+        # the next for loop will find all the children files with missing parents
+        # and set their parent file id to -1 instead, unless that child id already has
+        # a valid parent file for other lumi(s)
+        missingParents = set()
+        for runLumi in childFlatData.keys() - parentRunLumi.keys():
+            childFileId = childFlatData[runLumi]
+            msg = "Child file id: %s, with run/lumi: %s, has no match in the parent dataset. "
+            if childFileId in withParents:
+                msg += "It does have parent files for other run/lumis though."
+                self.logger.warning(msg, childFileId, runLumi)
+                continue
+            missingParents.add(childFileId)
+            listChildParent.append([childFileId, -1])
+            msg += "Adding it with -1 parentage information to DBS."
+            self.logger.warning(msg, childFileId, runLumi)
+        self.logger.debug("Files with parent: %s, without: %s, non-unique tuples: %d",
+                         withParents, missingParents, len(listChildParent))
+
+        # now find out files missing parent that do not have any other parent
+        missingParents = missingParents - withParents
+        # and make it a unique list of child/parent file ids
+        listChildParent = makeListElementsUnique(listChildParent)
+
+        self.logger.info("Block: %s has %d child/parent tuples and it is missing %d files",
+                         blockName, len(listChildParent), len(missingParents))
+        return listChildParent, len(missingParents)
+
     def getParentDatasetTrio(self, childDataset):
         """
         Provided a dataset name, return all the parent dataset information, such as:

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.2.4rc10'
+__version__ = '2.2.4.2'
 __all__ = []

{reqmgr2ms-output-2.2.4rc10 → reqmgr2ms-output-2.2.4.2/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.2.4rc10
+Version: 2.2.4.2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group