reqmgr2ms-output 2.3.1rc3__tar.gz → 2.3.2__tar.gz

{reqmgr2ms-output-2.3.1rc3/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.1rc3
+Version: 2.3.2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-output-2.3.2/bin/adhoc-scripts/adjustMongoDocs.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+"""
+File       : adjustMongoDocs.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: adjust documents in MongoDB with given schema
+The script loops over given db documents (from given db name
+and collection) and add to each document a new schema provided
+via input file name, e.g.
+
+# new schema addition:
+cat > /tmp/schema.json << EOF
+{"foo": 1}
+EOF
+
+# run script to update all documents in test.test database and add to them
+# a json structure presented in /tmp/schema.json file
+adjustMongoDocs.py --fin=/tmp/schema.json --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+# run script to add new transition record to all MSPileup records if they do not have it
+adjustMongoDocs.py --add-tran-record --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+NOTE: the script can run from any location assuming it has proper db uri with
+credentials and availability of mongo db outside of its own cluster, but most
+likely it will be run on mongo db k8s cluster
+"""
+
+import argparse
+import json
+from pymongo import MongoClient
+
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
+
+class OptionParser(object):
+    "Helper class to handle script options"
+
+    def __init__(self):
+        "User based option parser"
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields")
+        self.parser.add_argument("--dburi", action="store",
+                                 dest="dburi", default="", help="MongoDB URI")
+        self.parser.add_argument("--dbname", action="store",
+                                 dest="dbname", default="", help="MongoDB database")
+        self.parser.add_argument("--dbcoll", action="store",
+                                 dest="dbcoll", default="", help="MongoDB collection")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transaction record")
+        self.parser.add_argument("--verbose", action="store",
+                                 dest="verbose", default=0, help="verbosity level")
+
+
+def addTransitionRecord(dburi, dbname, dbcoll, userDN, verbose):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param userDN: string representing user DN
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    conn = MongoClient(host=dburi)
+    records = [r for r in conn[dbname][dbcoll].find()]
+    for rec in records:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            if verbose:
+                print(f"update rec={rec} with new transition record={tranRecord}")
+            spec = {'pileupName': rec['pileupName'], 'customName': rec.get('customName', '')}
+            # perform update of the record
+            conn[dbname][dbcoll].update_one(spec, {'$set': {'transition': [tranRecord]}}, upsert=True)
+
+
+def adjust(dburi, dbname, dbcoll, fin, verbose):
+    """
+    helper script to update documents in MongoDB with given db name/collection
+    and schema file (which should contain new addition, in form of JSON, to the
+    db documents)
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param fin: input file with schema content
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    conn = MongoClient(host=dburi)
+    schema = {}
+    with open(fin, 'r', encoding='utf-8') as istream:
+        schema = json.load(istream)
+    if schema:
+        if verbose:
+            print(f"read data from '{dburi}', {dbname}/{dbcoll}")
+        records = [r for r in conn[dbname][dbcoll].find()]
+        for doc in records:
+            if verbose:
+                print(f"update doc={doc} with schema={schema}")
+            conn[dbname][dbcoll].update_one(doc, {'$set': schema}, upsert=True)
+
+
+def main():
+    """
+    Main function
+    """
+    optmgr = OptionParser()
+    opts = optmgr.parser.parse_args()
+    if opts.transition:
+        addTransitionRecord(opts.dburi, opts.dbname, opts.dbcoll, opts.userDN, opts.verbose)
+    else:
+        adjust(opts.dburi, opts.dbname, opts.dbcoll, opts.fin, opts.verbose)
+
+
+if __name__ == '__main__':
+    main()

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/bin/adhoc-scripts/updatePileupObjects.py

@@ -9,6 +9,9 @@ used to update every single pileup document that it fetches from MSPileup.
 
 Example usage is:
 python3 updatePileupObjects.py --url=https://cmsweb.cern.ch --fin=override_data.json
+
+# run script to add new transition record to all MSPileup records if they do not have it
+python3 updatePileupObjects.py --add-tran-record -url=https://cmsweb.cern.ch --userDN=<user-dn>
 """
 import argparse
 import json
@@ -19,6 +22,9 @@ import urllib.request
 import ssl
 from http.client import HTTPSConnection
 
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
 
 class OptionParser():
     """Class to parse the command line arguments"""
@@ -26,13 +32,17 @@ class OptionParser():
     def __init__(self):
         "User based option parser"
         self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--fin", required=True,
-                                 dest="fin", help="Input JSON file")
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields (optional)")
         self.parser.add_argument("--dry-run", action="store_true",
                                  dest="dryrun", help="Fetch docs but do not update")
         self.parser.add_argument("--url", action="store",
                                  dest="url", default="https://cmsweb-testbed.cern.ch",
                                  help="URL for the MSPileup service")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transition record")
 
 
 class HTTPSClientAuthHandler(urllib.request.HTTPSHandler):
@@ -147,6 +157,79 @@ def writePileupDocs(mspileupUrl, puDocs, handler, logger):
                 logger.critical(msg)
 
 
+def addTransitionRecords(handler, url, userDN, logger, dryrun):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param userDN: string representing user DN
+    :param logger: logger to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+    recordsToUpdate = []
+    for rec in puDocs:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            logger.info("New pileup document is: %s", rec)
+            recordsToUpdate.append(rec)
+
+    if dryrun:
+        logger.info("%s documents needs to be updated but are not written due to --dry-run option\n", len(recordsToUpdate))
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, recordsToUpdate, handler, logger)
+        logger.info("%s documents are uploaded to MSPileup\n", len(recordsToUpdate))
+
+
+def updatePileupRecords(handler, url, logger, dryrun, fin=""):
+    """
+    helper function to update pileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param logger: logger object to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :param fin: input file name to read from (optional, string)
+    :return: nothing
+    """
+    if fin:
+        if os.path.exists(fin):
+            with open(fin, 'r', encoding='utf-8') as istream:
+                puOverride = json.load(istream)
+            logger.info("Pileup override file: %s has the following content: %s", fin, puOverride)
+        else:
+            logger.error("Provided %s file does not exist", fin)
+            os.exit(1)
+    else:
+        logger.info("No input file is provided to update pileup records")
+        os.exit(1)
+
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+
+    # update the documents with the override content
+    for doc in puDocs:
+        doc.update(puOverride)
+        logger.info("New pileup document is: %s", doc)
+
+    if dryrun:
+        logger.info("Updated documents (dry-run option):")
+        for doc in puDocs:
+            logger.info(doc)
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, puDocs, handler, logger)
+        logger.info("documents are uploaded to MSPileup\n")
+
+
 def main():
     """Executes everything"""
     optmgr = OptionParser()
@@ -179,24 +262,10 @@ def main():
     # initialize HTTPS client handler
     handler = HTTPSClientAuthHandler(key, cert)
 
-    with open(opts.fin, 'r', encoding='utf-8') as istream:
-        puOverride = json.load(istream)
-    logger.info("Pileup override file: %s has the following content: %s", opts.fin, puOverride)
-
-    puDocs = getPileupDocs(opts.url, handler, logger)
-    logger.info("Found %d documents in MSPileup", len(puDocs))
-
-    # update the documents with the override content
-    for doc in puDocs:
-        doc.update(puOverride)
-        logger.info("New pileup document is: %s", doc)
-
-    if opts.dryrun:
-        logger.info("documents are not written due to --dry-run option\n")
+    if opts.transition:
+        addTransitionRecords(handler, opts.url, opts.userDN, logger, opts.dryrun)
     else:
-        # finally, update the pileup documents in the database
-        writePileupDocs(opts.url, puDocs, handler, logger)
-        logger.info("documents are uploaded to MSPileup\n")
+        updatePileupRecords(handler, opts.url, logger, opts.dryrun, opts.fin)
 
 
 if __name__ == '__main__':

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/bin/wmagent-mod-config

@@ -288,13 +288,15 @@ def modifyConfiguration(config, **args):
     if hasattr(config, "ArchiveDataReporter"):
         config.ArchiveDataReporter.WMArchiveURL = args["wmarchive_url"]
 
-        # custom WMArchiveReporter
+    # custom WMArchiveReporter
     if hasattr(config, "WorkflowUpdater"):
+        config.WorkflowUpdater.dbsUrl = args.get("dbs3_reader_url", None)
         config.WorkflowUpdater.rucioUrl = args["rucio_host"]
         config.WorkflowUpdater.rucioAuthUrl = args["rucio_auth"]
         if args.get("mspileup_url", None):
             config.WorkflowUpdater.msPileupUrl = args["mspileup_url"]
 
+
     return config
 
 
@@ -319,7 +321,7 @@ def main(argv=None):
                                          "sb_hostname=", "sb_port=", "sb_basepath=", "ufc_hostname=", "ufc_port=",
                                          "ufc_service_url=", "ufc_cachedir=", "client_mapping=",
                                          "workload_summary_url=", "coredb_url=", "wmstats_url=", "ops_proxy=",
-                                         "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=",
+                                         "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=", "dbs3_reader_url=",
                                          "dqm_url=", "grafana_token=", "requestcouch_url=", "central_logdb_url=",
                                          "wmarchive_url=", "amq_credentials=",
                                          "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url="])
@@ -343,7 +345,7 @@ def main(argv=None):
                           '--ufc_port', '--ufc_service_url', '--ufc_cachedir',
                           '--client_mapping', '--workload_summary_url',
                           '--wmstats_url', '--ops_proxy', '--reqmgr2_url', '--acdc_url',
-                          '--amq_auth_file', '--dbs3_url', '--dqm_url',
+                          '--amq_auth_file', '--dbs3_url', '--dbs3_reader_url', '--dqm_url',
                           '--grafana_token', '--requestcouch_url', '--central_logdb_url',
                           '--wmarchive_url', '--amq_credentials',
                           '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url'):

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/setup_dependencies.py

@@ -223,7 +223,8 @@ dependencies = {
         'packages': ['WMCore.Credential', 'WMCore.Algorithms+', 'WMCore.WMSpec+',
                      'WMCore.JobSplitting', 'WMCore.Services+', 'Utils+'],
         'systems': ['wmc-database', 'wmc-runtime'],
-        'modules': ['WMCore.WMBS.File', 'WMCore.WMBS.WMBSBase', 'WMCore.WMBS.__init__'],
+        'modules': ['WMCore.WMBS.File', 'WMCore.WMBS.WMBSBase', 'WMCore.WMBS.__init__',
+                    'WMCore.BossAir.Plugins.BasePlugin', 'WMCore.BossAir.Plugins.__init__'],
     },
     'wmclient': {
         'systems': ['wmc-runtime', 'wmc-database']

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/Utils/FileTools.py

@@ -6,6 +6,7 @@ Utilities related to file handling
 
 import io
 import os
+import glob
 import stat
 import subprocess
 import time
@@ -13,6 +14,37 @@ import zlib
 
 from Utils.Utilities import decodeBytesToUnicode
 
+
+def findFiles(path, pat):
+    """
+    Find files within given path and matching given pattern.
+    :param path: starting directory path (string)
+    :param pat: match pattern (string), e.g. *.py or name of the file
+    :return: matched file names
+    """
+    files = []
+    for idir, _, _ in os.walk(path):
+        files.extend(glob.glob(os.path.join(idir, pat)))
+    return files
+
+
+def tarMode(tfile, opMode):
+    """
+    Extract proper mode of operation for given tar file. For instance,
+    if op='r' and tfile name is file.tar.gz we should get 'r:gz',
+    while if tfile name is file.tar.bz2 we should get 'r':bz2', while
+    if tfile name is file.tar we should get 'r', etc.
+    :param opMode: mode of operation (string), e.g. 'r', or 'w'
+    :param tfile: sandbox tar file name (string)
+    :return: mode of operation
+    """
+    ext = tfile.split(".")[-1]
+    if ext == 'tar':
+        return opMode
+    mode = opMode + ':' + ext
+    return mode
+
+
 def calculateChecksums(filename):
     """
     _calculateChecksums_
@@ -108,6 +140,7 @@ def findMagicStr(filename, matchString):
             if matchString in line:
                 yield line
 
+
 def getFullPath(name, envPath="PATH"):
     """
     :param name: file name

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/WMCore/Lexicon.py

@@ -26,7 +26,7 @@ from WMCore.WMException import WMException, WMEXCEPTION_START_STR, WMEXCEPTION_E
 # i.e production should end with v[number]
 PRIMARY_DS = {'re': '^[a-zA-Z][a-zA-Z0-9\-_]*$', 'maxLength': 99}
 PROCESSED_DS = {'re': '[a-zA-Z0-9\.\-_]+', 'maxLength': 199}
-TIER = {'re': '[A-Z\-_]+', 'maxLength': 99}
+TIER = {'re': '[A-Z0-9\-_]+', 'maxLength': 99}
 BLOCK_STR = {'re': '#[a-zA-Z0-9\.\-_]+', 'maxLength': 100}
 
 lfnParts = {
@@ -99,11 +99,11 @@ def searchblock(candidate):
     """
     A block name with a * wildcard one or more times in it.
     """
-    regexp = r"^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z\-\*]{1,99})(#(\*|[a-zA-Z0-9\.\-_\*]){0,100}){0,1}){0,1}$"
+    regexp = r"^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z0-9\-\*]{1,99})(#(\*|[a-zA-Z0-9\.\-_\*]){0,100}){0,1}){0,1}$"
     return check(regexp, candidate)
 
 
-SEARCHDATASET_RE = r'^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*\-]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z\-\*]{1,50})){0,1}$'
+SEARCHDATASET_RE = r'^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*\-]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z0-9\-\*]{1,99})){0,1}$'
 
 
 def searchdataset(candidate):
@@ -202,7 +202,7 @@ def globalTag(candidate):
     return check(r'[a-zA-Z0-9\s\.\-_:]{1,100}$', candidate)
 
 
-DATASET_RE = r'^/[a-zA-Z0-9\-_]{1,99}/[a-zA-Z0-9\.\-_]{1,199}/[A-Z\-]{1,50}$'
+DATASET_RE = r'^/[a-zA-Z0-9\-_]{1,99}/[a-zA-Z0-9\.\-_]{1,199}/[A-Z0-9\-]{1,99}$'
 
 
 def dataset(candidate):
@@ -354,8 +354,8 @@ def lfn(candidate):
 
     Add for LHE files: /data/lhe/...
     """
-    regexp1 = '/([a-z]+)/([a-z0-9]+)/(%(era)s)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root' % lfnParts
-    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root'
+    regexp1 = '/([a-z]+)/([a-z0-9]+)/(%(era)s)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root' % lfnParts
+    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root'
     regexp3 = '/store/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/%(procDS)s/%(version)s/%(counter)s/%(root)s' % lfnParts
     regexp4 = '/store/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/(%(subdir)s/)+%(root)s' % lfnParts
 
@@ -437,8 +437,8 @@ def lfnBase(candidate):
     As lfn above, but for doing the lfnBase
     i.e., for use in spec generation and parsing
     """
-    regexp1 = '/([a-z]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)'
-    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}'
+    regexp1 = '/([a-z]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)'
+    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}'
     regexp3 = '/(store)/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/%(procDS)s/%(version)s' % lfnParts
 
     tier0LFN = '/store/(backfill/[0-9]/){0,1}(t0temp/|unmerged/){0,1}(data|express|hidata)/%(era)s/%(primDS)s/%(tier)s/%(version)s/%(counter)s/%(counter)s/%(counter)s' % lfnParts

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/WMCore/MicroService/MSOutput/MSOutput.py

@@ -1,14 +1,10 @@
 """
-File       : MSOtput.py
+File       : MSOutput.py
 
 Description: MSOutput.py class provides the whole logic behind
 the Output data placement in WMCore MicroServices.
 """
 
-# futures
-from __future__ import division, print_function
-from future.utils import viewitems
-
 # system modules
 import time
 from pymongo import IndexModel, ReturnDocument, errors
@@ -88,7 +84,7 @@ class MSOutput(MSCore):
         self.msConfig.setdefault("enableRelValCustodial", False)
         self.msConfig.setdefault("excludeDataTier", [])
         self.msConfig.setdefault("rucioAccount", 'wmcore_transferor')
-        self.msConfig.setdefault("rucioRSEAttribute", 'ddm_quota')
+        self.msConfig.setdefault("rucioRSEAttribute", 'dm_weight')
         self.msConfig.setdefault("rucioDiskRuleWeight", 'ddm_quota')
         self.msConfig.setdefault("rucioTapeExpression", 'rse_type=TAPE\cms_type=test')
         # This Disk expression wil target all real DISK T1 and T2 RSEs
@@ -240,7 +236,7 @@ class MSOutput(MSCore):
 
         # filter out documents already produced
         finalRequests = []
-        for reqName, reqData in viewitems(requestRecords):
+        for reqName, reqData in requestRecords.items():
             if reqName in mongoDocNames:
                 self.logger.info("Mongo document already created for %s, skipping it.", reqName)
             else:
@@ -634,7 +630,7 @@ class MSOutput(MSCore):
         A function used to update one or few particular fields in a document
         :**kwargs: The keys/value pairs to be updated (will be tested against MSOutputTemplate)
         """
-        for key, value in viewitems(kwargs):
+        for key, value in kwargs.items():
             try:
                 msOutDoc.setKey(key, value)
                 msOutDoc.updateTime()

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/WMCore/MicroService/Tools/Common.py

@@ -106,26 +106,6 @@ def dbsInfo(datasets, dbsUrl):
     return datasetBlocks, datasetSizes, datasetTransfers
 
 
-def getPileupDocs(mspileupUrl, queryDict):
-    """
-    Fetch documents from MSPileup according to the query passed in.
-    :param mspileupUrl: string with the MSPileup url
-    :param queryDict: dictionary with the MongoDB query to run
-    :return: returns a list with all the pileup objects, or raises
-        an exception in case of failure
-    """
-    mgr = RequestHandler()
-    headers = {'Content-Type': 'application/json'}
-    data = mgr.getdata(mspileupUrl, queryDict, headers, verb='POST',
-                       ckey=ckey(), cert=cert(), encode=True, decode=True)
-    if data and data.get("result", []):
-        if "error" in data["result"][0]:
-            msg = f"Failed to retrieve MSPileup documents with query: {queryDict}"
-            msg += f" and error message: {data}"
-            raise RuntimeError(msg)
-    return data["result"]
-
-
 def getPileupDatasetSizes(datasets, phedexUrl):
     """
     Given a list of datasets, find all their blocks with replicas

reqmgr2ms-output-2.3.2/src/python/WMCore/Services/DBS/DBSConcurrency.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+File       : DBSConcurrency.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: dedicated module to holds DBS related functions executed
+concurrent calls to DBS APIs.
+"""
+
+import json
+import urllib
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getBlockInfo4PU(blockNames, dbsUrl, ckey, cert):
+    """
+    Fetch block information details, file list and number of events, from DBS
+    server. Here we use concrete set of parameters for DBS to use in this case, i.e.
+    we must look-up only valid files and get full details from the DBS API (in order
+    to get number of events).
+    :param blockNames: list of block names
+    :param dbsUrl: dbs URL
+    :param ckey: user keyfile
+    :param cert: user certificate
+    :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
+    """
+    urls = []
+    for blk in blockNames:
+        # need to encode block name properly
+        block = urllib.parse.quote_plus(blk)
+        url = f"{dbsUrl}/files?detail=true&validFileOnly=1&block_name={block}"
+        urls.append(url)
+    # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
+    # it does not put DBS results into the memory until this generator is iterated
+    results = multi_getdata(urls, ckey, cert)
+    # parse output of getdata in some form
+    blockInfo = {}
+    for row in results:
+        blk = row['url'].split('block_name=')[-1]
+        block = urllib.parse.unquote_plus(blk)
+        data = json.loads(row['data'])
+        files = [r['logical_file_name'] for r in data]
+        nevents = sum([r['event_count'] for r in data])
+        blockInfo[block] = {'FileList': files, 'NumberOfEvents': nevents}
+    return blockInfo

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/WMCore/Services/MSPileup/MSPileupUtils.py

@@ -21,7 +21,6 @@ def getPileupDocs(mspileupUrl, queryDict=None, method='GET'):
     queryDict = queryDict or {}
     mgr = RequestHandler()
     headers = {'Content-Type': 'application/json'}
-    # TODO: Use GET instead?
     data = mgr.getdata(mspileupUrl, queryDict, headers, verb=method,
                        ckey=ckey(), cert=cert(), encode=True, decode=True)
     if data and data.get("result", []):

{reqmgr2ms-output-2.3.1rc3 → reqmgr2ms-output-2.3.2}/src/python/WMCore/Services/Rucio/Rucio.py

@@ -169,7 +169,7 @@ class Rucio(object):
         :return: a list of block names
         """
         blockNames = []
-        if not self.isContainer(container):
+        if not self.isContainer(container, scope=scope):
             # input container wasn't really a container
             self.logger.warning("Provided DID name is not a CONTAINER type: %s", container)
             return blockNames
@@ -360,7 +360,9 @@ class Rucio(object):
         """
         if not isinstance(dids, list):
             dids = [dids]
-        dids = [{'scope': scope, 'name': did} for did in dids]
+        # NOTE: the attaching dids do not create new container within a scope
+        # and it is safe to use cms scope for it
+        dids = [{'scope': 'cms', 'name': did} for did in dids]
 
         response = False
         try:
@@ -881,7 +883,7 @@ class Rucio(object):
 
         # At this point, we might already have some of the RSEs where the data is available and locked
         # Now check dataset locks and compare those rules against our list of multi RSE rules
-        if self.isContainer(kwargs['name']):
+        if self.isContainer(kwargs['name'], scope=kwargs.get('scope', 'cms')):
             # It's a container! Find what those RSEs are and add them to the finalRSEs set
             rseLocks = self._getContainerLockedAndAvailable(multiRSERules, returnTape=returnTape, **kwargs)
             self.logger.debug("Data location for %s from multiple RSE locks and available at: %s",
@@ -988,7 +990,7 @@ class Rucio(object):
         a specific method for this process, even though that adds some code duplication.
         """
         result = dict()
-        if not self.isContainer(container):
+        if not self.isContainer(container, scope=scope):
             raise WMRucioException("Pileup location needs to be resolved for a container DID type")
 
         multiRSERules = []
@@ -1096,7 +1098,7 @@ class Rucio(object):
         """
         if 'name' not in kwargs:
             raise WMRucioException("A DID name must be provided to the getBlockLockedAndAvailable API")
-        if self.isContainer(kwargs['name']):
+        if self.isContainer(kwargs['name'], scope=kwargs.get('scope', 'cms')):
             # then resolve it at container level and all its blocks
             return self.getContainerLockedAndAvailable(**kwargs)