reqmgr2ms-output 2.3.2rc5__tar.gz → 2.3.2rc8__tar.gz

{reqmgr2ms-output-2.3.2rc5/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.2rc8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc5
+Version: 2.3.2rc8
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-output-2.3.2rc8/bin/adhoc-scripts/adjustMongoDocs.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+"""
+File       : adjustMongoDocs.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: adjust documents in MongoDB with given schema
+The script loops over given db documents (from given db name
+and collection) and add to each document a new schema provided
+via input file name, e.g.
+
+# new schema addition:
+cat > /tmp/schema.json << EOF
+{"foo": 1}
+EOF
+
+# run script to update all documents in test.test database and add to them
+# a json structure presented in /tmp/schema.json file
+adjustMongoDocs.py --fin=/tmp/schema.json --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+# run script to add new transition record to all MSPileup records if they do not have it
+adjustMongoDocs.py --add-tran-record --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+NOTE: the script can run from any location assuming it has proper db uri with
+credentials and availability of mongo db outside of its own cluster, but most
+likely it will be run on mongo db k8s cluster
+"""
+
+import argparse
+import json
+from pymongo import MongoClient
+
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
+
+class OptionParser(object):
+    "Helper class to handle script options"
+
+    def __init__(self):
+        "User based option parser"
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields")
+        self.parser.add_argument("--dburi", action="store",
+                                 dest="dburi", default="", help="MongoDB URI")
+        self.parser.add_argument("--dbname", action="store",
+                                 dest="dbname", default="", help="MongoDB database")
+        self.parser.add_argument("--dbcoll", action="store",
+                                 dest="dbcoll", default="", help="MongoDB collection")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transaction record")
+        self.parser.add_argument("--verbose", action="store",
+                                 dest="verbose", default=0, help="verbosity level")
+
+
+def addTransitionRecord(dburi, dbname, dbcoll, userDN, verbose):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param userDN: string representing user DN
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    conn = MongoClient(host=dburi)
+    records = [r for r in conn[dbname][dbcoll].find()]
+    for rec in records:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            if verbose:
+                print(f"update rec={rec} with new transition record={tranRecord}")
+            spec = {'pileupName': rec['pileupName'], 'customName': rec.get('customName', '')}
+            # perform update of the record
+            conn[dbname][dbcoll].update_one(spec, {'$set': {'transition': [tranRecord]}}, upsert=True)
+
+
+def adjust(dburi, dbname, dbcoll, fin, verbose):
+    """
+    helper script to update documents in MongoDB with given db name/collection
+    and schema file (which should contain new addition, in form of JSON, to the
+    db documents)
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param fin: input file with schema content
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    conn = MongoClient(host=dburi)
+    schema = {}
+    with open(fin, 'r', encoding='utf-8') as istream:
+        schema = json.load(istream)
+    if schema:
+        if verbose:
+            print(f"read data from '{dburi}', {dbname}/{dbcoll}")
+        records = [r for r in conn[dbname][dbcoll].find()]
+        for doc in records:
+            if verbose:
+                print(f"update doc={doc} with schema={schema}")
+            conn[dbname][dbcoll].update_one(doc, {'$set': schema}, upsert=True)
+
+
+def main():
+    """
+    Main function
+    """
+    optmgr = OptionParser()
+    opts = optmgr.parser.parse_args()
+    if opts.transition:
+        addTransitionRecord(opts.dburi, opts.dbname, opts.dbcoll, opts.userDN, opts.verbose)
+    else:
+        adjust(opts.dburi, opts.dbname, opts.dbcoll, opts.fin, opts.verbose)
+
+
+if __name__ == '__main__':
+    main()

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8}/bin/adhoc-scripts/updatePileupObjects.py

@@ -9,6 +9,9 @@ used to update every single pileup document that it fetches from MSPileup.
 
 Example usage is:
 python3 updatePileupObjects.py --url=https://cmsweb.cern.ch --fin=override_data.json
+
+# run script to add new transition record to all MSPileup records if they do not have it
+python3 updatePileupObjects.py --add-tran-record -url=https://cmsweb.cern.ch --userDN=<user-dn>
 """
 import argparse
 import json
@@ -19,6 +22,9 @@ import urllib.request
 import ssl
 from http.client import HTTPSConnection
 
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
 
 class OptionParser():
     """Class to parse the command line arguments"""
@@ -26,13 +32,17 @@ class OptionParser():
     def __init__(self):
         "User based option parser"
         self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--fin", required=True,
-                                 dest="fin", help="Input JSON file")
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields (optional)")
         self.parser.add_argument("--dry-run", action="store_true",
                                  dest="dryrun", help="Fetch docs but do not update")
         self.parser.add_argument("--url", action="store",
                                  dest="url", default="https://cmsweb-testbed.cern.ch",
                                  help="URL for the MSPileup service")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transition record")
 
 
 class HTTPSClientAuthHandler(urllib.request.HTTPSHandler):
@@ -147,6 +157,79 @@ def writePileupDocs(mspileupUrl, puDocs, handler, logger):
                 logger.critical(msg)
 
 
+def addTransitionRecords(handler, url, userDN, logger, dryrun):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param userDN: string representing user DN
+    :param logger: logger to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+    recordsToUpdate = []
+    for rec in puDocs:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            logger.info("New pileup document is: %s", rec)
+            recordsToUpdate.append(rec)
+
+    if dryrun:
+        logger.info("%s documents needs to be updated but are not written due to --dry-run option\n", len(recordsToUpdate))
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, recordsToUpdate, handler, logger)
+        logger.info("%s documents are uploaded to MSPileup\n", len(recordsToUpdate))
+
+
+def updatePileupRecords(handler, url, logger, dryrun, fin=""):
+    """
+    helper function to update pileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param logger: logger object to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :param fin: input file name to read from (optional, string)
+    :return: nothing
+    """
+    if fin:
+        if os.path.exists(fin):
+            with open(fin, 'r', encoding='utf-8') as istream:
+                puOverride = json.load(istream)
+            logger.info("Pileup override file: %s has the following content: %s", fin, puOverride)
+        else:
+            logger.error("Provided %s file does not exist", fin)
+            os.exit(1)
+    else:
+        logger.info("No input file is provided to update pileup records")
+        os.exit(1)
+
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+
+    # update the documents with the override content
+    for doc in puDocs:
+        doc.update(puOverride)
+        logger.info("New pileup document is: %s", doc)
+
+    if dryrun:
+        logger.info("Updated documents (dry-run option):")
+        for doc in puDocs:
+            logger.info(doc)
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, puDocs, handler, logger)
+        logger.info("documents are uploaded to MSPileup\n")
+
+
 def main():
     """Executes everything"""
     optmgr = OptionParser()
@@ -179,24 +262,10 @@ def main():
     # initialize HTTPS client handler
     handler = HTTPSClientAuthHandler(key, cert)
 
-    with open(opts.fin, 'r', encoding='utf-8') as istream:
-        puOverride = json.load(istream)
-    logger.info("Pileup override file: %s has the following content: %s", opts.fin, puOverride)
-
-    puDocs = getPileupDocs(opts.url, handler, logger)
-    logger.info("Found %d documents in MSPileup", len(puDocs))
-
-    # update the documents with the override content
-    for doc in puDocs:
-        doc.update(puOverride)
-        logger.info("New pileup document is: %s", doc)
-
-    if opts.dryrun:
-        logger.info("documents are not written due to --dry-run option\n")
+    if opts.transition:
+        addTransitionRecords(handler, opts.url, opts.userDN, logger, opts.dryrun)
     else:
-        # finally, update the pileup documents in the database
-        writePileupDocs(opts.url, puDocs, handler, logger)
-        logger.info("documents are uploaded to MSPileup\n")
+        updatePileupRecords(handler, opts.url, logger, opts.dryrun, opts.fin)
 
 
 if __name__ == '__main__':

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8}/bin/wmagent-mod-config

@@ -288,13 +288,15 @@ def modifyConfiguration(config, **args):
     if hasattr(config, "ArchiveDataReporter"):
         config.ArchiveDataReporter.WMArchiveURL = args["wmarchive_url"]
 
-        # custom WMArchiveReporter
+    # custom WMArchiveReporter
     if hasattr(config, "WorkflowUpdater"):
+        config.WorkflowUpdater.dbsUrl = args.get("dbs3_reader_url", None)
         config.WorkflowUpdater.rucioUrl = args["rucio_host"]
         config.WorkflowUpdater.rucioAuthUrl = args["rucio_auth"]
         if args.get("mspileup_url", None):
             config.WorkflowUpdater.msPileupUrl = args["mspileup_url"]
 
+
     return config
 
 
@@ -319,7 +321,7 @@ def main(argv=None):
                                          "sb_hostname=", "sb_port=", "sb_basepath=", "ufc_hostname=", "ufc_port=",
                                          "ufc_service_url=", "ufc_cachedir=", "client_mapping=",
                                          "workload_summary_url=", "coredb_url=", "wmstats_url=", "ops_proxy=",
-                                         "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=",
+                                         "reqmgr2_url=", "acdc_url=", "amq_auth_file=", "dbs3_url=", "dbs3_reader_url=",
                                          "dqm_url=", "grafana_token=", "requestcouch_url=", "central_logdb_url=",
                                          "wmarchive_url=", "amq_credentials=",
                                          "rucio_account=", "rucio_host=", "rucio_auth=", "mspileup_url="])
@@ -343,7 +345,7 @@ def main(argv=None):
                           '--ufc_port', '--ufc_service_url', '--ufc_cachedir',
                           '--client_mapping', '--workload_summary_url',
                           '--wmstats_url', '--ops_proxy', '--reqmgr2_url', '--acdc_url',
-                          '--amq_auth_file', '--dbs3_url', '--dqm_url',
+                          '--amq_auth_file', '--dbs3_url', '--dbs3_reader_url', '--dqm_url',
                           '--grafana_token', '--requestcouch_url', '--central_logdb_url',
                           '--wmarchive_url', '--amq_credentials',
                           '--rucio_account', '--rucio_host', '--rucio_auth', '--mspileup_url'):

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8}/src/python/WMCore/Lexicon.py

@@ -99,7 +99,7 @@ def searchblock(candidate):
     """
     A block name with a * wildcard one or more times in it.
     """
-    regexp = r"^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z\-\*]{1,99})(#(\*|[a-zA-Z0-9\.\-_\*]){0,100}){0,1}){0,1}$"
+    regexp = r"^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z0-9\-\*]{1,99})(#(\*|[a-zA-Z0-9\.\-_\*]){0,100}){0,1}){0,1}$"
     return check(regexp, candidate)
 
 
@@ -354,8 +354,8 @@ def lfn(candidate):
 
     Add for LHE files: /data/lhe/...
     """
-    regexp1 = '/([a-z]+)/([a-z0-9]+)/(%(era)s)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root' % lfnParts
-    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root'
+    regexp1 = '/([a-z]+)/([a-z0-9]+)/(%(era)s)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root' % lfnParts
+    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}/([0-9]+)/([a-zA-Z0-9\-_]+).root'
     regexp3 = '/store/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/%(procDS)s/%(version)s/%(counter)s/%(root)s' % lfnParts
     regexp4 = '/store/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/(%(subdir)s/)+%(root)s' % lfnParts
 
@@ -437,8 +437,8 @@ def lfnBase(candidate):
     As lfn above, but for doing the lfnBase
     i.e., for use in spec generation and parsing
     """
-    regexp1 = '/([a-z]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)'
-    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}'
+    regexp1 = '/([a-z]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)'
+    regexp2 = '/([a-z]+)/([a-z0-9]+)/([a-z0-9]+)/([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)/([A-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)((/[0-9]+){3}){0,1}'
     regexp3 = '/(store)/(temp/)*(user|group)/(%(hnName)s|%(physics_group)s)/%(primDS)s/%(procDS)s/%(version)s' % lfnParts
 
     tier0LFN = '/store/(backfill/[0-9]/){0,1}(t0temp/|unmerged/){0,1}(data|express|hidata)/%(era)s/%(primDS)s/%(tier)s/%(version)s/%(counter)s/%(counter)s/%(counter)s' % lfnParts

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8}/src/python/WMCore/Services/DBS/DBSConcurrency.py

@@ -11,20 +11,23 @@ import urllib
 from WMCore.Services.pycurl_manager import getdata as multi_getdata
 
 
-def getBlockInfo4PU(blockNames, ckey, cert):
+def getBlockInfo4PU(blockNames, dbsUrl, ckey, cert):
     """
     Fetch block information details, file list and number of events, from DBS
     server. Here we use concrete set of parameters for DBS to use in this case, i.e.
     we must look-up only valid files and get full details from the DBS API (in order
     to get number of events).
     :param blockNames: list of block names
+    :param dbsUrl: dbs URL
+    :param ckey: user keyfile
+    :param cert: user certificate
     :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
     """
     urls = []
     for blk in blockNames:
         # need to encode block name properly
         block = urllib.parse.quote_plus(blk)
-        url = f"https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/files?detail=true&validFileOnly=1&block_name={block}"
+        url = f"{dbsUrl}/files?detail=true&validFileOnly=1&block_name={block}"
         urls.append(url)
     # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
     # it does not put DBS results into the memory until this generator is iterated

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.2rc5'
+__version__ = '2.3.2rc8'
 __all__ = []

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc8/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc5
+Version: 2.3.2rc8
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-output-2.3.2rc5/bin/adhoc-scripts/adjustMongoDocs.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-"""
-File       : adjustMongoDocs.py
-Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
-Description: adjust documents in MongoDB with given schema
-The script loops over given db documents (from given db name
-and collection) and add to each document a new schema provided
-via input file name, e.g.
-
-# new schema addition:
-cat > /tmp/schema.json << EOF
-{"foo": 1}
-EOF
-
-# run script to update all documents in test.test database and add to them
-# a json structure presented in /tmp/schema.json file
-adjustMongoDocs.py --fin=/tmp/schema.json --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
-
-NOTE: the script can run from any location assuming it has proper db uri with
-credentials and availability of mongo db outside of its own cluster, but most
-likely it will be run on mongo db k8s cluster
-"""
-
-import argparse
-import json
-from pymongo import MongoClient
-
-
-class OptionParser(object):
-    "Helper class to handle script options"
-
-    def __init__(self):
-        "User based option parser"
-        self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--fin", action="store",
-                                 dest="fin", default="", help="Input file with update fields")
-        self.parser.add_argument("--dburi", action="store",
-                                 dest="dburi", default="", help="MongoDB URI")
-        self.parser.add_argument("--dbname", action="store",
-                                 dest="dbname", default="", help="MongoDB database")
-        self.parser.add_argument("--dbcoll", action="store",
-                                 dest="dbcoll", default="", help="MongoDB collection")
-        self.parser.add_argument("--verbose", action="store",
-                                 dest="verbose", default=0, help="verbosity level")
-
-
-def adjust():
-    """
-    helper script to update documents in MongoDB with given db name/collection
-    and schema file (which should contain new addition, in form of JSON, to the
-    db documents)
-    """
-    optmgr = OptionParser()
-    opts = optmgr.parser.parse_args()
-    conn = MongoClient(host=opts.dburi)
-    dbname = opts.dbname
-    dbcoll = opts.dbcoll
-    verbose = opts.verbose
-    schema = {}
-    with open(opts.fin, 'r', encoding='utf-8') as istream:
-        schema = json.load(istream)
-    if schema:
-        if verbose:
-            print("read data from '%s', %s/%s" % (opts.dburi, dbname, dbcoll))
-        records = [r for r in conn[dbname][dbcoll].find()]
-        for doc in records:
-            if verbose:
-                print("update doc=%s with schema=%s" % (doc, schema))
-            conn[dbname][dbcoll].update_one(doc, {'$set': schema}, upsert=True)
-
-
-if __name__ == '__main__':
-    adjust()