reqmgr2ms-output 2.3.2rc5__tar.gz → 2.3.2rc6__tar.gz

{reqmgr2ms-output-2.3.2rc5/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.2rc6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc5
+Version: 2.3.2rc6
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-output-2.3.2rc6/bin/adhoc-scripts/adjustMongoDocs.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+"""
+File       : adjustMongoDocs.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: adjust documents in MongoDB with given schema
+The script loops over given db documents (from given db name
+and collection) and add to each document a new schema provided
+via input file name, e.g.
+
+# new schema addition:
+cat > /tmp/schema.json << EOF
+{"foo": 1}
+EOF
+
+# run script to update all documents in test.test database and add to them
+# a json structure presented in /tmp/schema.json file
+adjustMongoDocs.py --fin=/tmp/schema.json --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+# run script to add new transition record to all MSPileup records if they do not have it
+adjustMongoDocs.py --add-tran-record --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
+
+NOTE: the script can run from any location assuming it has proper db uri with
+credentials and availability of mongo db outside of its own cluster, but most
+likely it will be run on mongo db k8s cluster
+"""
+
+import argparse
+import json
+from pymongo import MongoClient
+
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
+
+class OptionParser(object):
+    "Helper class to handle script options"
+
+    def __init__(self):
+        "User based option parser"
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields")
+        self.parser.add_argument("--dburi", action="store",
+                                 dest="dburi", default="", help="MongoDB URI")
+        self.parser.add_argument("--dbname", action="store",
+                                 dest="dbname", default="", help="MongoDB database")
+        self.parser.add_argument("--dbcoll", action="store",
+                                 dest="dbcoll", default="", help="MongoDB collection")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transaction record")
+        self.parser.add_argument("--verbose", action="store",
+                                 dest="verbose", default=0, help="verbosity level")
+
+
+def addTransitionRecord(dburi, dbname, dbcoll, userDN, verbose):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param userDN: string representing user DN
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    conn = MongoClient(host=dburi)
+    records = [r for r in conn[dbname][dbcoll].find()]
+    for rec in records:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            if verbose:
+                print(f"update rec={rec} with new transition record={tranRecord}")
+            spec = {'pileupName': rec['pileupName'], 'customName': rec.get('customName', '')}
+            # perform update of the record
+            conn[dbname][dbcoll].update_one(spec, {'$set': {'transition': [tranRecord]}}, upsert=True)
+
+
+def adjust(dburi, dbname, dbcoll, fin, verbose):
+    """
+    helper script to update documents in MongoDB with given db name/collection
+    and schema file (which should contain new addition, in form of JSON, to the
+    db documents)
+    :param dburi: database URI (string)
+    :param dbname: database name (string)
+    :param dbcoll: database collection name (string)
+    :param fin: input file with schema content
+    :param verbose: verbose flag (boolean)
+    :return: nothing
+    """
+    conn = MongoClient(host=dburi)
+    schema = {}
+    with open(fin, 'r', encoding='utf-8') as istream:
+        schema = json.load(istream)
+    if schema:
+        if verbose:
+            print(f"read data from '{dburi}', {dbname}/{dbcoll}")
+        records = [r for r in conn[dbname][dbcoll].find()]
+        for doc in records:
+            if verbose:
+                print(f"update doc={doc} with schema={schema}")
+            conn[dbname][dbcoll].update_one(doc, {'$set': schema}, upsert=True)
+
+
+def main():
+    """
+    Main function
+    """
+    optmgr = OptionParser()
+    opts = optmgr.parser.parse_args()
+    if opts.transition:
+        addTransitionRecord(opts.dburi, opts.dbname, opts.dbcoll, opts.userDN, opts.verbose)
+    else:
+        adjust(opts.dburi, opts.dbname, opts.dbcoll, opts.fin, opts.verbose)
+
+
+if __name__ == '__main__':
+    main()

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc6}/bin/adhoc-scripts/updatePileupObjects.py

@@ -9,6 +9,9 @@ used to update every single pileup document that it fetches from MSPileup.
 
 Example usage is:
 python3 updatePileupObjects.py --url=https://cmsweb.cern.ch --fin=override_data.json
+
+# run script to add new transition record to all MSPileup records if they do not have it
+python3 updatePileupObjects.py --add-tran-record -url=https://cmsweb.cern.ch --userDN=<user-dn>
 """
 import argparse
 import json
@@ -19,6 +22,9 @@ import urllib.request
 import ssl
 from http.client import HTTPSConnection
 
+# WMCore services
+from Utils.Timers import gmtimeSeconds
+
 
 class OptionParser():
     """Class to parse the command line arguments"""
@@ -26,13 +32,17 @@ class OptionParser():
     def __init__(self):
         "User based option parser"
         self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--fin", required=True,
-                                 dest="fin", help="Input JSON file")
+        self.parser.add_argument("--fin", action="store",
+                                 dest="fin", default="", help="Input file with update fields (optional)")
         self.parser.add_argument("--dry-run", action="store_true",
                                  dest="dryrun", help="Fetch docs but do not update")
         self.parser.add_argument("--url", action="store",
                                  dest="url", default="https://cmsweb-testbed.cern.ch",
                                  help="URL for the MSPileup service")
+        self.parser.add_argument("--userdn", action="store",
+                                 dest="userDN", default='', help="user DN")
+        self.parser.add_argument("--add-tran-record", action="store_true",
+                                 dest="transition", help="add transition record")
 
 
 class HTTPSClientAuthHandler(urllib.request.HTTPSHandler):
@@ -147,6 +157,79 @@ def writePileupDocs(mspileupUrl, puDocs, handler, logger):
                 logger.critical(msg)
 
 
+def addTransitionRecords(handler, url, userDN, logger, dryrun):
+    """
+    Helper function to add and update in place transition record for existing MSPileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param userDN: string representing user DN
+    :param logger: logger to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :return: nothing
+    """
+    if not userDN:
+        raise Exception("No userDN provided")
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+    recordsToUpdate = []
+    for rec in puDocs:
+        if not rec.get("transition"):
+            tranRecord = {'containerFraction': 1.0,
+                          'customDID': rec['pileupName'],
+                          'updateTime': gmtimeSeconds(),
+                          'DN': userDN}
+            rec['transition'] = [tranRecord]
+            logger.info("New pileup document is: %s", rec)
+            recordsToUpdate.append(rec)
+
+    if dryrun:
+        logger.info("%s documents needs to be updated but are not written due to --dry-run option\n", len(recordsToUpdate))
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, recordsToUpdate, handler, logger)
+        logger.info("%s documents are uploaded to MSPileup\n", len(recordsToUpdate))
+
+
+def updatePileupRecords(handler, url, logger, dryrun, fin=""):
+    """
+    helper function to update pileup records
+    :param handler: HTTP handler to use
+    :param url: url to use (string)
+    :param logger: logger object to use
+    :param dryrun: option to run dry-run mode (boolean)
+    :param fin: input file name to read from (optional, string)
+    :return: nothing
+    """
+    if fin:
+        if os.path.exists(fin):
+            with open(fin, 'r', encoding='utf-8') as istream:
+                puOverride = json.load(istream)
+            logger.info("Pileup override file: %s has the following content: %s", fin, puOverride)
+        else:
+            logger.error("Provided %s file does not exist", fin)
+            os.exit(1)
+    else:
+        logger.info("No input file is provided to update pileup records")
+        os.exit(1)
+
+    puDocs = getPileupDocs(url, handler, logger)
+    logger.info("Found %d documents in MSPileup", len(puDocs))
+
+    # update the documents with the override content
+    for doc in puDocs:
+        doc.update(puOverride)
+        logger.info("New pileup document is: %s", doc)
+
+    if dryrun:
+        logger.info("Updated documents (dry-run option):")
+        for doc in puDocs:
+            logger.info(doc)
+    else:
+        # finally, update the pileup documents in the database
+        writePileupDocs(url, puDocs, handler, logger)
+        logger.info("documents are uploaded to MSPileup\n")
+
+
 def main():
     """Executes everything"""
     optmgr = OptionParser()
@@ -179,24 +262,10 @@ def main():
     # initialize HTTPS client handler
     handler = HTTPSClientAuthHandler(key, cert)
 
-    with open(opts.fin, 'r', encoding='utf-8') as istream:
-        puOverride = json.load(istream)
-    logger.info("Pileup override file: %s has the following content: %s", opts.fin, puOverride)
-
-    puDocs = getPileupDocs(opts.url, handler, logger)
-    logger.info("Found %d documents in MSPileup", len(puDocs))
-
-    # update the documents with the override content
-    for doc in puDocs:
-        doc.update(puOverride)
-        logger.info("New pileup document is: %s", doc)
-
-    if opts.dryrun:
-        logger.info("documents are not written due to --dry-run option\n")
+    if opts.transition:
+        addTransitionRecords(handler, opts.url, opts.userDN, logger, opts.dryrun)
     else:
-        # finally, update the pileup documents in the database
-        writePileupDocs(opts.url, puDocs, handler, logger)
-        logger.info("documents are uploaded to MSPileup\n")
+        updatePileupRecords(handler, opts.url, logger, opts.dryrun, opts.fin)
 
 
 if __name__ == '__main__':

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc6}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.2rc5'
+__version__ = '2.3.2rc6'
 __all__ = []

{reqmgr2ms-output-2.3.2rc5 → reqmgr2ms-output-2.3.2rc6/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.2rc5
+Version: 2.3.2rc6
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-output-2.3.2rc5/bin/adhoc-scripts/adjustMongoDocs.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-"""
-File       : adjustMongoDocs.py
-Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
-Description: adjust documents in MongoDB with given schema
-The script loops over given db documents (from given db name
-and collection) and add to each document a new schema provided
-via input file name, e.g.
-
-# new schema addition:
-cat > /tmp/schema.json << EOF
-{"foo": 1}
-EOF
-
-# run script to update all documents in test.test database and add to them
-# a json structure presented in /tmp/schema.json file
-adjustMongoDocs.py --fin=/tmp/schema.json --dburi=localhost:8230 --dbname=test --dbcoll=test --verbose 1
-
-NOTE: the script can run from any location assuming it has proper db uri with
-credentials and availability of mongo db outside of its own cluster, but most
-likely it will be run on mongo db k8s cluster
-"""
-
-import argparse
-import json
-from pymongo import MongoClient
-
-
-class OptionParser(object):
-    "Helper class to handle script options"
-
-    def __init__(self):
-        "User based option parser"
-        self.parser = argparse.ArgumentParser(prog='PROG')
-        self.parser.add_argument("--fin", action="store",
-                                 dest="fin", default="", help="Input file with update fields")
-        self.parser.add_argument("--dburi", action="store",
-                                 dest="dburi", default="", help="MongoDB URI")
-        self.parser.add_argument("--dbname", action="store",
-                                 dest="dbname", default="", help="MongoDB database")
-        self.parser.add_argument("--dbcoll", action="store",
-                                 dest="dbcoll", default="", help="MongoDB collection")
-        self.parser.add_argument("--verbose", action="store",
-                                 dest="verbose", default=0, help="verbosity level")
-
-
-def adjust():
-    """
-    helper script to update documents in MongoDB with given db name/collection
-    and schema file (which should contain new addition, in form of JSON, to the
-    db documents)
-    """
-    optmgr = OptionParser()
-    opts = optmgr.parser.parse_args()
-    conn = MongoClient(host=opts.dburi)
-    dbname = opts.dbname
-    dbcoll = opts.dbcoll
-    verbose = opts.verbose
-    schema = {}
-    with open(opts.fin, 'r', encoding='utf-8') as istream:
-        schema = json.load(istream)
-    if schema:
-        if verbose:
-            print("read data from '%s', %s/%s" % (opts.dburi, dbname, dbcoll))
-        records = [r for r in conn[dbname][dbcoll].find()]
-        for doc in records:
-            if verbose:
-                print("update doc=%s with schema=%s" % (doc, schema))
-            conn[dbname][dbcoll].update_one(doc, {'$set': schema}, upsert=True)
-
-
-if __name__ == '__main__':
-    adjust()