wmglobalqueue 2.4.5.1__py3-none-any.whl

Utils/Timestamps.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""
+_Timestamps_ module designed to insert proper WM timing into WMCore FJR.
+Usage: python3 Timestamps.py --reportFile=$outputFile --wmJobStart=<sec> --wmJobEnd=<sec>
+where outputFile represents output FJR file, and wmJobStart/wmJobEnd represent
+start and end time of WM job, respectively.
+
+NOTE: this script should reside in Utils area of WMCore repository since it is required
+at run time on condor node where there is no WMCore installation and we only provide WMCore.zip
+archive, and worker node may not have unzip tool to extract this script.
+
+"""
+
+import getopt
+import logging
+import os
+import pickle
+import sys
+
+
+# script options
+options = {"reportFile=": "", "wmJobStart=": '', "wmJobEnd=": ''}
+
+
+def addTimestampMetrics(config, wmJobStart, wmJobEnd):
+    """
+    Adjust timestamp metrics of provied FJR data.
+
+    :param config: input FJR data
+    :param wmJobStart: start time of WM job in seconds
+    :param wmJobEnd: end time of WM job in seconds
+    """
+    wmTiming = config.section_('WMTiming')
+    wmTiming.WMJobStart = wmJobStart
+    wmTiming.WMJobEnd = wmJobEnd
+    wmTiming.WMTotalWallClockTime = wmJobEnd - wmJobStart
+    return config
+
+
+def main():
+    """
+    Main function of the script performs business logic:
+    - parse input arguments
+    - read provided input FJR
+    - adjust timestamp metrics of FJR data
+    - write out FJR
+    """
+    try:
+        opts, _ = getopt.getopt(sys.argv[1:], "", list(options.keys()))
+    except getopt.GetoptError as ex:
+        msg = "Error processing commandline args:\n"
+        msg += str(ex)
+        logging.error(msg)
+        sys.exit(1)
+
+    for opt, arg in opts:
+        if opt == "--reportFile":
+            reportFile = arg
+        if opt == "--wmJobStart":
+            wmJobStart = float(arg)
+        if opt == "--wmJobEnd":
+            wmJobEnd = float(arg)
+
+    # read content of given FJR report file
+    data = {}
+    with open(reportFile, 'rb') as istream:
+        data = pickle.load(istream)
+
+    # adjust FJR data with provided metrics
+    data = addTimestampMetrics(data, wmJobStart, wmJobEnd)
+
+    # write content of FJR back to report file
+    reportOutFile = reportFile + ".new"
+    msg = f"Adding wmJobTime metric to {reportFile}"
+    with open(reportOutFile, 'wb') as ostream:
+        logging.info(msg)
+        pickle.dump(data, ostream)
+
+    # if we successfully wrote reportOutFile we can swap it with input one
+    sizeStatus = os.path.getsize(reportOutFile) >= os.path.getsize(reportFile)
+    if os.path.isfile(reportOutFile) and sizeStatus:
+        os.rename(reportOutFile, reportFile)
+
+
+if __name__ == '__main__':
+    main()

Utils/TokenManager.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#pylint: disable=W0212
+"""
+File       : TokenManager.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: IAM token manager
+             by default it relies on the following environment/parameters:
+             - IAM_TOKEN is either file name or actual token value
+             - https://cms-auth.web.cern.ch/jwk is default CMS IAM provider
+             - https://wlcg.cern.ch/jwt/v1/any is default for audience parameter
+"""
+
+# system modules
+import os
+import ssl
+import time
+import logging
+import traceback
+
+# third part library
+try:
+    import jwt
+except ImportError:
+    traceback.print_exc()
+    jwt = None
+
+from Utils.Utilities import encodeUnicodeToBytes
+
+# prevent "SSL: CERTIFICATE_VERIFY_FAILED" error
+# this will cause pylint warning W0212, therefore we ignore it above
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+def readToken(name=None):
+    """
+    Read IAM token either from environment or file name
+    :param name: ether file name containing token or environment name which hold the token value.
+    If not provided it will be assumed to read token from IAM_TOKEN environment.
+    :return: token or None
+    """
+    if name and os.path.exists(name):
+        token = None
+        with open(name, 'r', encoding='utf-8') as istream:
+            token = istream.read()
+        return token
+    if name:
+        return os.environ.get(name)
+    return os.environ.get("IAM_TOKEN")
+
+
+def tokenData(token, url="https://cms-auth.web.cern.ch/jwk", audUrl="https://wlcg.cern.ch/jwt/v1/any"):
+    """
+    inspect and extract token data
+    :param token: token string
+    :param url: IAM provider URL
+    :param audUrl: audience string
+    """
+    if not token or not jwt:
+        return {}
+    if isinstance(token, str):
+        token = encodeUnicodeToBytes(token)
+    jwksClient = jwt.PyJWKClient(url)
+    signingKey = jwksClient.get_signing_key_from_jwt(token)
+    key = signingKey.key
+    headers = jwt.get_unverified_header(token)
+    alg = headers.get('alg', 'RS256')
+    data = jwt.decode(
+        token,
+        key,
+        algorithms=[alg],
+        audience=audUrl,
+        options={"verify_exp": True},
+        )
+    return data
+
+
+def isValidToken(token):
+    """
+    check if given token is valid or not
+
+    :param token: token string
+    :return: true or false
+    """
+    tokenDict = {}
+    tokenDict = tokenData(token)
+    exp = tokenDict.get('exp', 0)  # expire, seconds since epoch
+    if not exp or exp < time.time():
+        return False
+    return True
+
+
+class TokenManager():
+    """
+    TokenManager class handles IAM tokens
+    """
+
+    def __init__(self,
+                 name=None,
+                 url="https://cms-auth.web.cern.ch/jwk",
+                 audUrl="https://wlcg.cern.ch/jwt/v1/any",
+                 logger=None):
+        """
+        Token manager reads IAM tokens either from file or env.
+        It caches token along with expiration timestamp.
+        By default the env variable to use is IAM_TOKEN.
+        :param name: string representing either file or env where we should read token from
+        :param url: IAM provider URL
+        :param audUrl: audience string
+        :param logger: logger object or none to use default one
+        """
+        self.name = name
+        self.url = url
+        self.audUrl = audUrl
+        self.expire = 0
+        self.token = None
+        self.logger = logger if logger else logging.getLogger()
+        try:
+            self.token = self.getToken()
+        except Exception as exc:
+            self.logger.exception("Failed to get token. Details: %s", str(exc))
+
+    def getToken(self):
+        """
+        Return valid token and sets its expire timestamp
+        """
+        if not self.token or not isValidToken(self.token):
+            self.token = readToken(self.name)
+        tokenDict = {}
+        try:
+            tokenDict = tokenData(self.token, url=self.url, audUrl=self.audUrl)
+            self.logger.debug(tokenDict)
+        except Exception as exc:
+            self.logger.exception(str(exc))
+            raise
+        self.expire = tokenDict.get('exp', 0)
+        return self.token
+
+    def getLifetime(self):
+        """
+        Return reamaining lifetime of existing token
+        """
+        return self.expire - int(time.time())

Utils/Tracing.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+"""
+Stuff to help with debugging from https://pymotw.com/3/sys/tracing.html
+
+Use like this:
+
+        import sys
+        from Utils.Tracing import trace_calls_and_returns, dont_trace
+
+        sys.settrace(trace_calls_and_returns)
+        # Do the things you want to trace
+        sys.settrace(dont_trace)
+
+"""
+
+
+def trace_calls(frame, event, arg, to_be_traced=None):
+    if event != 'call':
+        return
+    co = frame.f_code
+    func_name = co.co_name
+    if func_name == 'write':
+        return  # Ignore write() calls from printing
+    line_no = frame.f_lineno
+    filename = co.co_filename
+    print('* Call to {} on line {} of {}'.format(
+        func_name, line_no, filename))
+    if func_name in to_be_traced:
+        # Trace into this function
+        return trace_lines
+    return
+
+
+def trace_calls_and_returns(frame, event, arg):
+    co = frame.f_code
+    func_name = co.co_name
+    if func_name == 'write':
+        return  # Ignore write() calls from printing
+    line_no = frame.f_lineno
+    filename = co.co_filename
+    if event == 'call':
+        print('Call to {} on line {} of {}'.format(func_name, line_no, filename))
+        return trace_calls_and_returns
+    elif event == 'return':
+        print(' {} => {} ({})'.format(func_name, arg, filename))
+    return
+
+
+def trace_lines(frame, event, arg):
+    if event != 'line':
+        return
+    co = frame.f_code
+    func_name = co.co_name
+    line_no = frame.f_lineno
+    print('*  {} line {}'.format(func_name, line_no))
+
+
+def dont_trace(frame, event, arg):
+    return

Utils/TwPrint.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+"""
+File       : TwPrint.py
+
+Description:
+
+A simple textwrap based printer for nested dictionaries.
+
+"""
+from textwrap import TextWrapper
+from collections import OrderedDict
+
+
+def twClosure(replace_whitespace=False,
+              break_long_words=False,
+              maxWidth=120,
+              maxLength=-1,
+              maxDepth=-1,
+              initial_indent=''):
+    """
+    Deals with indentation of dictionaries with very long key, value pairs.
+    replace_whitespace: Replace each whitespace character with a single space.
+    break_long_words: If True words longer than width will be broken.
+    width: The maximum length of wrapped lines.
+    initial_indent: String that will be prepended to the first line of the output
+
+    Wraps all strings for both keys and values to 120 chars.
+    Uses 4 spaces indentation for both keys and values.
+    Nested dictionaries and lists go to next line.
+    """
+    twr = TextWrapper(replace_whitespace=replace_whitespace,
+                      break_long_words=break_long_words,
+                      width=maxWidth,
+                      initial_indent=initial_indent)
+
+    def twEnclosed(obj, ind='', depthReached=0, reCall=False):
+        """
+        The inner function of the closure
+        ind: Initial indentation for the single output string
+        reCall: Flag to indicate a recursive call (should not be used outside)
+        """
+        output = ''
+        if isinstance(obj, dict):
+            obj = OrderedDict(sorted(list(obj.items()),
+                                     key=lambda t: t[0],
+                                     reverse=False))
+            if reCall:
+                output += '\n'
+            ind += '    '
+            depthReached += 1
+            lengthReached = 0
+            for key, value in list(obj.items()):
+                lengthReached += 1
+                if lengthReached > maxLength and maxLength >= 0:
+                    output += "%s...\n" % ind
+                    break
+                if depthReached <= maxDepth or maxDepth < 0:
+                    output += "%s%s: %s" % (ind,
+                                            ''.join(twr.wrap(key)),
+                                            twEnclosed(value, ind, depthReached=depthReached, reCall=True))
+
+        elif isinstance(obj, (list, set)):
+            if reCall:
+                output += '\n'
+            ind += '    '
+            lengthReached = 0
+            for value in obj:
+                lengthReached += 1
+                if lengthReached > maxLength and maxLength >= 0:
+                    output += "%s...\n" % ind
+                    break
+                if depthReached <= maxDepth or maxDepth < 0:
+                    output += "%s%s" % (ind, twEnclosed(value, ind, depthReached=depthReached, reCall=True))
+        else:
+            output += "%s\n" % str(obj)  # join(twr.wrap(str(obj)))
+        return output
+
+    return twEnclosed
+
+
+def twPrint(obj, maxWidth=120, maxLength=-1, maxDepth=-1):
+    """
+    A simple caller of twClosure (see docstring for twClosure)
+    """
+    twPrinter = twClosure(maxWidth=maxWidth,
+                          maxLength=maxLength,
+                          maxDepth=maxDepth)
+    print(twPrinter(obj))
+
+
+def twFormat(obj, maxWidth=120, maxLength=-1, maxDepth=-1):
+    """
+    A simple caller of twClosure (see docstring for twClosure)
+    """
+    twFormatter = twClosure(maxWidth=maxWidth,
+                            maxLength=maxLength,
+                            maxDepth=maxDepth)
+    return twFormatter(obj)

Utils/Utilities.py

@@ -0,0 +1,318 @@
+#! /usr/bin/env python
+
+from builtins import str, bytes
+
+import subprocess
+import os
+import re
+import zlib
+import base64
+import sys
+from types import ModuleType, FunctionType
+from gc import get_referents
+
+import xml.etree.ElementTree as ET
+
+def extractFromXML(xmlFile, xmlElement):
+    tree = ET.parse(xmlFile)
+    root = tree.getroot()
+    element = root.find(f".//{xmlElement}")
+    if element is not None:
+        return element.get("Value")
+    return None
+
+def lowerCmsHeaders(headers):
+    """
+    Lower CMS headers in provided header's dict. The WMCore Authentication
+    code check only cms headers in lower case, e.g. cms-xxx-yyy.
+    """
+    lheaders = {}
+    for hkey, hval in list(headers.items()): # perform lower-case
+        # lower header keys since we check lower-case in headers
+        if hkey.startswith('Cms-') or hkey.startswith('CMS-'):
+            lheaders[hkey.lower()] = hval
+        else:
+            lheaders[hkey] = hval
+    return lheaders
+
+def makeList(stringList):
+    """
+    _makeList_
+
+    Make a python list out of a comma separated list of strings,
+    throws a ValueError if the input is not well formed.
+    If the stringList is already of type list, then return it untouched.
+    """
+    if isinstance(stringList, list):
+        return stringList
+    if isinstance(stringList, str):
+        toks = stringList.lstrip(' [').rstrip(' ]').split(',')
+        if toks == ['']:
+            return []
+        return [str(tok.strip(' \'"')) for tok in toks]
+    raise ValueError("Can't convert to list %s" % stringList)
+
+
+def makeNonEmptyList(stringList):
+    """
+    _makeNonEmptyList_
+
+    Given a string or a list of strings, return a non empty list of strings.
+    Throws an exception in case the final list is empty or input data is not
+    a string or a python list
+    """
+    finalList = makeList(stringList)
+    if not finalList:
+        raise ValueError("Input data cannot be an empty list %s" % stringList)
+    return finalList
+
+
+def strToBool(string):
+    """
+    Try to convert different variations of True or False (including a string
+    type object) to a boolean value.
+    In short:
+     * True gets mapped from: True, "True", "true", "TRUE".
+     * False gets mapped from: False, "False", "false", "FALSE"
+     * anything else will fail
+    :param string: expects a boolean or a string, but it could be anything else
+    :return: a boolean value, or raise an exception if value passed in is not supported
+    """
+    if string is False or string is True:
+        return string
+    elif string in ["True", "true", "TRUE"]:
+        return True
+    elif string in ["False", "false", "FALSE"]:
+        return False
+    raise ValueError("Can't convert to bool: %s" % string)
+
+
+def safeStr(string):
+    """
+    _safeStr_
+
+    Cast simple data (int, float, basestring) to string.
+    """
+    if not isinstance(string, (tuple, list, set, dict)):
+        return str(string)
+    raise ValueError("We're not supposed to convert %s to string." % string)
+
+
+def diskUse():
+    """
+    This returns the % use of each disk partition
+    """
+    diskPercent = []
+    df = subprocess.Popen(["df", "-klP"], stdout=subprocess.PIPE)
+    output = df.communicate()[0]
+    output = decodeBytesToUnicode(output).split("\n")
+    for x in output:
+        split = x.split()
+        if split != [] and split[0] != 'Filesystem':
+            diskPercent.append({'filesystem': split[0],
+                                'mounted':    split[5], 
+                                'percent':    split[4]})
+
+    return diskPercent
+
+
+def numberCouchProcess():
+    """
+    This returns the number of couch process
+    """
+    ps = subprocess.Popen(["ps", "-ef"], stdout=subprocess.PIPE)
+    process = ps.communicate()[0]
+    process = decodeBytesToUnicode(process).count('couchjs')
+
+    return process
+
+
+def rootUrlJoin(base, extend):
+    """
+    Adds a path element to the path within a ROOT url
+    """
+    if base:
+        match = re.match("^root://([^/]+)/(.+)", base)
+        if match:
+            host = match.group(1)
+            path = match.group(2)
+            newpath = os.path.join(path, extend)
+            newurl = "root://%s/%s" % (host, newpath)
+            return newurl
+    return None
+
+
+def zipEncodeStr(message, maxLen=5120, compressLevel=9, steps=100, truncateIndicator=" (...)"):
+    """
+    _zipEncodeStr_
+    Utility to zip a string and encode it.
+    If zipped encoded length is greater than maxLen,
+    truncate message until zip/encoded version
+    is within the limits allowed.
+    """
+    message = encodeUnicodeToBytes(message)
+    encodedStr = zlib.compress(message, compressLevel)
+    encodedStr = base64.b64encode(encodedStr)
+    if len(encodedStr) < maxLen or maxLen == -1:
+        return encodedStr
+
+    compressRate = 1. * len(encodedStr) / len(base64.b64encode(message))
+
+    # Estimate new length for message zip/encoded version
+    # to be less than maxLen.
+    # Also, append truncate indicator to message.
+    truncateIndicator = encodeUnicodeToBytes(truncateIndicator)
+    strLen = int((maxLen - len(truncateIndicator)) / compressRate)
+    message = message[:strLen] + truncateIndicator
+
+    encodedStr = zipEncodeStr(message, maxLen=-1)
+
+    # If new length is not short enough, truncate
+    # recursively by steps
+    while len(encodedStr) > maxLen:
+        message = message[:-steps - len(truncateIndicator)] + truncateIndicator
+        encodedStr = zipEncodeStr(message, maxLen=-1)
+
+    return encodedStr
+
+
+def getSize(obj):
+    """
+    _getSize_
+
+    Function to traverse an object and calculate its total size in bytes
+    :param obj: a python object
+    :return: an integer representing the total size of the object
+
+    Code extracted from Stack Overflow:
+    https://stackoverflow.com/questions/449560/how-do-i-determine-the-size-of-an-object-in-python
+    """
+    # Custom objects know their class.
+    # Function objects seem to know way too much, including modules.
+    # Exclude modules as well.
+    BLACKLIST = type, ModuleType, FunctionType
+
+    if isinstance(obj, BLACKLIST):
+        raise TypeError('getSize() does not take argument of type: '+ str(type(obj)))
+    seen_ids = set()
+    size = 0
+    objects = [obj]
+    while objects:
+        need_referents = []
+        for obj in objects:
+            if not isinstance(obj, BLACKLIST) and id(obj) not in seen_ids:
+                seen_ids.add(id(obj))
+                size += sys.getsizeof(obj)
+                need_referents.append(obj)
+        objects = get_referents(*need_referents)
+    return size
+
+
+def decodeBytesToUnicode(value, errors="strict"):
+    """
+    Accepts an input "value" of generic type.
+
+    If "value" is a string of type sequence of bytes (i.e. in py2 `str` or
+    `future.types.newbytes.newbytes`, in py3 `bytes`), then it is converted to
+    a sequence of unicode codepoints.
+
+    This function is useful for cleaning input data when using the
+    "unicode sandwich" approach, which involves converting bytes (i.e. strings
+    of type sequence of bytes) to unicode (i.e. strings of type sequence of
+    unicode codepoints, in py2 `unicode` or `future.types.newstr.newstr`,
+    in py3 `str` ) as soon as possible when recieving input data, and
+    converting unicode back to bytes as late as possible.
+    achtung!:
+    - converting unicode back to bytes is not covered by this function
+    - converting unicode back to bytes is not always necessary. when in doubt,
+      do not do it.
+    Reference: https://nedbatchelder.com/text/unipain.html
+
+    py2:
+    - "errors" can be: "strict", "ignore", "replace",
+    - ref: https://docs.python.org/2/howto/unicode.html#the-unicode-type
+    py3:
+    - "errors" can be: "strict", "ignore", "replace", "backslashreplace"
+    - ref: https://docs.python.org/3/howto/unicode.html#the-string-type
+    """
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors)
+    return value
+
+def decodeBytesToUnicodeConditional(value, errors="ignore", condition=True):
+    """
+    if *condition*, then call decodeBytesToUnicode(*value*, *errors*),
+    else return *value*
+    
+    This may be useful when we want to conditionally apply decodeBytesToUnicode,
+    maintaining brevity.
+
+    Parameters
+    ----------
+    value : any
+        passed to decodeBytesToUnicode
+    errors: str
+        passed to decodeBytesToUnicode
+    condition: boolean of object with attribute __bool__()
+        if True, then we run decodeBytesToUnicode. Usually PY2/PY3
+    """
+    if condition:
+        return decodeBytesToUnicode(value, errors)
+    return value
+
+def encodeUnicodeToBytes(value, errors="strict"):
+    """
+    Accepts an input "value" of generic type.
+
+    If "value" is a string of type sequence of unicode (i.e. in py2 `unicode` or
+    `future.types.newstr.newstr`, in py3 `str`), then it is converted to
+    a sequence of bytes.
+
+    This function is useful for encoding output data when using the
+    "unicode sandwich" approach, which involves converting unicode (i.e. strings
+    of type sequence of unicode codepoints) to bytes (i.e. strings of type
+    sequence of bytes, in py2 `str` or `future.types.newbytes.newbytes`,
+    in py3 `bytes`) as late as possible when passing a string to a third-party
+    function that only accepts bytes as input (pycurl's curl.setop is an
+    example).
+    py2:
+    - "errors" can be: "strict", "ignore", "replace", "xmlcharrefreplace"
+    - ref: https://docs.python.org/2/howto/unicode.html#the-unicode-type
+    py3:
+    - "errors" can be: "strict", "ignore", "replace", "backslashreplace", 
+      "xmlcharrefreplace", "namereplace"
+    - ref: https://docs.python.org/3/howto/unicode.html#the-string-type
+    """
+    if isinstance(value, str):
+        return value.encode("utf-8", errors)
+    return value
+
+def encodeUnicodeToBytesConditional(value, errors="ignore", condition=True):
+    """
+    if *condition*, then call encodeUnicodeToBytes(*value*, *errors*),
+    else return *value*
+    
+    This may be useful when we want to conditionally apply encodeUnicodeToBytes,
+    maintaining brevity.
+
+    Parameters
+    ----------
+    value : any
+        passed to encodeUnicodeToBytes
+    errors: str
+        passed to encodeUnicodeToBytes
+    condition: boolean of object with attribute __bool__()
+        if True, then we run encodeUnicodeToBytes. Usually PY2/PY3
+    """
+    if condition:
+        return encodeUnicodeToBytes(value, errors)
+    return value
+
+def normalize_spaces(text):
+    """
+    Helper function to remove any number of empty spaces within given text and replace
+    then with single space.
+    :param text: string
+    :return: normalized string
+    """
+    return re.sub(r'\s+', ' ', text).strip()