wmglobalqueue 2.4.5.1__py3-none-any.whl

Utils/CPMetrics.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+File       : CPMetrics.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide functions to flatten cherrypy stats
+and provide them in Prometheus format for scraping.
+
+Below we present three different outputs:
+    - CherryPy stats default output
+    - flatten structure of stats
+    - prometheus output for stats
+
+### Defautl CherryPy metrics output
+
+{
+    "Cheroot HTTPServer 4388603856": {
+    "Accepts": 0,
+    "Accepts/sec": 0.0,
+    "Bind Address": "('127.0.0.1', 8080)",
+    "Bytes Read": -1,
+    "Bytes Written": -1,
+    "Enabled": false,
+    "Queue": 0,
+    "Read Throughput": -1,
+    "Requests": -1,
+    "Run time": -1,
+    "Socket Errors": 0,
+    "Threads": 20,
+    "Threads Idle": 19,
+    "Work Time": -1,
+    "Worker Threads": {
+        "CP Server Thread-10": {
+        "Bytes Read": 0,
+        "Bytes Written": 0,
+        "Read Throughput": 0.0,
+        "Requests": 0,
+        "Work Time": 0,
+        "Write Throughput": 0.0
+        },
+        .....
+    },
+    "Write Throughput": -1
+    },
+    "CherryPy Applications": {
+        "Bytes Read/Request": 0.0,
+        "Bytes Read/Second": 0.0,
+        "Bytes Written/Request": 0.0,
+        "Bytes Written/Second": 0.0,
+        "Current Requests": 0,
+        "Current Time": 1601039589.916887,
+        "Enabled": true,
+        "Requests": {},
+        "Requests/Second": 0.0,
+        "Server Version": "17.4.2",
+        "Start Time": 1601039576.541158,
+        "Total Bytes Read": 0,
+        "Total Bytes Written": 0,
+        "Total Requests": 0,
+        "Total Time": 0,
+        "Uptime": 13.375718116760254
+    }
+}
+
+### flatten structure via flattenStats function
+{
+"cherrypy_app_requests_second": 0.0,
+"cherrypy_app_uptime": 1.7454230785369873,
+"cherrypy_app_total_time": 0,
+"cherrypy_app_current_time": 1601040713.369842,
+"cherrypy_app_bytes_read_second": 0.0,
+"cherrypy_app_requests": {},
+"cherrypy_http_server_queue": 0,
+"cherrypy_app_bytes_written_second": 0.0,
+"cherrypy_app_total_requests": 0,
+"cherrypy_http_server_write_throughput": -1,
+"cherrypy_app_enabled": true,
+"cherrypy_app_start_time": 1601040711.624408,
+"cherrypy_http_server_threads": 20,
+"cherrypy_http_server_work_time": -1,
+"cherrypy_app_bytes_read_request": 0.0,
+"cherrypy_http_server_bytes_read": -1,
+"cherrypy_http_server_accepts_sec": 0.0,
+"cherrypy_server_worker_threads": [
+   {"thread_name": "cp_server_thread_3", "read_throughput": 0.0,
+   "work_time": 0, "write_throughput": 0.0, "bytes_written": 0,
+   "bytes_read": 0, "requests": 0}, ....],
+"cherrypy_http_server_run_time": -1,
+"cherrypy_http_server_bind_address": "('127.0.0.1', 8080)",
+"cherrypy_app_server_version": "17.4.2",
+"cherrypy_http_server_enabled": false,
+"cherrypy_http_server_socket_errors": 0,
+"cherrypy_app_bytes_written_request": 0.0,
+"cherrypy_app_current_requests": 0,
+"cherrypy_app_total_bytes_read": 0,
+"cherrypy_http_server_bytes_written": -1,
+"cherrypy_http_server_threads_idle": 19,
+"cherrypy_http_server_requests": -1,
+"cherrypy_app_total_bytes_written": 0,
+"cherrypy_http_server_read_throughput": -1,
+"cherrypy_http_server_accepts": 0}
+
+### prometheus exporter structure provided via promMetrics function
+
+# HELP cherrypy_app_requests_second
+# TYPE cherrypy_app_requests_second gauge
+cherrypy_app_requests_second 0.0
+# HELP cherrypy_app_uptime
+# TYPE cherrypy_app_uptime gauge
+cherrypy_app_uptime 2.03766894341
+# HELP cherrypy_app_total_time
+# TYPE cherrypy_app_total_time counter
+cherrypy_app_total_time 0
+# HELP cherrypy_app_current_time
+# TYPE cherrypy_app_current_time gauge
+cherrypy_app_current_time 1601043839.47
+# HELP cherrypy_app_bytes_read_second
+# TYPE cherrypy_app_bytes_read_second gauge
+cherrypy_app_bytes_read_second 0.0
+# HELP cherrypy_http_server_queue
+# TYPE cherrypy_http_server_queue counter
+cherrypy_http_server_queue 0
+# HELP cherrypy_app_bytes_written_second
+# TYPE cherrypy_app_bytes_written_second gauge
+cherrypy_app_bytes_written_second 0.0
+# HELP cherrypy_app_total_requests
+# TYPE cherrypy_app_total_requests counter
+cherrypy_app_total_requests 0
+# HELP cherrypy_http_server_write_throughput
+# TYPE cherrypy_http_server_write_throughput counter
+cherrypy_http_server_write_throughput -1
+# HELP cherrypy_server_worker_threads
+# TYPE cherrypy_server_worker_threads histogram
+cherrypy_server_worker_threads{thread_name="cp_server_thread_3",...} 1
+# HELP cherrypy_app_start_time
+# TYPE cherrypy_app_start_time gauge
+cherrypy_app_start_time 1601043837.44
+# HELP cherrypy_http_server_threads
+# TYPE cherrypy_http_server_threads counter
+cherrypy_http_server_threads 20
+# HELP cherrypy_http_server_work_time
+# TYPE cherrypy_http_server_work_time counter
+cherrypy_http_server_work_time -1
+# HELP cherrypy_app_bytes_read_request
+# TYPE cherrypy_app_bytes_read_request gauge
+cherrypy_app_bytes_read_request 0.0
+# HELP cherrypy_http_server_bytes_read
+# TYPE cherrypy_http_server_bytes_read counter
+cherrypy_http_server_bytes_read -1
+# HELP cherrypy_http_server_accepts_sec
+# TYPE cherrypy_http_server_accepts_sec gauge
+cherrypy_http_server_accepts_sec 0.0
+# HELP cherrypy_http_server_run_time
+# TYPE cherrypy_http_server_run_time counter
+cherrypy_http_server_run_time -1
+# HELP cherrypy_http_server_socket_errors
+# TYPE cherrypy_http_server_socket_errors counter
+cherrypy_http_server_socket_errors 0
+# HELP cherrypy_app_bytes_written_request
+# TYPE cherrypy_app_bytes_written_request gauge
+cherrypy_app_bytes_written_request 0.0
+# HELP cherrypy_app_current_requests
+# TYPE cherrypy_app_current_requests counter
+cherrypy_app_current_requests 0
+# HELP cherrypy_app_total_bytes_read
+# TYPE cherrypy_app_total_bytes_read counter
+cherrypy_app_total_bytes_read 0
+# HELP cherrypy_http_server_bytes_written
+# TYPE cherrypy_http_server_bytes_written counter
+cherrypy_http_server_bytes_written -1
+# HELP cherrypy_http_server_threads_idle
+# TYPE cherrypy_http_server_threads_idle counter
+cherrypy_http_server_threads_idle 19
+# HELP cherrypy_http_server_requests
+# TYPE cherrypy_http_server_requests counter
+cherrypy_http_server_requests -1
+# HELP cherrypy_app_total_bytes_written
+# TYPE cherrypy_app_total_bytes_written counter
+cherrypy_app_total_bytes_written 0
+# HELP cherrypy_http_server_read_throughput
+# TYPE cherrypy_http_server_read_throughput counter
+cherrypy_http_server_read_throughput -1
+# HELP cherrypy_http_server_accepts
+# TYPE cherrypy_http_server_accepts counter
+cherrypy_http_server_accepts 0
+"""
+
+# system modules
+import json
+
+# WMCore dependencies
+from Utils.Utilities import decodeBytesToUnicode
+
+
+def promMetrics(data, exporter):
+    """
+    Provide cherrypy stats prometheus metrics for given exporter name.
+    """
+    # exporter name should not contain dashes, see
+    # https://its.cern.ch/jira/browse/CMSMONIT-514
+    exporter = exporter.replace("-", "_")
+    metrics = flattenStats(data)
+    if isinstance(metrics, str):
+        metrics = json.loads(metrics)
+    # the following keys will be skipped
+    skip = [
+        'cherrypy_app_enabled',
+        'cherrypy_http_server_bind_address',
+        'cherrypy_app_requests',
+        'cherrypy_app_server_version',
+        'cherrypy_http_server_enabled']
+    # our prometheus data representation
+    pdata = ""
+    for key, val in list(metrics.items()):
+        if key in skip:
+            continue
+        # add exporter name as a prefix for each key
+        key = '{}_{}'.format(exporter, key)
+        mhelp = "# HELP {}\n".format(key)
+        if isinstance(val, list):
+            mtype = "# TYPE {} histogram\n".format(key)
+            pdata += mhelp
+            pdata += mtype
+            for wdict in val:
+                entries = []
+                for kkk, vvv in list(wdict.items()):
+                    entries.append('{}="{}"'.format(kkk, vvv))
+                entry = "{%s}" % ','.join(entries)
+                pdata += "{}{} 1\n".format(key, entry)
+        elif isinstance(val, (str, tuple)):
+            continue
+        else:
+            pdata += mhelp
+            if isinstance(val, int):
+                mtype = "# TYPE {} counter\n".format(key)
+            if isinstance(val, float):
+                mtype = "# TYPE {} gauge\n".format(key)
+            pdata += mtype
+            pdata += "{} {}\n".format(key, val)
+    return pdata
+
+
+def flattenStats(cpdata):
+    "Flatten cherrypy stats to make them suitable for monitoring"
+    if isinstance(cpdata, str) or isinstance(cpdata, bytes):
+        cpdata = json.loads(decodeBytesToUnicode(cpdata))
+    data = {}
+    for cpKey, cpVal in list(cpdata.items()):
+        if cpKey.lower().find('cherrypy') != -1:
+            for cpnKey, cpnVal in list(cpVal.items()):
+                nkey = 'cherrypy_app_%s' % cpnKey
+                nkey = nkey.lower().replace(" ", "_").replace("/", "_")
+                data[nkey] = cpnVal
+        if cpKey.lower().find('cheroot') != -1:
+            for cpnKey, cpnVal in list(cpVal.items()):
+                if cpnKey == 'Worker Threads':
+                    wdata = []
+                    for workerKey, threadValue in list(cpnVal.items()):
+                        workerKey = workerKey.lower().replace(" ", "_").replace("/", "_").replace("-", "_")
+                        threadValue['thread_name'] = workerKey
+                        nval = {}
+                        for tkey, tval in list(threadValue.items()):
+                            tkey = tkey.lower().replace(" ", "_").replace("/", "_")
+                            nval[tkey] = tval
+                        wdata.append(nval)
+                    data["cherrypy_server_worker_threads"] = wdata
+                else:
+                    nkey = 'cherrypy_http_server_%s' % cpnKey
+                    nkey = nkey.lower().replace(" ", "_").replace("/", "_")
+                    data[nkey] = cpnVal
+    return data

Utils/CertTools.py

@@ -0,0 +1,100 @@
+"""
+Module to deal with user certificates and CAs
+"""
+from builtins import str
+import os
+
+
+def ckey():
+    "Return user CA key either from proxy or userkey.pem"
+    pair = getKeyCertFromEnv()
+    return pair[0]
+
+
+def cert():
+    "Return user CA cert either from proxy or usercert.pem"
+    pair = getKeyCertFromEnv()
+    return pair[1]
+
+
+def getKeyCertFromEnv():
+    """
+    gets key and certificate from environment variables
+    If no env variable is set return None, None for key, cert tuple
+
+    First preference to HOST Certificate, This is how it set in Tier0
+
+    """
+    envPairs = [('X509_HOST_KEY', 'X509_HOST_CERT'),  # First preference to HOST Certificate,
+                ('X509_USER_PROXY', 'X509_USER_PROXY'),  # Second preference to User Proxy, very common
+                ('X509_USER_KEY', 'X509_USER_CERT')]  # Third preference to User Cert/Proxy combinition
+
+    for keyEnv, certEnv in envPairs:
+        localKey = os.environ.get(keyEnv)
+        localCert = os.environ.get(certEnv)
+        if localKey and localCert and os.path.exists(localKey) and os.path.exists(localCert):
+            # if it is found in env return key, cert
+            return localKey, localCert
+
+    # TODO: only in linux, unix case, add other os case
+    # look for proxy at default location /tmp/x509up_u$uid
+    localKey = localCert = '/tmp/x509up_u' + str(os.getuid())
+    if os.path.exists(localKey):
+        return localKey, localCert
+
+    # Finary look for globaus location
+    if (os.environ.get('HOME') and
+        os.path.exists(os.environ['HOME'] + '/.globus/usercert.pem')  and
+        os.path.exists(os.environ['HOME'] + '/.globus/userkey.pem')):
+
+        localKey = os.environ['HOME'] + '/.globus/userkey.pem'
+        localCert = os.environ['HOME'] + '/.globus/usercert.pem'
+        return localKey, localCert
+    # couldn't find the key, cert files
+    return None, None
+
+
+def getCAPathFromEnv():
+    """
+    _getCAPathFromEnv_
+
+    Return the path of the CA certificates. The check is loose in the pycurl_manager:
+    is capath == None then the server identity is not verified. To enable this check
+    you need to set either the X509_CERT_DIR variable or the cacert key of the request.
+    """
+    return os.environ.get("X509_CERT_DIR")
+
+
+def caBundle(caPath="/etc/grid-security/certificates"):
+    """
+    Load all PEM certificates from the given caPath and write them as single CA bundle PEM.
+
+    :param caPath: Path to directory containing .pem certificate files.
+    :return: A single string containing all concatenated PEM pemCertificates which may be
+    written to a caBundleFile if necessary (used by requests library)
+    """
+    if not os.path.isdir(caPath):
+        raise ValueError(f"Invalid caPath: {caPath} is not a directory")
+
+    pemCertificates = []
+
+    for fileName in sorted(os.listdir(caPath)):
+        filePath = os.path.join(caPath, fileName)
+
+        # Only consider readable files that look like PEM certificates
+        if not os.path.isfile(filePath):
+            continue
+        if not fileName.endswith(".pem"):
+            continue
+
+        try:
+            with open(filePath, "r", encoding="utf-8") as istream:
+                certData = istream.read()
+                if "BEGIN CERTIFICATE" in certData:
+                    pemCertificates.append(certData)
+        except Exception as e:
+            print(f"Warning: Could not read {filePath}: {e}")
+
+    if len(pemCertificates) == 0:
+        raise ValueError(f"No PEM files found in {caPath}")
+    return "\n".join(pemCertificates)

Utils/EmailAlert.py

@@ -0,0 +1,50 @@
+"""
+EmailAlert - send alerts via email
+
+NOTICE:
+This class does not work from kubernetes pods. The AlertManagerAPI class should be used for alerting from k8s.
+More details at: https://github.com/dmwm/WMCore/issues/10234
+"""
+
+
+from builtins import str, object
+import smtplib
+import logging
+
+
+class EmailAlert(object):
+    """
+    A simple class to send alerts via email
+    """
+
+    EMAIL_HEADER = "From: %s\r\nSubject: %s\r\nTo: %s\r\n\r\n"
+
+    def __init__(self, configDict):
+        self.serverName = configDict.get("smtpServer", "localhost")
+        self.fromAddr = configDict.get("fromAddr", "noreply@cern.ch")
+        self.toAddr = configDict.get("toAddr", "cms-service-production-admins@cern.ch")
+        if not isinstance(self.toAddr, (list, set)):
+            self.toAddr = [self.toAddr]
+
+    def send(self, subject, message):
+        """
+        Send an email
+        :param subject: Email subject
+        :param message: Email body
+        """
+        msg = self.EMAIL_HEADER % (self.fromAddr, subject, ", ".join(self.toAddr))
+        msg += message
+
+        try:
+            smtp = smtplib.SMTP(self.serverName)
+            smtp.sendmail(self.fromAddr, self.toAddr, msg)
+        except Exception as ex:
+            logging.exception("Error sending alert email.\nDetails: %s", str(ex))
+
+        try:
+            # clean up smtp connection
+            smtp.quit()
+        except UnboundLocalError:
+            # it means our client failed connecting to the SMTP server
+            pass
+

Utils/ExtendedUnitTestCase.py

@@ -0,0 +1,62 @@
+#! /usr/bin/env python
+"""
+Unit testing base class with our extensions
+"""
+
+import copy
+import unittest
+
+
+class ExtendedUnitTestCase(unittest.TestCase):
+    """
+    Class that can be imported to switch to 'mock'ed versions of
+    services.
+    """
+
+    def assertContentsEqual(self, expected_obj, actual_obj, msg=None):
+        """
+        A nested object comparison without regard for the ordering of contents. It asserts that
+        expected_obj and actual_obj contain the same elements and that their sub-elements are the same.
+        However, all sequences are allowed to contain the same elements, but in different orders.
+        """
+
+        def traverse_dict(dictionary):
+            for key, value in list(dictionary.items()):
+                if isinstance(value, dict):
+                    traverse_dict(value)
+                elif isinstance(value, list):
+                    traverse_list(value)
+            return
+
+        def get_dict_sortkey(x):
+            if isinstance(x, dict):
+                return list(x.keys())
+            else:
+                return x
+
+        def traverse_list(theList):
+            for value in theList:
+                if isinstance(value, dict):
+                    traverse_dict(value)
+                elif isinstance(value, list):
+                    traverse_list(value)
+            theList.sort(key=get_dict_sortkey)
+            return
+
+        if not isinstance(expected_obj, type(actual_obj)):
+            self.fail(msg="The two objects are different type and cannot be compared: %s and %s" % (
+            type(expected_obj), type(actual_obj)))
+
+        expected = copy.deepcopy(expected_obj)
+        actual = copy.deepcopy(actual_obj)
+
+        if isinstance(expected, dict):
+            traverse_dict(expected)
+            traverse_dict(actual)
+        elif isinstance(expected, list):
+            traverse_list(expected)
+            traverse_list(actual)
+        else:
+            self.fail(msg="The two objects are different type (%s) and cannot be compared." % type(expected_obj))
+
+        return self.assertEqual(expected, actual)

Utils/FileTools.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+"""
+Utilities related to file handling
+"""
+
+
+import io
+import os
+import glob
+import stat
+import subprocess
+import time
+import zlib
+import logging
+
+from Utils.Utilities import decodeBytesToUnicode
+
+
+def findFiles(path, pat):
+    """
+    Find files within given path and matching given pattern.
+    :param path: starting directory path (string)
+    :param pat: match pattern (string), e.g. *.py or name of the file
+    :return: matched file names
+    """
+    files = []
+    for idir, _, _ in os.walk(path):
+        files.extend(glob.glob(os.path.join(idir, pat)))
+    return files
+
+
+def tarMode(tfile, opMode):
+    """
+    Extract proper mode of operation for given tar file. For instance,
+    if op='r' and tfile name is file.tar.gz we should get 'r:gz',
+    while if tfile name is file.tar.bz2 we should get 'r':bz2', while
+    if tfile name is file.tar we should get 'r', etc.
+    :param opMode: mode of operation (string), e.g. 'r', or 'w'
+    :param tfile: sandbox tar file name (string)
+    :return: mode of operation
+    """
+    ext = tfile.split(".")[-1]
+    if ext == 'tar':
+        return opMode
+    mode = opMode + ':' + ext
+    return mode
+
+
+def calculateChecksums(filename):
+    """
+    _calculateChecksums_
+
+    Get the adler32 and crc32 checksums of a file. Return None on error
+
+    Process line by line and adjust for known signed vs. unsigned issues
+      http://docs.python.org/library/zlib.html
+
+    The cksum UNIX command line tool implements a CRC32 checksum that is
+    different than any of the python algorithms, therefore open cksum
+    in a subprocess and feed it the same chunks of data that are used
+    to calculate the adler32 checksum.
+
+    """
+    adler32Checksum = 1  # adler32 of an empty string
+    cksumProcess = subprocess.Popen("cksum", stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    # the lambda basically creates an iterator function with zero
+    # arguments that steps through the file in 4096 byte chunks
+    with open(filename, 'rb') as f:
+        for chunk in iter((lambda: f.read(4096)), b''):
+            adler32Checksum = zlib.adler32(chunk, adler32Checksum)
+            cksumProcess.stdin.write(chunk)
+
+    cksumProcess.stdin.close()
+    cksumProcess.wait()
+
+    cksumStdout = cksumProcess.stdout.read().split()
+    cksumProcess.stdout.close()
+
+    # consistency check on the cksum output
+    filesize = os.stat(filename)[stat.ST_SIZE]
+    if len(cksumStdout) != 2 or int(cksumStdout[1]) != filesize:
+        raise RuntimeError("Something went wrong with the cksum calculation !")
+
+    cksumStdout[0] = decodeBytesToUnicode(cksumStdout[0])
+    return (format(adler32Checksum & 0xffffffff, '08x'), cksumStdout[0])
+
+
+def tail(filename, nLines=20):
+    """
+    _tail_
+
+    A version of tail
+    Adapted from code on http://stackoverflow.com/questions/136168/get-last-n-lines-of-a-file-with-python-similar-to-tail
+    """
+    assert nLines >= 0
+    pos, lines = nLines + 1, []
+
+    # make sure only valid utf8 encoded chars will be passed along
+    with io.open(filename, 'r', encoding='utf8', errors='ignore') as f:
+        while len(lines) <= nLines:
+            try:
+                f.seek(-pos, 2)
+            except IOError:
+                f.seek(0)
+                break
+            finally:
+                lines = list(f)
+            pos *= 2
+
+    text = "".join(lines[-nLines:])
+
+    return text
+
+
+def getFileInfo(filename):
+    """
+    _getFileInfo_
+
+    Return file info in a friendly format
+    """
+
+    filestats = os.stat(filename)
+
+    fileInfo = {'Name': filename,
+                'Size': filestats[stat.ST_SIZE],
+                'LastModification': time.strftime("%m/%d/%Y %I:%M:%S %p", time.localtime(filestats[stat.ST_MTIME])),
+                'LastAccess': time.strftime("%m/%d/%Y %I:%M:%S %p", time.localtime(filestats[stat.ST_ATIME]))}
+    return fileInfo
+
+
+def findMagicStr(filename, matchString):
+    """
+    _findMagicStr_
+
+    Parse a log file looking for a pattern string
+    """
+    with io.open(filename, 'r', encoding='utf8', errors='ignore') as logfile:
+        # TODO: can we avoid reading the whole file
+        for line in logfile:
+            if matchString in line:
+                yield line
+
+
+def getFullPath(name, envPath="PATH"):
+    """
+    :param name: file name
+    :param envPath: any environment variable specified for path (PATH, PYTHONPATH, etc)
+    :return: full path if it is under PATH env
+    """
+    for path in os.getenv(envPath).split(os.path.pathsep):
+        fullPath = os.path.join(path, name)
+        if os.path.exists(fullPath):
+            return fullPath
+    return None
+
+
+def loadEnvFile(wmaEnvFilePath, logger=None):
+    """
+    _loadEnvFile_
+    A simple function to load an additional bash env file into the current script
+    runtime environment
+    :param wmaEnvFilePath: The path to the environment file to be loaded
+    :return:               True if the script has loaded successfully, False otherwise.
+    """
+    if not logger:
+        logger = logging.getLogger()
+    subProc = subprocess.run(['bash', '-c', f'source {wmaEnvFilePath} && python -c "import os; print(repr(os.environ.copy()))" '],
+                                   capture_output=True, check=False)
+    if subProc.returncode == 0:
+        newEnv = eval(subProc.stdout)
+        os.environ.update(newEnv)
+        if subProc.stderr:
+            logger.warning("Environment file: %s loaded with errors:", wmaEnvFilePath)
+            logger.warning(subProc.stderr.decode())
+        else:
+            logger.info("Environment file: %s loaded successfully", wmaEnvFilePath)
+        return True
+    else:
+        logger.error("Failed to load environment file: %s", wmaEnvFilePath)
+        logger.error(subProc.stderr.decode())
+        return False