wmglobalqueue 2.4.2rc7__tar.gz → 2.4.2rc8__tar.gz

{wmglobalqueue-2.4.2rc7/src/python/wmglobalqueue.egg-info → wmglobalqueue-2.4.2rc8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wmglobalqueue
-Version: 2.4.2rc7
+Version: 2.4.2rc8
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group
 Maintainer-email: hn-cms-wmdevelopment@cern.ch

wmglobalqueue-2.4.2rc8/bin/adhoc-scripts/simulateReplicationFailure.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : simulateReplicationFailure.py
+author     : valentin kuznetsov <vkuznet AT gmail DOT com>
+Description: helper script to simulate replication failure
+"""
+
+import os
+import sys
+import json
+import tempfile
+import time
+import requests
+
+# 3rd party modules
+from Utils.CertTools import ckey, cert, caBundle
+
+
+def wipeDb(couchUrl, dbName, auth=None, certTuple=None, caFileName=None):
+    """
+    wipe out couch database
+    :param couchUrl: couch url string
+    :param dbName: database name
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    deleteResp = requests.delete(f"{couchUrl}/{dbName}", auth=auth, cert=certTuple, verify=caFileName)
+
+    if deleteResp.status_code not in (200, 202, 404):
+        raise Exception(f"Failed to delete database: {deleteResp.text}")
+    print(f"Deleted database '{dbName}' (status {deleteResp.status_code})")
+
+def createDb(couchUrl, dbName, auth, certTuple=None, caFileName=None):
+    """
+    creates new couchdb database
+    :param couchUrl: couch url string
+    :param dbName: database name
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    r = requests.put(f"{couchUrl}/{dbName}", auth=auth, cert=certTuple, verify=caFileName)
+    if r.status_code not in (201, 412):  # 412 = already exists
+        raise Exception(f"Failed to create DB {couchUrl}/{dbName}: {r.text} status code {r.status_code}")
+
+def setupValidator(couchUrl, targetDb, auth, certTuple=None, caFileName=None):
+    """
+    setup validator in couch database
+    :param couchUrl: couch url string
+    :param targetDb: database name
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    designDoc = {
+        "_id": "_design/rejectBadDocs",
+        "validate_doc_update": (
+            "function(newDoc, oldDoc, userCtx) { "
+            "if (newDoc.blockMe === true) { throw({forbidden: 'Blocked by validator'}); } "
+            "}"
+        )
+    }
+    headers = {'Content-Type': 'application/json'}
+
+    r = requests.put(f"{couchUrl}/{targetDb}/{designDoc['_id']}",
+                     data=json.dumps(designDoc), headers=headers, auth=auth, cert=certTuple, verify=caFileName)
+    if r.status_code not in (201, 202):
+        raise Exception(f"Failed to upload validator: {r.text}")
+
+def insertValidDocs(couchUrl, sourceDb, auth, certTuple=None, caFileName=None):
+    """
+    insert set of valid documents into couch database
+    wipe out couch database
+    :param couchUrl: couch url string
+    :param sourceDb: database name
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    validDocIds = []
+    headers = {'Content-Type': 'application/json'}
+    for i in range(3):
+        doc = {"_id": f"validDoc{i}", "msg": f"This is valid doc #{i}"}
+        r = requests.put(f"{couchUrl}/{sourceDb}/{doc['_id']}",
+                         data=json.dumps(doc), headers=headers, auth=auth, cert=certTuple, verify=caFileName)
+        if r.status_code in (201, 202):
+            validDocIds.append(doc['_id'])
+        else:
+            raise Exception(f"Failed to insert valid doc: {r.text}")
+    return validDocIds
+
+def insertInvalidDoc(couchUrl, sourceDb, auth, certTuple=None, caFileName=None):
+    """
+    insert invalid document into couch database
+    :param couchUrl: couch url string
+    :param sourceDb: database name
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    doc = {"_id": "badDoc", "blockMe": True, "msg": "This should fail replication"}
+    headers = {'Content-Type': 'application/json'}
+    r = requests.put(f"{couchUrl}/{sourceDb}/{doc['_id']}",
+                     data=json.dumps(doc), headers=headers, auth=auth, cert=certTuple, verify=caFileName)
+    if r.status_code not in (201, 202):
+        raise Exception(f"Failed to insert bad doc: {r.text}")
+    return doc['_id']
+
+def deleteReplicationDoc(sourceUrl, replicationId, auth, certTuple=None, caFileName=None):
+    """
+    delete replication document
+    :param sourceUrl: source url string
+    :param replicationId: replication id string
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    docUrl = f"{sourceUrl}/_replicator/{replicationId}"
+    r = requests.get(docUrl, auth=auth, cert=certTuple, verify=caFileName)
+
+    if r.status_code == 200:
+        rev = r.json()['_rev']
+        delResp = requests.delete(f"{docUrl}?rev={rev}", auth=auth, cert=certTuple, verify=caFileName)
+        if delResp.status_code not in (200, 202):
+            raise Exception(f"Failed to delete replication doc: {delResp.text}")
+        print(f"Deleted existing replication doc '{replicationId}'")
+    elif r.status_code != 404:
+        raise Exception(f"Failed to check replication doc: {r.text}")
+
+def setupReplication(sourceUrl, sourceDb, targetUrl, targetDb, auth, replicationId, certTuple=None, caFileName=None):
+    """
+    setup replication between source and target databases
+    :param sourceUrl: source couchdb url string
+    :param sourceDb: source database name
+    :param targetUrl: target couchdb url string
+    :param targetDb: target database name
+    :param auth: authentication tuple
+    :param replicationId: replication id string
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    replicationDoc = {
+        "_id": replicationId,
+        "source": sourceDb,
+        "target": f"{targetUrl}/{targetDb}",
+        "create_target": False,
+        "continuous": True,
+        "user_ctx": {"name": "admin", "roles": ["_admin"]}
+    }
+    headers = {'Content-Type': 'application/json'}
+    r = requests.put(f"{sourceUrl}/_replicator/{replicationId}",
+                     data=json.dumps(replicationDoc), headers=headers, auth=auth, cert=certTuple, verify=caFileName)
+    if r.status_code not in (201, 202):
+        raise Exception(f"Failed to create replication: {r.text}")
+    print(f"Replication '{replicationId}' set up successfully with status {r.status_code}")
+
+def checkReplicationStatus(couchUrl, replicationId, auth, certTuple=None, caFileName=None):
+    """
+    checks replication status for given replication id
+    :param couchUrl: couch url string
+    :param replicationId: replication id string
+    :param auth: authentication tuple
+    :param certTuple: cert and ckey tuple
+    :param caFileName: CA file name
+    :return: nothing
+    """
+    for _ in range(10):
+        time.sleep(2)
+        r = requests.get(f"{couchUrl}/_replicator/{replicationId}", auth=auth, cert=certTuple, verify=caFileName)
+        doc = r.json()
+        state = doc.get("_replication_state", "unknown")
+        reason = doc.get("_replication_state_reason", "")
+        print(f"Replication state: {state}")
+        if reason:
+            print(f"Reason: {reason}")
+        if state in ("error", "completed"):
+            return state, reason
+    return "unknown", ""
+
+def main():
+    """
+    main function for testing purposes
+    """
+    if os.getenv('COUCH_SOURCE_URL') == '':
+        print('ERROR: no COUCH_SOURCE_URL found')
+        sys.exit(1)
+    if os.getenv('COUCH_TARGET_URL') == '':
+        print('ERROR: no COUCH_TARGET_URL found')
+        sys.exit(1)
+    if os.getenv('COUCH_CREDENTIALS') == '':
+        print('ERROR: no COUCH_CREDENTIALS found, should be username:password')
+        sys.exit(1)
+
+    certTuple = (cert(), ckey())
+    capath = os.environ.get("X509_CERT_DIR", '/etc/grid-security/certificates')
+    cacerts = caBundle(capath)
+
+    with tempfile.NamedTemporaryFile(mode='w+', suffix=".pem", delete=True) as tfile:
+        tfile.write(cacerts)
+        tfile.flush()
+        caFileName = tfile.name
+
+        sourceUrl = os.getenv('COUCH_SOURCE_URL', '')
+        targetUrl = os.getenv('COUCH_TARGET_URL', '')
+        authEnv = os.getenv('COUCH_CREDENTIALS', '')
+        auth = tuple(authEnv.split(':')) if ':' in authEnv else None
+        sourceDb = 'test_db'
+        targetDb = 'test_db'
+        replicationId = 'replicationFailTest'
+
+        print(f"INFO: wipe out {sourceUrl}/{sourceDb}")
+        wipeDb(sourceUrl, sourceDb, auth, certTuple, caFileName)
+        print(f"INFO: wipe out {targetUrl}/{targetDb}")
+        wipeDb(targetUrl, targetDb, auth, certTuple, caFileName)
+
+        print("INFO: Setting up test environment...")
+        createDb(sourceUrl, sourceDb, auth, certTuple, caFileName)
+        createDb(targetUrl, targetDb, auth, certTuple, caFileName)
+        setupValidator(targetUrl, targetDb, auth, certTuple, caFileName)
+
+        print("INFO: Inserting valid documents...")
+        validDocIds = insertValidDocs(sourceUrl, sourceDb, auth, certTuple, caFileName)
+
+        print("INFO: Inserting a document that will be rejected by the target...")
+        badDocId = insertInvalidDoc(sourceUrl, sourceDb, auth, certTuple, caFileName)
+
+        print("INFO: Starting replication...")
+        deleteReplicationDoc(sourceUrl, replicationId, auth, certTuple, caFileName)
+        setupReplication(sourceUrl, sourceDb, targetUrl, targetDb, auth, replicationId, certTuple, caFileName)
+
+        print("INFO: Waiting for replication to complete or fail...")
+        state, reason = checkReplicationStatus(sourceUrl, replicationId, auth, certTuple, caFileName)
+
+        print("\nINFO: Successfully inserted document IDs:")
+        for docId in validDocIds:
+            print(f" - {docId}")
+
+        print(f"\nWARNING: Problematic document ID (should fail replication): {badDocId}")
+
+        print(f"\nINFO: Replication final state: {state}")
+        if reason:
+            print(f"ERROR: Replication failure reason: {reason}")
+        else:
+            print("INFO: No error reported.")
+
+if __name__ == "__main__":
+    main()
+

{wmglobalqueue-2.4.2rc7 → wmglobalqueue-2.4.2rc8}/src/python/Utils/CertTools.py

@@ -1,3 +1,6 @@
+"""
+Module to deal with user certificates and CAs
+"""
 from builtins import str
 import os
 
@@ -60,3 +63,38 @@ def getCAPathFromEnv():
     you need to set either the X509_CERT_DIR variable or the cacert key of the request.
     """
     return os.environ.get("X509_CERT_DIR")
+
+
+def caBundle(caPath="/etc/grid-security/certificates"):
+    """
+    Load all PEM certificates from the given caPath and write them as single CA bundle PEM.
+
+    :param caPath: Path to directory containing .pem certificate files.
+    :return: A single string containing all concatenated PEM pemCertificates which may be
+    written to a caBundleFile if necessary (used by requests library)
+    """
+    if not os.path.isdir(caPath):
+        raise ValueError(f"Invalid caPath: {caPath} is not a directory")
+
+    pemCertificates = []
+
+    for fileName in sorted(os.listdir(caPath)):
+        filePath = os.path.join(caPath, fileName)
+
+        # Only consider readable files that look like PEM certificates
+        if not os.path.isfile(filePath):
+            continue
+        if not fileName.endswith(".pem"):
+            continue
+
+        try:
+            with open(filePath, "r", encoding="utf-8") as istream:
+                certData = istream.read()
+                if "BEGIN CERTIFICATE" in certData:
+                    pemCertificates.append(certData)
+        except Exception as e:
+            print(f"Warning: Could not read {filePath}: {e}")
+
+    if len(pemCertificates) == 0:
+        raise ValueError(f"No PEM files found in {caPath}")
+    return "\n".join(pemCertificates)

{wmglobalqueue-2.4.2rc7 → wmglobalqueue-2.4.2rc8}/src/python/WMCore/Database/CMSCouch.py

@@ -25,12 +25,14 @@ import logging
 import re
 import time
 import sys
+from pprint import pformat
 from datetime import datetime
 from http.client import HTTPException
 
 from Utils.IteratorTools import grouper, nestedDictUpdate
 from WMCore.Lexicon import sanitizeURL
 from WMCore.Services.Requests import JSONRequests
+from WMCore.Database.CouchMonitoring import checkStatus
 
 
 def check_name(dbname):
@@ -1274,6 +1276,55 @@ class CouchMonitor(object):
             return resp
         return data.get("docs", resp)
 
+    def couchReplicationStatus(self):
+        """
+        check couchdb replication status with compatible output of checkCouchReplications
+
+        :return: a list of dictionaries with the status of the replications and an
+            error message
+        """
+        output = []
+        sdict = checkStatus(kind='scheduler')
+        rdict = checkStatus(kind='replicator')
+        method = 'scheduler+replicator'
+        # update sdict only with entries from replicator dict which are not present in scheduler
+        for key, val in rdict['current_status'].items():
+            if key not in sdict['current_status']:
+                sdict['current_status'][key] = val
+        stateFailures = ['error', 'failed']
+        for rid, record in sdict['current_status'].items():
+            if record['state'] in stateFailures:
+                status['state'] = 'error'
+                source = sanitizeURL(record['source'])
+                target = sanitizeURL(record['target'])
+                error = record['error']
+                history = pformat(record['history'])
+                msg = f"Replication from {source} to {target} for document {rid} is in a bad state: {error}; "
+                msg += f"History: {history}"
+                status = {'name': 'CouchServer', 'status': 'error', 'error_message': msg, 'method': method}
+                output.append(status)
+
+        # if our replication is fine we should check that it is not in a stale phase
+        activeTasks = self.getActiveTasks()
+        activeTasks = [task for task in activeTasks if task["type"].lower() == "replication"]
+        resp = self.checkReplicationState()
+        for replTask in activeTasks:
+            if self.isReplicationStale(replTask):
+                source = sanitizeURL(replTask['source'])['url']
+                target = sanitizeURL(replTask['target'])['url']
+                msg = f"Replication from {source} to {target} is stale and it's last"
+                msg += f"update time was at: {replTask.get('updated_on')}"
+                resp['status'] = 'error'
+                resp['error_message'] += msg
+                resp['method'] = 'stale phase'
+                resp['name'] = 'CouchServer'
+                output.append(resp)
+        # check if we did not record any replication status, then add the ok status
+        if len(output) == 0:
+            status = {'name': 'CouchServer', 'status': 'ok', 'error_message': ''}
+            output.append(status)
+        return output
+
     def checkCouchReplications(self, replicationsList):
         """
         Check whether the list of expected replications exist in CouchDB
@@ -1281,9 +1332,11 @@ class CouchMonitor(object):
 
         :param replicationsList: a list of dictionary with the replication
             document setup.
-        :return: a dictionary with the status of the replications and an
+        :return: a list of dictionaries with the status of the replications and an
             error message
         """
+        output = []
+        method = 'comparison of replications docs vs active tasks'
         activeTasks = self.getActiveTasks()
         # filter out any task that is not a database replication
         activeTasks = [task for task in activeTasks if task["type"].lower() == "replication"]
@@ -1292,12 +1345,12 @@ class CouchMonitor(object):
             msg = f"Expected to have {len(replicationsList)} replication tasks, "
             msg += f"but only {len(activeTasks)} in CouchDB. "
             msg += f"Current replications are: {activeTasks}"
-            return {'status': 'error', 'error_message': msg}
+            status = {'name': 'CouchServer', 'status': 'error', 'error_message': msg, 'method': method}
+            output.append(status)
 
         resp = self.checkReplicationState()
         if resp['status'] != 'ok':
-            # then there is a problem, return its status
-            return resp
+            output.append(resp)
 
         # finally, check if replications are being updated in a timely fashion
         for replTask in activeTasks:
@@ -1308,7 +1361,15 @@ class CouchMonitor(object):
                 msg += f"update time was at: {replTask.get('updated_on')}"
                 resp['status'] = 'error'
                 resp['error_message'] += msg
-        return resp
+                resp['method'] = method
+                resp['name'] = 'CouchServer'
+                output.append(resp)
+
+        # check if we did not record any replication status, then add the ok status
+        if len(output) == 0:
+            status = {'name': 'CouchServer', 'status': 'ok', 'error_message': ''}
+            output.append(status)
+        return output
 
     def checkReplicationState(self):
         """
@@ -1347,3 +1408,20 @@ class CouchMonitor(object):
             # then it has been recently updated
             return True
         return False
+
+    def isReplicationStale(self, replInfo, niter=10):
+        """
+        Ensure that the replication document is up-to-date as a
+        function of the checkpoint interval.
+
+        :param replInfo: dictionary with the replication information
+        :param niter: number of iteration for checkpoint interval
+        :return: True if replication is working fine, otherwise False
+        """
+        maxUpdateInterval = niter * replInfo['checkpoint_interval'] / 1000
+        lastUpdate = replInfo["updated_on"]
+
+        if lastUpdate + maxUpdateInterval > int(time.time()):
+            # then it has been recently updated and it means replication is not stale
+            return False
+        return True