wmglobalqueue 2.4.2rc7__py3-none-any.whl → 2.4.2rc8__py3-none-any.whl

Utils/CertTools.py

@@ -1,3 +1,6 @@
+"""
+Module to deal with user certificates and CAs
+"""
 from builtins import str
 import os
 
@@ -60,3 +63,38 @@ def getCAPathFromEnv():
     you need to set either the X509_CERT_DIR variable or the cacert key of the request.
     """
     return os.environ.get("X509_CERT_DIR")
+
+
+def caBundle(caPath="/etc/grid-security/certificates"):
+    """
+    Load all PEM certificates from the given caPath and write them as single CA bundle PEM.
+
+    :param caPath: Path to directory containing .pem certificate files.
+    :return: A single string containing all concatenated PEM pemCertificates which may be
+    written to a caBundleFile if necessary (used by requests library)
+    """
+    if not os.path.isdir(caPath):
+        raise ValueError(f"Invalid caPath: {caPath} is not a directory")
+
+    pemCertificates = []
+
+    for fileName in sorted(os.listdir(caPath)):
+        filePath = os.path.join(caPath, fileName)
+
+        # Only consider readable files that look like PEM certificates
+        if not os.path.isfile(filePath):
+            continue
+        if not fileName.endswith(".pem"):
+            continue
+
+        try:
+            with open(filePath, "r", encoding="utf-8") as istream:
+                certData = istream.read()
+                if "BEGIN CERTIFICATE" in certData:
+                    pemCertificates.append(certData)
+        except Exception as e:
+            print(f"Warning: Could not read {filePath}: {e}")
+
+    if len(pemCertificates) == 0:
+        raise ValueError(f"No PEM files found in {caPath}")
+    return "\n".join(pemCertificates)

WMCore/Database/CMSCouch.py

@@ -25,12 +25,14 @@ import logging
 import re
 import time
 import sys
+from pprint import pformat
 from datetime import datetime
 from http.client import HTTPException
 
 from Utils.IteratorTools import grouper, nestedDictUpdate
 from WMCore.Lexicon import sanitizeURL
 from WMCore.Services.Requests import JSONRequests
+from WMCore.Database.CouchMonitoring import checkStatus
 
 
 def check_name(dbname):
@@ -1274,6 +1276,55 @@ class CouchMonitor(object):
             return resp
         return data.get("docs", resp)
 
+    def couchReplicationStatus(self):
+        """
+        check couchdb replication status with compatible output of checkCouchReplications
+
+        :return: a list of dictionaries with the status of the replications and an
+            error message
+        """
+        output = []
+        sdict = checkStatus(kind='scheduler')
+        rdict = checkStatus(kind='replicator')
+        method = 'scheduler+replicator'
+        # update sdict only with entries from replicator dict which are not present in scheduler
+        for key, val in rdict['current_status'].items():
+            if key not in sdict['current_status']:
+                sdict['current_status'][key] = val
+        stateFailures = ['error', 'failed']
+        for rid, record in sdict['current_status'].items():
+            if record['state'] in stateFailures:
+                status['state'] = 'error'
+                source = sanitizeURL(record['source'])
+                target = sanitizeURL(record['target'])
+                error = record['error']
+                history = pformat(record['history'])
+                msg = f"Replication from {source} to {target} for document {rid} is in a bad state: {error}; "
+                msg += f"History: {history}"
+                status = {'name': 'CouchServer', 'status': 'error', 'error_message': msg, 'method': method}
+                output.append(status)
+
+        # if our replication is fine we should check that it is not in a stale phase
+        activeTasks = self.getActiveTasks()
+        activeTasks = [task for task in activeTasks if task["type"].lower() == "replication"]
+        resp = self.checkReplicationState()
+        for replTask in activeTasks:
+            if self.isReplicationStale(replTask):
+                source = sanitizeURL(replTask['source'])['url']
+                target = sanitizeURL(replTask['target'])['url']
+                msg = f"Replication from {source} to {target} is stale and it's last"
+                msg += f"update time was at: {replTask.get('updated_on')}"
+                resp['status'] = 'error'
+                resp['error_message'] += msg
+                resp['method'] = 'stale phase'
+                resp['name'] = 'CouchServer'
+                output.append(resp)
+        # check if we did not record any replication status, then add the ok status
+        if len(output) == 0:
+            status = {'name': 'CouchServer', 'status': 'ok', 'error_message': ''}
+            output.append(status)
+        return output
+
     def checkCouchReplications(self, replicationsList):
         """
         Check whether the list of expected replications exist in CouchDB
@@ -1281,9 +1332,11 @@ class CouchMonitor(object):
 
         :param replicationsList: a list of dictionary with the replication
             document setup.
-        :return: a dictionary with the status of the replications and an
+        :return: a list of dictionaries with the status of the replications and an
             error message
         """
+        output = []
+        method = 'comparison of replications docs vs active tasks'
         activeTasks = self.getActiveTasks()
         # filter out any task that is not a database replication
         activeTasks = [task for task in activeTasks if task["type"].lower() == "replication"]
@@ -1292,12 +1345,12 @@ class CouchMonitor(object):
             msg = f"Expected to have {len(replicationsList)} replication tasks, "
             msg += f"but only {len(activeTasks)} in CouchDB. "
             msg += f"Current replications are: {activeTasks}"
-            return {'status': 'error', 'error_message': msg}
+            status = {'name': 'CouchServer', 'status': 'error', 'error_message': msg, 'method': method}
+            output.append(status)
 
         resp = self.checkReplicationState()
         if resp['status'] != 'ok':
-            # then there is a problem, return its status
-            return resp
+            output.append(resp)
 
         # finally, check if replications are being updated in a timely fashion
         for replTask in activeTasks:
@@ -1308,7 +1361,15 @@ class CouchMonitor(object):
                 msg += f"update time was at: {replTask.get('updated_on')}"
                 resp['status'] = 'error'
                 resp['error_message'] += msg
-        return resp
+                resp['method'] = method
+                resp['name'] = 'CouchServer'
+                output.append(resp)
+
+        # check if we did not record any replication status, then add the ok status
+        if len(output) == 0:
+            status = {'name': 'CouchServer', 'status': 'ok', 'error_message': ''}
+            output.append(status)
+        return output
 
     def checkReplicationState(self):
         """
@@ -1347,3 +1408,20 @@ class CouchMonitor(object):
             # then it has been recently updated
             return True
         return False
+
+    def isReplicationStale(self, replInfo, niter=10):
+        """
+        Ensure that the replication document is up-to-date as a
+        function of the checkpoint interval.
+
+        :param replInfo: dictionary with the replication information
+        :param niter: number of iteration for checkpoint interval
+        :return: True if replication is working fine, otherwise False
+        """
+        maxUpdateInterval = niter * replInfo['checkpoint_interval'] / 1000
+        lastUpdate = replInfo["updated_on"]
+
+        if lastUpdate + maxUpdateInterval > int(time.time()):
+            # then it has been recently updated and it means replication is not stale
+            return False
+        return True

WMCore/Database/CouchMonitoring.py

@@ -0,0 +1,450 @@
+#!/usr/bin/env python
+"""
+This module provides helper functions to obtain and handle CouchDB Replication data:
+    - getSchedulerJobDocs get replication status based on scheduler information
+    - getReplicatorDocs get replication status based on replicator information
+    - compareReplicationStatus compares previous and current statuses
+    - formatPrometheusMetrics format status metrics in Prometheus format
+    - createAlerts create alerts from given status dict
+    - checkStatus perform all checks for couchdb replication
+
+Example of using Flask framework to serve prometheus metrics about CouchDB replication
+
+import requests
+from flask import Flask, Response
+import threading
+import time
+
+app = Flask(__name__)
+status_cache = {}
+
+@app.route("/metrics")
+def metrics():
+    return Response(formatPrometheusMetrics(status_cache), mimetype="text/plain")
+
+def daemonCouchReplicationStatus(interval=30):
+    global status_cache
+    while True:
+        new_status = getSchedulerJobDocs(COUCHDB_URL, USERNAME, PASSWORD)
+        status_cache = new_status
+        time.sleep(interval)
+
+if __name__ == "__main__":
+    # Start the background thread to update replication status periodically
+    threading.Thread(target=daemonCouchReplicationStatus, daemon=True).start()
+    # Run the Flask app
+    app.run(host="0.0.0.0", port=8000)
+"""
+
+import os
+import json
+import requests
+import tempfile
+
+# WMCore modules
+from Utils.CertTools import cert, ckey, caBundle
+
+
+def getSchedulerJobDocs(couchdbUrl):
+    """
+    Fetch CouchDB replication statuses. The logic is based on /_scheduler/jobs CouchDB end-point
+    see https://docs.couchdb.org/en/stable/api/server/common.html#api-server-scheduler-jobs
+    :param couchdbUrl: url of couch db
+    :return: dictionary of statuses for all found replication documents
+    """
+    username, password = couchCredentials()
+    auth = (username, password) if username and password else None
+    try:
+        response = requests.get(f"{couchdbUrl}/_scheduler/jobs", auth=auth)
+        response.raise_for_status()
+        data = response.json()
+
+        statuses = {}
+        for job in data.get('jobs', []):
+            doc_id = job.get('doc_id') or job.get('id')
+            source = job.get('source')
+            target = job.get('target')
+            history = job.get('history', [])
+            info = job.get('info', {})
+
+            # Determine current state from latest history item
+            state = history[0]['type'] if history else 'unknown'
+
+            # Detect error if 'crashed' exists in any history entry
+            error = None
+            for h in history:
+                if h.get('type') == 'crashed':
+                    error = f"Job previous crashed at {h.get('timestamp')} due to {h.get('reason')}"
+                    break
+
+            # check info document
+            if info and info.get('doc_write_failures', 0) != 0:
+                error = f"found failure of replication jobs in {couchdbUrl}/_scheduler/jobs "
+                state = "error"
+                # try to get more info about the error
+                try:
+                    response = requests.get(f"{couchdbUrl}/_scheduler/docs/_replicator/{doc_id}", auth=auth)
+                    response.raise_for_status()
+                    data = response.json()
+                    error += f" Replicator state for {doc_id}: "
+                    error += json.dumps(data)
+                except:
+                    pass
+
+            statuses[doc_id] = {
+                'state': state,
+                'source': source,
+                'target': target,
+                'error': error,
+                'history': history
+                }
+
+        return statuses
+    except requests.RequestException as e:
+        print(f"Error fetching scheduler jobs: {e}")
+        return {}
+
+
+def getReplicatorDocs(url=None):
+    """
+    Helper function to get all replicator docs and return summary dictionary
+    :param url: url of the couchdb
+    :return: replication summary dictionary
+    """
+    username, password = couchCredentials()
+    auth = (username, password) if username and password else None
+    if not url:
+        url = "http://localhost:5984"
+    headers = {"Accept": "application/json"}
+
+    # Get list of all documents in _replicator
+    r = requests.get(f"{url}/_replicator/_all_docs?include_docs=true",
+                     headers=headers, auth=auth)
+ 
+    if r.status_code != 200:
+        raise Exception(f"Failed to fetch replication docs: {r.text}")
+
+    data = r.json()
+    result = {}
+
+    for row in data.get("rows", []):
+        doc = row.get("doc", {})
+        doc_id = doc.get("_id")
+        if doc_id.startswith("_design/"):
+            continue  # skip design docs
+
+        summary = {
+            "state": doc.get("_replication_state"),
+            "source": doc.get("source"),
+            "target": doc.get("target"),
+            "error": doc.get("_replication_state_reason"),
+            "history": []
+        }
+
+        history = doc.get("_replication_history", [])
+        for h in history:
+            entry = {
+                "timestamp": h.get("start_time") or h.get("end_time"),
+                "type": h.get("type") or "unknown"
+            }
+            summary["history"].append(entry)
+
+        result[doc_id] = summary
+
+    return result
+
+
+def compareReplicationStatus(prev, curr):
+    """
+    Helper function to compare replication status from previous to current state
+    :param prev: previous replication status dictionary
+    :param curr: current replication status dictionary
+    :return: dictionary of changes
+    """
+    changes = {}
+    for key in curr:
+        if key not in prev or prev[key] != curr[key]:
+            changes[key] = {
+                    'old': prev.get(key),
+                    'new': curr[key]
+                    }
+    return changes
+
+
+def formatPrometheusMetrics(statuses):
+    """
+    Helper function to provide Prometheus metrics from given status dictionary
+    :param statuses: replication status dictionary
+    :return: prometheus metrics
+    """
+    states = {'error': -1, 'completed': 0, 'started': 1, 'added': 2, 'waiting': 3, 'triggered': 4, 'failed': 5}
+    lines = [
+            f'# HELP couchdb_replication_state Replication state: {states}',
+            '# TYPE couchdb_replication_state gauge'
+            ]
+    for key, status in statuses.items():
+        label = f'replId="{key}",source="{status["source"]}",target="{status["target"]}"'
+        value = 0   # default error/other
+        for k, v in states.items():
+            if status['state'] == k:
+                value = v
+                break
+        lines.append(f'couchdb_replication_state{{{label}}} {value}')
+    return '\n'.join(lines)
+
+
+def createAlerts(statuses):
+    """
+    Helper function to check alerts of replication status dictionary
+    :param statuses: replication status dictionary
+    :return: alerts dictionary
+    """
+    alerts = {}
+    for key, status in statuses.items():
+        if status['state'] != 'completed':
+            alerts[key] = f"Replication state for {key} is '{status['state']}', error: {status['error']}"
+    return alerts
+
+
+def couchCredentials():
+    """
+    Select CouchDB credentials from provided secrets file
+    :return: tuple of (user, password)
+    """
+    fname = os.getenv('WMAGENT_SECRETS_LOCATION', '')
+    if fname == "":
+        raise Exception("No WMAGENT_SECRETS_LOCATION in environment")
+    user = ''
+    password = ''
+    data = ''
+    with open(fname, 'r', encoding="utf-8") as istream:
+        data = istream.read()
+    for item in data.split('\n'):
+        if 'COUCH_USER' in item:
+            user = item.split('=')[-1]
+        if 'COUCH_PASS' in item:
+            password = item.split('=')[-1]
+    return user, password
+
+
+def checkStatus(url=None, prevStatus=None, kind="scheduler"):
+    """
+    Perform check of replication statuses
+    :param url: couchdb URL
+    :param prevStatus: previous status dictionary
+    :param kind: kind of data look-up, e.g. scheduler or replicator
+    :return: dictionary of current couchdb replication
+
+    Here is an example of such dictionary structure:
+    {'current_status': currStatus (dictionary),
+     'previous_status': prevStatus (dictionary),
+     'changes': changes (dictionary),
+     'metrics': metrics (string),
+     'alerts': alerts (dictionary)}
+
+    Then, current and previous status dictionaries have the following form:
+    {
+      "14843c24643f8960eb159f5912f0f938": {
+        "state": "started",
+        "source": "https://xxx.cern.ch/couchdb/workqueue/",
+        "target": "http://127.0.0.1:5984/workqueue_inbox/",
+        "error": "Job previously crashed at 2025-05-05T18:47:11Z due to {changes_reader_died,{timeout,ibrowse_stream_cleanup}}",
+        "history": [
+          {
+            "timestamp": "2025-05-05T18:47:11Z",
+            "type": "started"
+          },
+          ...
+         ]
+      },
+      "14843c24643f8960eb159f5912f0e51e": {
+        "state": "started",
+        "source": "http://127.0.0.1:5984/wmagent_summary/",
+        "target": "https://xxx.cern.ch/couchdb/wmstats/",
+        "error": null,
+        "history": [
+          {
+            "timestamp": "2025-04-09T11:19:36Z",
+            "type": "started"
+          },
+          {
+            "timestamp": "2025-04-09T11:19:36Z",
+            "type": "added"
+          }
+        ]
+      },
+    ...
+    }
+    """
+    if not prevStatus:
+        prevStatus = {}
+    if not url:
+        url = "http://localhost:5984"
+
+    # first let's get statuses of documents
+    if kind == "scheduler":
+        currStatus = getSchedulerJobDocs(url)
+    elif kind == "replicator":
+        currStatus = getReplicatorDocs(url)
+    else:
+        raise Exception("Unsupported kind of documents '{kind}', should be either scheduler or replicator")
+
+    # now we can find out changes from previous statuses
+    changes = compareReplicationStatus(prevStatus, currStatus)
+
+    # construct prometheus metrics with current statuses
+    metrics = formatPrometheusMetrics(currStatus)
+
+    # construct alerts with current statuses
+    alerts = createAlerts(currStatus)
+
+    # build final dictionary to return upstream
+    sdict = {'current_status': currStatus,
+             'previous_status': prevStatus,
+             'changes': changes,
+             'metrics': metrics,
+             'alerts': alerts}
+    return sdict
+
+
+def getDocCount(url, auth, certTuple, caCert):
+    """
+    helper function to get document counts
+    :param url: url of the couchdb
+    :param auth: couchdb authentication credentials tuple
+    :param caCert: ca bundle file name
+    :return: document count
+    """
+    resp = requests.get(url, auth=auth, cert=certTuple, verify=caCert or True)
+    resp.raise_for_status()
+    return resp.json().get('doc_count', -1)
+
+
+def getReplicationState(url, auth, certTuple, caCert):
+    """
+    helper function to get replication state from given couchdb url
+    :param url: url of the couchdb
+    :param auth: couchdb authentication credentials tuple
+    :param caCert: ca bundle file name
+    :return: tuple of replication state and its time
+    """
+    resp = requests.get(url, auth=auth, cert=certTuple, verify=caCert or True)
+    resp.raise_for_status()
+    doc = resp.json()
+    return doc.get('_replication_state'), doc.get('_replication_state_time')
+
+
+def compareCouchInstances(sourceUrl, targetUrl, replUrl):
+    """
+    Compare the number of documents between source and destination CouchDB databases.
+    Monitor replication if the counts differ but replication status is OK.
+ 
+    Parameters:
+    :param sourceUrl: str, e.g. http://localhost:5984/source_db
+    :param targetUrl: str, e.g. http://localhost:5984/dest_db
+    :param replUrl: str, e.g. http://localhost:5984/_replicator/<replId>
+    """
+    user, password = couchCredentials()
+    auth = (user, password)
+    sdict = {}
+    userCert = cert() if cert() else ''
+    userCkey = ckey() if ckey() else ''
+    if userCkey == '' or userCert == '':
+        return sdict
+    certTuple = (userCert, userCkey)
+    with tempfile.NamedTemporaryFile(mode='w+', suffix=".pem", delete=True) as tfile:
+        capath = os.environ.get("X509_CERT_DIR", '/etc/grid-security/certificates')
+        cacerts = caBundle(capath)
+        tfile.write(cacerts)
+        tfile.flush()
+
+        sourceCount = getDocCount(sourceUrl, auth, certTuple, tfile.name)
+        targetCount = getDocCount(targetUrl, auth, certTuple, tfile.name)
+        state, stateTime = getReplicationState(replUrl, auth, certTuple, tfile.name)
+
+        sdict = {
+                "source": sourceUrl,
+                "target": targetUrl,
+                "source_count": sourceCount,
+                "target_count": targetCount,
+                "state": state,
+                "state_timestamp": stateTime
+                }
+    return sdict
+
+
+def exampleReplicationStatus(sourceUrl=None):
+    """
+    Example function to test replication status either based on scheduler or replicator info
+    This function should run on a node with local CouchDB access as all of its logic
+    relies on using localhost:5984 URL
+    """
+
+    try:
+        print(f"checking {sourceUrl}")
+
+        # let's first test scheduler info
+        sdict = checkStatus(url=sourceUrl, kind="scheduler")
+        print('--- status based on scheduler info ---')
+        print(sdict['current_status'])
+        print('--- metrics ---')
+        print(sdict['metrics'])
+        if sdict.get('alerts', None):
+            print('--- alerts ---')
+            for k, msg in sdict['alerts'].items():
+                print(f"{k}: {msg}")
+
+        print()
+
+        # now let's test replicator info
+        rdict = checkStatus(url=sourceUrl, kind="replicator")
+        print('--- status based on replicator info ---')
+        print(rdict['current_status'])
+        print('--- metrics ---')
+        print(rdict['metrics'])
+        if rdict.get('alerts', None):
+            print('--- alerts ---')
+            for k, msg in rdict['alerts'].items():
+                print(f"{k}: {msg}")
+
+    except Exception as exp:
+        print(str(exp))
+
+
+def exampleIndividualDocument(sourceUrl, targetUrl, replUrl):
+    """
+    Example function how to test check status of particular replication document
+    This function should run through CMSWEB frontend URLs as we need to compare
+    documents in both source and target CouchDB instances
+    :param sourceUrl: source couchdb URL, e.g. https://xxx.cern.ch/couchdb/test_db
+    :param targetUrl: target couchdb URL, e.g. https://xxx.cern.ch/couchdb/test_db
+    :param replUrl: replication URL, e.g. https://xxx.cern.ch/couchdb/test_db/_replicator/bla
+    """
+    try:
+        result = compareCouchInstances(sourceUrl, targetUrl, replUrl)
+        print('--- compare CouchDB Instances ---')
+        print('source: ', sourceUrl)
+        print('target: ', targetUrl)
+        print(result)
+    except:
+        pass
+
+
+def test():
+    """
+    test functions
+    """
+    import sys
+    if len(sys.argv) > 1:
+        sourceUrl = sys.argv[1]
+        exampleReplicationStatus(sourceUrl)
+    else:
+        print("Cannot run tests, please provide at least CouchDB source URL, or <srcUrl> <targetUrl> <replicationId>")
+    if len(sys.argv) == 4:
+        sourceUrl = sys.argv[1]
+        targetUrl = sys.argv[2]
+        replUrl = sys.argv[3]
+        exampleIndividualDocument(sourceUrl, targetUrl, replUrl)
+
+if __name__ == '__main__':
+    test()

WMCore/Services/Rucio/Rucio.py

@@ -706,15 +706,18 @@ class Rucio(object):
         Update rule information for a given rule id
         :param ruleId: string with the rule id
         :param opts: dictionary, rule id options passed to Rucio
-        :return: boolean status of update call
+        :return: boolean status to represent whether it succeeded or not.
+                 ok status code and RuleNotFound exception are considered as succeeded,
+                 any other Exception case is considered as failed.
         """
-        status = None
+        status = True
         try:
             status = self.cli.update_replication_rule(ruleId, opts)
         except RuleNotFound:
             self.logger.error("Cannot find any information for rule id: %s", ruleId)
         except Exception as ex:
             self.logger.error("Exception updating rule id: %s. Error: %s", ruleId, str(ex))
+            status = False
         return status
 
     def deleteRule(self, ruleId, purgeReplicas=False):

WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.4.2rc7'
+__version__ = '2.4.2rc8'
 __all__ = []

{wmglobalqueue-2.4.2rc7.dist-info → wmglobalqueue-2.4.2rc8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wmglobalqueue
-Version: 2.4.2rc7
+Version: 2.4.2rc8
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group
 Maintainer-email: hn-cms-wmdevelopment@cern.ch