wmglobalqueue 2.3.11.1__tar.gz → 2.4.0rc1__tar.gz

{wmglobalqueue-2.3.11.1/src/python/wmglobalqueue.egg-info → wmglobalqueue-2.4.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: wmglobalqueue
-Version: 2.3.11.1
+Version: 2.4.0rc1
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/bin/wmcoreD

@@ -35,12 +35,15 @@ __version__ = "$Revision: 1.16 $"
 __author__ = "fvlingen@caltech.edu"
 
 import getopt
-import logging
 import os
 import subprocess
 import sys
 import time
+import json
+import psutil
 
+from Utils.Utilities import extractFromXML
+from Utils.ProcFS import processStatus
 from WMCore.Agent.Daemon.Details import Details
 from WMCore.Configuration import loadConfigurationFile
 from WMCore.WMFactory import WMFactory
@@ -177,7 +180,7 @@ def connectionTest(configFile):
         print("skipped.")
         return
 
-    (dialect, junk) = config.CoreDatabase.connectUrl.split(":", 1)
+    dialect, _ = config.CoreDatabase.connectUrl.split(":", 1)
     socket = getattr(config.CoreDatabase, "socket", None)
 
     try:
@@ -235,6 +238,15 @@ def startup(configFile):
         else:
             print('Path for daemon file does not exist!')
             sys.exit(1)
+        # write into component area process status information
+        cpath = os.path.join(compDir, "threads.json")
+        if os.path.exists(cpath):
+            os.remove(cpath)
+        cpid = extractFromXML(daemonXML, "ProcessID")
+        with open(cpath, 'w', encoding="utf-8") as istream:
+            procStatus = processStatus(cpid)
+            istream.write(json.dumps(procStatus))
+            print("Component %s started with %s threads, see %s\n" % (component, len(procStatus), cpath))
 
     return
     
@@ -274,6 +286,10 @@ def shutdown(configFile):
                 daemon.removeAndBackupDaemonFile()
             else:
                 daemon.killWithPrejudice()
+        # remove component threads.json file
+        cpath = os.path.join(compDir, "threads.json")
+        if os.path.exists(cpath):
+            os.remove(cpath)
         if doLogCleanup:
             #  //
             # // Log Cleanup
@@ -295,7 +311,7 @@ def shutdown(configFile):
             exitCode = subprocess.call(["rm", "-rf", "%s" % compDir])
             if exitCode:
                 msg = "Failed to clean up dir: %s\n" % compDir
-                msg += output
+                msg += f"with exit code {exitCode}"
                 print(msg)
 
     return
@@ -320,18 +336,62 @@ def status(configFile):
             return
         compDir = config.section_(component).componentDir
         compDir = os.path.expandvars(compDir)
-        daemonXml = os.path.join(compDir, "Daemon.xml")
-        if not os.path.exists(daemonXml):
-            print("Component:%s Not Running" % component)
-            continue
-        daemon = Details(daemonXml)
-        if not daemon.isAlive():
-            print("Component:%s Not Running" % component)
-        else:
-            print("Component:%s Running:%s" % (component, daemon['ProcessID']))
+        checkProcessThreads(component, compDir)
 
     sys.exit(0)
 
+def checkProcessThreads(component, compDir):
+    """
+    Helper function to check process and its threads for their statuses
+    :param component: component name
+    :return: prints status of the component process and its threads
+    """
+    # check if component daemon exists
+    daemonXml = os.path.join(compDir, "Daemon.xml")
+    if not os.path.exists(daemonXml):
+        print("Component:%s Not Running" % component)
+        return
+    pid = extractFromXML(daemonXML, "ProcessID")
+
+    jsonFile = os.path.join(compDir, "threads.json")
+    with open(jsonFile, "r", encoding="utf-8") as istream:
+        data = json.load(istream)
+
+    # Extract process and its threads
+    threadPids = []
+
+    for entry in data:
+        if str(entry["pid"]) == str(pid) and entry["type"] == "process":
+            continue
+        elif entry["type"] == "thread":
+            threadPids.append(entry["pid"])
+
+    # Check if process is running
+    processRunning = psutil.pid_exists(int(pid))
+
+    # Check if threads are running
+    runningThreads = []
+    orphanThreads = []
+    process = psutil.Process(int(pid))
+    for thread in process.threads():
+        if str(thread.id) == str(pid):
+            continue  # skip pid thread
+        if str(thread.id) in threadPids:
+            runningThreads.append(thread.id)
+        else:
+            orphanThreads.append(thread.id)
+
+    # Output result
+    status = "running" if processRunning else "not-running"
+    msg = f"Component:{component} {pid} {status} with threads {runningThreads}"
+    if status == "running":
+        if len(orphanThreads) > 0:
+            status = "partially-running" if processRunning else "not-running"
+            msg = f"Component:{component} {pid} {status} with threads {runningThreads}, lost {orphanThreads} threads"
+    else:
+        msg = f"Component:{component} {pid} {status}"
+    print(msg)
+
 def restart(config):
     """
     _restart_

wmglobalqueue-2.4.0rc1/src/python/Utils/ProcFS.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File       : ProcFS.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provides function which rely on Linux proc fs
+"""
+# system modules
+import os
+
+# 3rd party modules
+try:
+    import psutil
+    usePsutil = True
+except ImportError:
+    usePsutil = False
+
+
+def processStatus(pid, method=None):
+    """
+    Return status of the given process PID along with its threads using psutil.
+    :param pid: Process ID to inspect
+    :param method: define which method to use procfs or psutil
+    :return: list of dictionaries of `{process, pid, status}` data-structure
+    """
+    if method == 'procfs' or usePsutil == False:
+        return processStatusViaProcFS(pid)
+    return processStatusViaPsutil(pid)
+
+
+def processStatusViaPsutil(pid):
+    """
+    Return status of the given process PID along with its threads using psutil.
+    :param pid: Process ID to inspect
+    :return: list of dictionaries with {process, pid, status, type}
+    """
+    statusList = []
+
+    try:
+        proc = psutil.Process(pid)
+        processName = proc.name()
+        processStatus = proc.status()
+        statusList.append({
+            "process": processName,
+            "pid": str(pid),
+            "status": processStatus,
+            "type": "process"
+        })
+
+        # Iterate over threads
+        threads = proc.threads()
+        for thread in threads:
+            if str(pid) == str(thread.id):
+                continue
+            threadId = thread.id
+            # psutil doesn't provide thread name or detailed state, so we approximate
+            statusList.append({
+                "process": f"{processName}-thread",
+                "pid": str(threadId),
+                "status": "running",  # threads are assumed running if present
+                "type": "thread"
+            })
+
+    except psutil.NoSuchProcess:
+        return [{"error": f"Process {pid} not found"}]
+    except Exception as e:
+        return [{"error": str(e)}]
+
+    return statusList
+
+
+def processStatusViaProcFS(pid):
+    """
+    Return status of given process PID along with its threads via /proc FS look-up (available on all Linux OSes)
+    :return: list of dictionaries of `{process, pid, status}` data-structure
+    """
+    statusList = []
+    procPath = f"/proc/{pid}"
+
+    if not os.path.exists(procPath):
+        return [{"error": f"Process {pid} not found"}]
+
+    try:
+        with open(f"{procPath}/status", encoding='utf-8') as f:
+            processName = ""
+            processState = ""
+            for line in f:
+                if line.startswith("Name:"):
+                    processName = line.split(":")[1].strip()
+                elif line.startswith("State:"):
+                    processState = line.split(":")[1].strip()
+            statusList.append({"process": processName, "pid": str(pid), "status": processState, "type": "process"})
+
+        taskPath = f"{procPath}/task"
+        if os.path.exists(taskPath):
+            for threadId in os.listdir(taskPath):
+                if str(threadId) == str(pid):
+                    continue
+                with open(f"{taskPath}/{threadId}/status", encoding='utf-8') as f:
+                    threadName = ""
+                    threadState = ""
+                    for line in f:
+                        if line.startswith("Name:"):
+                            threadName = line.split(":")[1].strip()
+                        elif line.startswith("State:"):
+                            threadState = line.split(":")[1].strip()
+                    statusList.append({"process": threadName, "pid": threadId, "status": threadState, "type": "thread"})
+
+    except Exception as e:
+        return [{"error": str(e)}]
+
+    return statusList

wmglobalqueue-2.4.0rc1/src/python/Utils/ProcessStats.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+File       : ProcessStats.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: This module provide ability to get detailed information about
+certain UNIX process, referred by PID. It is based on psutil and threading
+python modules. It can be used in any web/data-server to get live status
+about the running server. Here is an example
+
+.. doctest:
+
+    from Utils.procstats import status threadStack
+
+    class SomeServer(object):
+        def __init__(self):
+        self.time0 = time.time()
+        def status(self):
+            sdict = {'server': processStatus()}
+            sdict['server'].update({'uptime': time.time()-self.time0})
+            sdict.update(threadStack())
+            return sdict
+
+The exposed status endpoint will return a json dictionary containing server
+upteima and cpu/mem/threads information.
+"""
+
+# system modules
+import os
+import sys
+import json
+import time
+import argparse
+import threading
+import traceback
+
+try:
+    import psutil
+except ImportError:
+    pass
+
+def _baseProcessStatusFormat(pid=None):
+    if not pid:
+        pid = os.getpid()
+    ttime = time.localtime()
+    tstamp = time.strftime('%d/%b/%Y:%H:%M:%S', ttime)
+    return {'pid': pid, 'timestamp': tstamp, 'time': time.time()}
+
+def processStatus(pid=None):
+    "Return status of the process in a dictionary format"
+
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update(proc.as_dict())
+    return pdict
+
+def processStatusDict(pid=None):
+    "Return status of the process in a dictionary format"
+    pdict = _baseProcessStatusFormat(pid)
+    if 'psutil' not in sys.modules:
+        return pdict
+    proc = psutil.Process(pid)
+    pdict.update({"cpu_times": dict(proc.cpu_times()._asdict())})
+    pdict.update({"cpu_percent": proc.cpu_percent(interval=1.0)})
+    pdict.update({"cpu_num": proc.cpu_num()})
+    pdict.update({"memory_full_info": dict(proc.memory_full_info()._asdict())})
+    pdict.update({"memory_percent": proc.memory_percent()})
+    return pdict
+
+def threadStack():
+    """
+    Return context of all threads in dictionary format where individual
+    thread information stored in its own dicts and all threads are groupped
+    into threads list. Code based on example from StackOverflow:
+    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
+    """
+    tdict = {}
+    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    threads = []
+    for tid, stack in list(sys._current_frames().items()):
+        tdict = {"thead": id2name.get(tid, ""), "thead_id": tid}
+        stacklist = []
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            sdict = dict(filename=filename, line_number=lineno, name=name, line=line)
+            stacklist.append(sdict)
+        tdict.update({"stack": stacklist})
+        threads.append(tdict)
+    return dict(threads=threads)
+
+def processThreadsInfo(pid):
+    """
+    Provides information about process and its threads using psutils
+    :param pid: process PID, string
+    :return: dictionary statistics about process and threads
+    """
+    try:
+        process = psutil.Process(pid)
+
+        # Get overall process details
+        process_info = {
+            "process_name": process.name(),
+            "pid": process.pid,
+            "status": process.status(),
+            "ppid": process.ppid(),  # Parent Process ID
+            "cmdline": process.cmdline(),  # Full command-line arguments
+            "cpu_usage_percent": process.cpu_percent(interval=1.0),  # CPU usage in percentage
+            "memory_usage_percent": process.memory_percent(),  # RAM usage in percentage
+            "memory_rss": process.memory_info().rss,  # Resident Set Size (bytes)
+            "num_open_files": len(process.open_files()),  # Number of open file descriptors
+            "num_connections": len(process.connections()),  # Number of active connections
+            "threads": []
+        }
+
+        # Iterate over threads to get per-thread details
+        for thread in process.threads():
+            if str(pid) == str(thread.id):
+                continue
+            thread_id = thread.id
+
+            thread_info = {
+                "thread_id": thread_id,
+                "user_time": thread.user_time,  # CPU time spent in user mode
+                "system_time": thread.system_time,  # CPU time spent in system mode
+                "cpu_usage_percent": None,
+                "memory_usage_bytes": None,
+                "num_open_files": None,
+                "num_connections": None,
+                "state": "unknown",
+                "name": "thread"
+            }
+            if str(pid) == str(thread_id):
+                thread_info['name'] = "process"
+
+            # Try getting thread status
+            try:
+                thread_status = psutil.Process(thread_id).status()
+                thread_info["state"] = thread_status
+            except psutil.NoSuchProcess:
+                thread_info["state"] = "zombie"
+
+            # Try getting per-thread CPU usage
+            try:
+                thread_info["cpu_usage_percent"] = psutil.Process(thread_id).cpu_percent(interval=1.0)
+            except Exception:
+                thread_info["cpu_usage_percent"] = "N/A"
+
+            # Try getting per-thread memory usage
+            try:
+                thread_info["memory_usage_bytes"] = psutil.Process(thread_id).memory_info().rss
+            except Exception:
+                thread_info["memory_usage_bytes"] = "N/A"
+
+            # Try getting per-thread open file descriptors
+            try:
+                thread_info["num_open_files"] = len(psutil.Process(thread_id).open_files())
+            except Exception:
+                thread_info["num_open_files"] = "N/A"
+
+            # Try getting per-thread open connections
+            try:
+                thread_info["num_connections"] = len(psutil.Process(thread_id).connections())
+            except Exception:
+                thread_info["num_connections"] = "N/A"
+
+            # Append to process_info
+            process_info["threads"].append(thread_info)
+
+        return process_info
+    except psutil.NoSuchProcess:
+        return {"error": f"No process found with PID {pid}"}
+    except Exception as e:
+        return {"error": str(e)}
+
+def main():
+    "Main function to use this module as a stand-along script."
+    parser = argparse.ArgumentParser(prog='PROG')
+    parser.add_argument("--pid", action="store", dest="pid", help="process id")
+    opts = parser.parse_args()
+
+    pdict = processStatus(int(opts.pid))
+    pdict.update(threadStack())
+    print(f"Process status for {opts.pid}")
+    print(json.dumps(pdict, indent=4))
+
+    processInfo = processThreadsInfo(int(opts.pid))
+    print(f"Process/threads status for {opts.pid}")
+    print(json.dumps(processInfo, indent=4))
+
+if __name__ == '__main__':
+    main()

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/src/python/Utils/Utilities.py

@@ -11,6 +11,16 @@ import sys
 from types import ModuleType, FunctionType
 from gc import get_referents
 
+import xml.etree.ElementTree as ET
+
+def extractFromXML(xmlFile, xmlElement):
+    tree = ET.parse(xmlFile)
+    root = tree.getroot()
+    element = root.find(f".//{xmlElement}")
+    if element is not None:
+        return element.get("Value")
+    return None
+
 def lowerCmsHeaders(headers):
     """
     Lower CMS headers in provided header's dict. The WMCore Authentication

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/src/python/WMCore/Services/DBS/DBSErrors.py

@@ -3,7 +3,6 @@
 DBSErrors represents generic class to handle DBS Go server errors
 """
 
-
 import json
 
 from WMCore.Services.DBS.ProdException import ProdException

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/src/python/WMCore/Services/DBS/DBSUtils.py

@@ -7,12 +7,14 @@ set of common utilities for DBS3Reader
 """
 import json
 import urllib
+import logging
 from urllib.parse import urlparse, parse_qs, quote_plus
 from collections import defaultdict
 
 from Utils.CertTools import cert, ckey
 from dbs.apis.dbsClient import aggFileLumis, aggFileParents
 from WMCore.Services.pycurl_manager import getdata as multi_getdata
+from WMCore.Services.pycurl_manager import RequestHandler
 from Utils.PortForward import PortForward
 
 
@@ -137,3 +139,28 @@ def urlParams(url):
         if len(vals) == 1:
             rdict[key] = vals[0]
     return rdict
+
+def DBSErrors(dbsUrl):
+    """
+    Fetch and return all DBS server errors
+    :param dbsUrl: DBS url to use
+    :return: dictionary of DBS server errors and their meaning
+    """
+    dbsErrors = {}
+    method = "GET"
+    payload = {}
+    headers = {'Content-Type': 'application/json'}
+    mgr = RequestHandler()
+    # dbs server url for error codes
+    rurl = f"{dbsUrl}/errors"
+    data = ""   # initialize data variable to be used in logging.error if necessary
+    try:
+        data = mgr.getdata(rurl, payload, headers, verb=method, ckey=ckey(), cert=cert())
+        # check if we received proper data from DBS server
+        if 'code' in str(data) and 'meaning' in str(data):
+            errorCodes = json.loads(data)
+            for row in errorCodes:
+                dbsErrors[row['code']] = row['meaning']
+    except Exception as exp:
+        logging.error("Unable to query DBS url %s, error %s, data=%s", dbsUrl, str(exp), data)
+    return dbsErrors

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.11.1'
+__version__ = '2.4.0rc1'
 __all__ = []

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1/src/python/wmglobalqueue.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: wmglobalqueue
-Version: 2.3.11.1
+Version: 2.4.0rc1
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{wmglobalqueue-2.3.11.1 → wmglobalqueue-2.4.0rc1}/src/python/wmglobalqueue.egg-info/SOURCES.txt

@@ -755,6 +755,7 @@ src/python/Utils/MemoryCache.py
 src/python/Utils/Patterns.py
 src/python/Utils/Pipeline.py
 src/python/Utils/PortForward.py
+src/python/Utils/ProcFS.py
 src/python/Utils/ProcessStats.py
 src/python/Utils/PythonVersion.py
 src/python/Utils/Signals.py

wmglobalqueue-2.3.11.1/src/python/Utils/ProcessStats.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-File       : ProcessStats.py
-Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
-Description: This module provide ability to get detailed information about
-certain UNIX process, referred by PID. It is based on psutil and threading
-python modules. It can be used in any web/data-server to get live status
-about the running server. Here is an example
-
-.. doctest:
-
-    from Utils.procstats import status threadStack
-
-    class SomeServer(object):
-        def __init__(self):
-        self.time0 = time.time()
-        def status(self):
-            sdict = {'server': processStatus()}
-            sdict['server'].update({'uptime': time.time()-self.time0})
-            sdict.update(threadStack())
-            return sdict
-
-The exposed status endpoint will return a json dictionary containing server
-upteima and cpu/mem/threads information.
-"""
-
-# system modules
-import os
-import sys
-import json
-import time
-import argparse
-import threading
-import traceback
-
-try:
-    import psutil
-except ImportError:
-    pass
-
-def _baseProcessStatusFormat(pid=None):
-    if not pid:
-        pid = os.getpid()
-    ttime = time.localtime()
-    tstamp = time.strftime('%d/%b/%Y:%H:%M:%S', ttime)
-    return {'pid': pid, 'timestamp': tstamp, 'time': time.time()}
-
-def processStatus(pid=None):
-    "Return status of the process in a dictionary format"
-
-    pdict = _baseProcessStatusFormat(pid)
-    if 'psutil' not in sys.modules:
-        return pdict
-    proc = psutil.Process(pid)
-    pdict.update(proc.as_dict())
-    return pdict
-
-def processStatusDict(pid=None):
-    "Return status of the process in a dictionary format"
-    pdict = _baseProcessStatusFormat(pid)
-    if 'psutil' not in sys.modules:
-        return pdict
-    proc = psutil.Process(pid)
-    pdict.update({"cpu_times": dict(proc.cpu_times()._asdict())})
-    pdict.update({"cpu_percent": proc.cpu_percent(interval=1.0)})
-    pdict.update({"cpu_num": proc.cpu_num()})
-    pdict.update({"memory_full_info": dict(proc.memory_full_info()._asdict())})
-    pdict.update({"memory_percent": proc.memory_percent()})
-    return pdict
-
-def threadStack():
-    """
-    Return context of all threads in dictionary format where individual
-    thread information stored in its own dicts and all threads are groupped
-    into threads list. Code based on example from StackOverflow:
-    http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
-    """
-    tdict = {}
-    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
-    threads = []
-    for tid, stack in list(sys._current_frames().items()):
-        tdict = {"thead": id2name.get(tid, ""), "thead_id": tid}
-        stacklist = []
-        for filename, lineno, name, line in traceback.extract_stack(stack):
-            sdict = dict(filename=filename, line_number=lineno, name=name, line=line)
-            stacklist.append(sdict)
-        tdict.update({"stack": stacklist})
-        threads.append(tdict)
-    return dict(threads=threads)
-
-def main():
-    "Main function to use this module as a stand-along script."
-    parser = argparse.ArgumentParser(prog='PROG')
-    parser.add_argument("--pid", action="store", dest="pid", help="process id")
-    opts = parser.parse_args()
-
-    pdict = processStatus(int(opts.pid))
-    pdict.update(threadStack())
-    print(json.dumps(pdict))
-
-if __name__ == '__main__':
-    main()