panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedicore/JediTaskBufferInterface.py

@@ -0,0 +1,50 @@
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore import Interaction
+
+
+# interface to JediTaskBuffer
+class JediTaskBufferInterface:
+    # constructor
+    def __init__(self):
+        self.interface = None
+
+    # setup interface
+    def setupInterface(self, max_size=None):
+        vo = "any"
+        maxSize = max_size if max_size is not None else jedi_config.db.nWorkers
+        moduleName = "pandajedi.jedicore.JediTaskBuffer"
+        className = "JediTaskBuffer"
+        self.interface = Interaction.CommandSendInterface(vo, maxSize, moduleName, className)
+        self.interface.initialize()
+
+    # method emulation
+    def __getattr__(self, attrName):
+        return getattr(self.interface, attrName)
+
+
+if __name__ == "__main__":
+
+    def dummyClient(dif, stime):
+        print("client test")
+        import time
+
+        for i in range(3):
+            # time.sleep(i*stime)
+            try:
+                print(dif.getCloudList())
+            except Exception:
+                print("exp")
+        print("client done")
+
+    dif = JediTaskBufferInterface()
+    dif.setupInterface()
+    print("master test")
+    print(dif.getCloudList())
+    print("master done")
+    import multiprocessing
+
+    pList = []
+    for i in range(5):
+        p = multiprocessing.Process(target=dummyClient, args=(dif, i))
+        pList.append(p)
+        p.start()

pandajedi/jedicore/MsgWrapper.py

@@ -0,0 +1,124 @@
+import re
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandajedi.jediconfig import jedi_config
+from pandaserver.userinterface import Client
+
+
+class MsgWrapper:
+    def __init__(self, logger, token=None, lineLimit=500, monToken=None):
+        self.logger = logger
+        # use timestamp as token if undefined
+        if token is None:
+            self.token = f"<{naive_utcnow().isoformat('/')}>"
+        else:
+            self.token = token
+        # token for http logger
+        if monToken is None:
+            self.monToken = self.token
+        else:
+            self.monToken = monToken
+        # remove <> for django
+        try:
+            self.monToken = re.sub("<(?P<name>[^>]+)>", "\g<name>", self.monToken)
+        except Exception:
+            pass
+        # message buffer
+        self.msgBuffer = []
+        self.bareMsg = []
+        self.lineLimit = lineLimit
+        self.message_slot = None
+
+    def keepMsg(self, msg):
+        # check if message slot is defined and available
+        if self.message_slot is not None:
+            if self.message_slot < 0:
+                return
+            if self.message_slot == 0:
+                msg = "   ..."
+            self.message_slot -= 1
+        # keep max message depth
+        if len(self.msgBuffer) > self.lineLimit:
+            self.msgBuffer.pop(0)
+            self.bareMsg.pop(0)
+        timeNow = naive_utcnow()
+        self.msgBuffer.append(f"{timeNow.isoformat(' ')} : {msg}")
+        self.bareMsg.append(msg)
+
+    def set_message_slot(self, slot: int = 10):
+        self.message_slot = slot
+
+    def unset_message_slot(self):
+        self.message_slot = None
+
+    def info(self, msg):
+        msg = str(msg)
+        self.logger.info(self.token + " " + msg)
+        self.keepMsg(msg)
+
+    def debug(self, msg):
+        msg = str(msg)
+        self.logger.debug(self.token + " " + msg)
+
+    def error(self, msg):
+        msg = str(msg)
+        self.logger.error(self.token + " " + msg)
+        self.keepMsg(msg)
+
+    def warning(self, msg):
+        msg = str(msg)
+        self.logger.warning(self.token + " " + msg)
+        self.keepMsg(msg)
+
+    def dumpToString(self):
+        strMsg = ""
+        for msg in self.msgBuffer:
+            strMsg += msg
+            strMsg += "\n"
+        return strMsg
+
+    def uploadLog(self, id):
+        strMsg = self.dumpToString()
+        s, o = Client.uploadLog(strMsg, id)
+        if s != 0:
+            return f"failed to upload log with {s} {o}."
+
+        success = o["success"]
+        message = o["message"]
+        url = o["data"]
+
+        if success and url.startswith("http"):
+            return f"<a href=\"{url}\">log</a> : {'. '.join(self.bareMsg[-2:])}."
+
+        return message
+
+    # send message to logger
+    def sendMsg(self, message, msgType, msgLevel="info", escapeChar=False):
+        try:
+            # get logger
+            tmpPandaLogger = PandaLogger()
+            # lock HTTP handler
+            tmpPandaLogger.lock()
+            tmpPandaLogger.setParams({"Type": msgType})
+            # get logger
+            tmpLogger = tmpPandaLogger.getHttpLogger(jedi_config.master.loggername)
+            # escape special characters
+            if escapeChar:
+                message = message.replace("<", "&lt;")
+                message = message.replace(">", "&gt;")
+            # add message
+            message = self.monToken + " " + message
+            if msgLevel == "error":
+                tmpLogger.error(message)
+            elif msgLevel == "warning":
+                tmpLogger.warning(message)
+            elif msgLevel == "info":
+                tmpLogger.info(message)
+            else:
+                tmpLogger.debug(message)
+            # release HTTP handler
+            tmpPandaLogger.release()
+        except Exception:
+            pass

pandajedi/jedicore/ProcessUtils.py

@@ -0,0 +1,34 @@
+import datetime
+import multiprocessing
+import sys
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandaserver.srvcore import CoreUtils
+
+
+# wrapper for multiprocessing.Process
+class ProcessWrapper(multiprocessing.Process):
+    # constructor
+    def __init__(self, target, args):
+        multiprocessing.Process.__init__(self, target=self.wrappedMain)
+        self.target = target
+        self.args = args
+
+    # main
+    def wrappedMain(self):
+        while True:
+            proc = multiprocessing.Process(target=self.target, args=self.args)
+            proc.start()
+            pid = proc.pid
+            while True:
+                try:
+                    proc.join(20)
+                    if not CoreUtils.checkProcess(pid):
+                        timeNow = naive_utcnow()
+                        print(f"{str(timeNow)} {self.__class__.__name__}: INFO    pid={pid} not exist")
+                        break
+                except Exception:
+                    timeNow = naive_utcnow()
+                    errType, errValue = sys.exc_info()[:2]
+                    print(f"{str(timeNow)} {self.__class__.__name__}: INFO    failed to check pid={pid} with {errType} {errValue}")

pandajedi/jedicore/SiteCandidate.py

@@ -0,0 +1,90 @@
+class SiteCandidate(object):
+    def __init__(self, siteName, unifiedName=None):
+        # the site name
+        self.siteName = siteName
+        # unified name
+        self.unifiedName = unifiedName
+        # the weight for the brokerage
+        self.weight = 0
+        # the list of files copied from SE disk
+        self.localDiskFiles = set()
+        # the list of files copied from SE tape
+        self.localTapeFiles = set()
+        # the list of files cached in non-SE, e.g. on CVMFS
+        self.cacheFiles = set()
+        # the list of files read from SE using remote I/O
+        self.remoteFiles = set()
+        # the list of all files
+        self.allFiles = None
+        # remote access protocol
+        self.remoteProtocol = None
+        # remote source if any
+        self.remoteSource = None
+        # number of running job
+        self.nRunningJobs = None
+        # number of queued jobs
+        self.nQueuedJobs = None
+        # number of assigned jobs
+        self.nAssignedJobs = None
+        # cap on running jobs
+        self.nRunningJobsCap = None
+        # overridden attributes
+        self.overriddenAttrs = {}
+
+    # add local disk files
+    def add_local_disk_files(self, files):
+        self.localDiskFiles = self.localDiskFiles.union([f.fileID for f in files])
+
+    # add local tape files
+    def add_local_tape_files(self, files):
+        self.localTapeFiles = self.localTapeFiles.union([f.fileID for f in files])
+
+    # add cache files
+    def add_cache_files(self, files):
+        self.cacheFiles = self.cacheFiles.union([f.fileID for f in files])
+
+    # add remote files
+    def add_remote_files(self, files):
+        self.remoteFiles = self.remoteFiles.union([f.fileID for f in files])
+
+    # get locality of a file
+    def getFileLocality(self, fileSpec):
+        if fileSpec.fileID in self.localDiskFiles:
+            return "localdisk"
+        if fileSpec.fileID in self.localTapeFiles:
+            return "localtape"
+        if fileSpec.fileID in self.cacheFiles:
+            return "cache"
+        if fileSpec.fileID in self.remoteFiles:
+            return "remote"
+        return None
+
+    # add available files
+    def addAvailableFiles(self, fileList):
+        if self.allFiles is None:
+            self.allFiles = set()
+        for tmpFileSpec in fileList:
+            self.allFiles.add(tmpFileSpec.fileID)
+
+    # check if file is available
+    def isAvailableFile(self, tmpFileSpec):
+        # N/A
+        if self.allFiles is None:
+            return True
+        return tmpFileSpec.fileID in self.allFiles
+
+    # check if still can accept jobs
+    def can_accept_jobs(self):
+        if self.nRunningJobsCap is None or self.nQueuedJobs is None:
+            return True
+        if self.nRunningJobsCap > self.nQueuedJobs:
+            return True
+        return False
+
+    # override an attribute
+    def override_attribute(self, key, value):
+        self.overriddenAttrs[key] = value
+
+    # get an overridden attribute
+    def get_overridden_attribute(self, key):
+        return self.overriddenAttrs.get(key)

pandajedi/jedicore/ThreadUtils.py

@@ -0,0 +1,205 @@
+import multiprocessing
+import sys
+import threading
+import time
+
+
+# list with lock
+class ListWithLock:
+    def __init__(self, dataList):
+        self.lock = threading.Lock()
+        self.dataList = dataList
+        self.dataIndex = 0
+
+    def __iter__(self):
+        return self
+
+    def __contains__(self, item):
+        self.lock.acquire()
+        ret = self.dataList.__contains__(item)
+        self.lock.release()
+        return ret
+
+    def __next__(self):
+        if self.dataIndex >= len(self.dataList):
+            self.dataIndex = 0
+            raise StopIteration
+        val = self.dataList[self.dataIndex]
+        self.dataIndex += 1
+        return val
+
+    def next(self):
+        return self.__next__()
+
+    def append(self, item):
+        self.lock.acquire()
+        appended = False
+        if item not in self.dataList:
+            self.dataList.append(item)
+            appended = True
+        self.lock.release()
+        return appended
+
+    def get(self, num):
+        self.lock.acquire()
+        retList = self.dataList[self.dataIndex : self.dataIndex + num]
+        self.dataIndex += len(retList)
+        self.lock.release()
+        return retList
+
+    def stat(self):
+        self.lock.acquire()
+        total = len(self.dataList)
+        nIndx = self.dataIndex
+        self.lock.release()
+        return total, nIndx
+
+    def __len__(self):
+        self.lock.acquire()
+        ret = len(self.dataList)
+        self.lock.release()
+        return ret
+
+    def dump(self):
+        self.lock.acquire()
+        if len(self.dataList) > self.dataIndex:
+            ret = ",".join(map(str, self.dataList[self.dataIndex :]))
+        else:
+            ret = "None"
+        self.lock.release()
+        return ret
+
+
+# map with lock
+class MapWithLock:
+    def __init__(self, dataMap=None):
+        self.lock = threading.Lock()
+        if dataMap is None:
+            dataMap = {}
+        self.dataMap = dataMap
+
+    def __getitem__(self, item):
+        ret = self.dataMap.__getitem__(item)
+        return ret
+
+    def __setitem__(self, item, value):
+        self.dataMap.__setitem__(item, value)
+
+    def __contains__(self, item):
+        ret = self.dataMap.__contains__(item)
+        return ret
+
+    def acquire(self):
+        self.lock.acquire()
+
+    def release(self):
+        self.lock.release()
+
+    def add(self, item, value):
+        if item not in self.dataMap:
+            self.dataMap[item] = 0
+        self.dataMap[item] += value
+
+    def get(self, item):
+        if item not in self.dataMap:
+            return 0
+        return self.dataMap[item]
+
+    def items(self):
+        return self.dataMap.items()
+
+    def iteritems(self):
+        return self.items()
+
+
+# thread pool
+class ThreadPool:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.list = []
+
+    # add thread
+    def add(self, obj):
+        self.lock.acquire()
+        self.list.append(obj)
+        self.lock.release()
+
+    # remove thread
+    def remove(self, obj):
+        self.lock.acquire()
+        try:
+            self.list.remove(obj)
+        except Exception:
+            pass
+        self.lock.release()
+
+    # join
+    def join(self, timeOut=None):
+        thrlist = tuple(self.list)
+        for thr in thrlist:
+            try:
+                thr.join(timeOut)
+                if thr.is_alive():
+                    break
+            except Exception:
+                pass
+
+    # remove inactive threads
+    def clean(self):
+        thrlist = tuple(self.list)
+        for thr in thrlist:
+            if not thr.is_alive():
+                self.remove(thr)
+
+    # dump contents
+    def dump(self):
+        thrlist = tuple(self.list)
+        nActv = 0
+        for thr in thrlist:
+            if thr.is_alive():
+                nActv += 1
+        return f"nActive={nActv}"
+
+
+# thread class working with semaphore and thread pool
+class WorkerThread(threading.Thread):
+    # constructor
+    def __init__(self, workerSemaphore, threadPool, logger):
+        threading.Thread.__init__(self)
+        self.workerSemaphore = workerSemaphore
+        self.threadPool = threadPool
+        if self.threadPool is not None:
+            self.threadPool.add(self)
+        self.logger = logger
+
+    # main loop
+    def run(self):
+        # get slot
+        if self.workerSemaphore is not None:
+            self.workerSemaphore.acquire()
+        # execute real work
+        try:
+            self.runImpl()
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            self.logger.error(f"{self.__class__.__name__} crashed in WorkerThread.run() with {errtype.__name__}:{errvalue}")
+        # remove self from thread pool
+        if self.threadPool is not None:
+            self.threadPool.remove(self)
+        # release slot
+        if self.workerSemaphore is not None:
+            self.workerSemaphore.release()
+
+
+# thread class to cleanup zombi processes
+class ZombieCleaner(threading.Thread):
+    # constructor
+    def __init__(self, interval=20):
+        threading.Thread.__init__(self)
+        self.interval = interval
+
+    # main loop
+    def run(self):
+        while True:
+            x = multiprocessing.active_children()
+            time.sleep(self.interval)