panda-server-source 0.8.0__py3-none-any.whl

pandaserver/daemons/scripts/panda_activeusers_query.py

@@ -0,0 +1,80 @@
+import sys
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.config import panda_config
+from pandaserver.proxycache import panda_proxy_cache, token_cache
+from pandaserver.srvcore import CoreUtils
+
+# logger
+_logger = PandaLogger().getLogger("activeusers_query")
+
+
+# main
+def main(tbuf=None, **kwargs):
+    # logger
+    tmpLog = LogWrapper(_logger)
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+
+    tmpLog.debug("================= start ==================")
+    # instantiate TB
+    if tbuf is None:
+        tmpLog.debug("Getting new connection")
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            useTimeout=True,
+            requester=requester_id,
+        )
+        tmpLog.debug("Getting new connection - done")
+    else:
+        tmpLog.debug("Reusing connection")
+        taskBuffer = tbuf
+
+    # instantiate MyProxy I/F
+    my_proxy_interface_instance = panda_proxy_cache.MyProxyInterface()
+
+    # roles
+    if hasattr(panda_config, "proxy_cache_roles"):
+        roles = panda_config.proxy_cache_roles.split(",")
+    else:
+        roles = ["atlas", "atlas:/atlas/Role=production", "atlas:/atlas/Role=pilot"]
+
+    # get users
+    sql = "select distinct DN FROM ATLAS_PANDAMETA.users WHERE GRIDPREF LIKE :patt"
+    varMap = {}
+    varMap[":patt"] = "%p%"
+    tmpLog.debug("Querying users")
+    tmpStat, tmpRes = taskBuffer.querySQLS(sql, varMap)
+    tmpLog.debug("Querying done")
+    for (realDN,) in tmpRes:
+        if realDN is None:
+            continue
+        realDN = CoreUtils.get_bare_dn(realDN, keep_digits=False)
+        name = CoreUtils.clean_user_id(realDN)
+        # check proxy
+        tmpLog.debug(f"check proxy cache for {name}")
+        for role in roles:
+            my_proxy_interface_instance.checkProxy(realDN, role=role, name=name)
+
+    # instantiate Token Cache
+    tmpLog.debug("Token Cache start")
+    token_cacher = token_cache.TokenCache(task_buffer=taskBuffer)
+    token_cacher.run()
+    tmpLog.debug("Token Cache done")
+
+    # stop taskBuffer if created inside this script
+    if tbuf is None:
+        taskBuffer.cleanup(requester=requester_id)
+        tmpLog.debug("Stopped the new connection")
+    tmpLog.debug("done")
+
+
+# run
+if __name__ == "__main__":
+    main()

pandaserver/daemons/scripts/pilotStreaming.py

@@ -0,0 +1,109 @@
+import sys
+import time
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.config import panda_config
+
+_logger = PandaLogger().getLogger("pilot_streaming")
+
+
+class PilotStreaming(object):
+    def __init__(self, tbuf):
+        self._logger = _logger
+        self.tbuf = tbuf
+        return
+
+    def run(self):
+        """
+        Gets and iterates over ups queues, deciding the job requirements and sending these to Harvester
+        via the command interface
+        :return:
+        """
+
+        # timing
+        time_start = time.time()
+        self._logger.debug("Start.")
+
+        # get unified pilot streaming (ups) queues
+        ups_queues = self.tbuf.ups_get_queues()
+        self._logger.debug(f"UPS queues: {ups_queues}")
+
+        # load the worker stats from the database
+        worker_stats = self.tbuf.ups_load_worker_stats()
+
+        # iterate over the UPS queues
+        for ups_queue in ups_queues:
+            # separate the worker and job stats for the queue
+            try:
+                tmp_worker_stats = worker_stats[ups_queue]
+                self._logger.debug(f"worker_stats for queue {ups_queue}: {tmp_worker_stats}")
+            except KeyError:
+                # skip queue if no data available
+                self._logger.debug(f"No worker stats for queue {ups_queue}")
+                continue
+
+            # calculate the new worker distribution and save the harvester commands in the database
+            try:
+                new_workers_per_harvester = self.tbuf.ups_new_worker_distribution(ups_queue, tmp_worker_stats)
+                self._logger.info(f"queue: {ups_queue}, results: {new_workers_per_harvester}")
+
+                # variables for the harvester command
+                command = f"SET_N_WORKERS_JOBTYPE:{ups_queue}"
+                status = "new"
+                ack_requested = False
+                lock_interval = None
+                com_interval = None
+
+                for harvester_id in new_workers_per_harvester:
+                    params = new_workers_per_harvester[harvester_id]
+                    self.tbuf.commandToHarvester(
+                        harvester_id,
+                        command,
+                        ack_requested,
+                        status,
+                        lock_interval,
+                        com_interval,
+                        params,
+                    )
+            except Exception:
+                self._logger.error(traceback.format_exc())
+
+        # log the timing
+        time_stop = time.time()
+        self._logger.debug(f"Done. Pilot streaming took: {time_stop - time_start} s")
+
+        return
+
+
+# main
+def main(tbuf=None, **kwargs):
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+
+    # instantiate TB
+    if tbuf is None:
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            useTimeout=True,
+            requester=requester_id,
+        )
+    else:
+        taskBuffer = tbuf
+
+    # run the pilot streaming logic
+    PilotStreaming(tbuf=taskBuffer).run()
+
+    # stop the taskBuffer if it was created inside this script
+    if tbuf is None:
+        taskBuffer.cleanup(requester=requester_id)
+
+
+# run
+if __name__ == "__main__":
+    main()

pandaserver/daemons/scripts/process_workflow_files_daemon.py

@@ -0,0 +1,166 @@
+import datetime
+import glob
+import os.path
+import sys
+import threading
+import time
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.workflow_processor import WorkflowProcessor
+
+# logger
+_logger = PandaLogger().getLogger("process_workflow_files")
+
+
+# main
+def main(tbuf=None, **kwargs):
+    _logger.debug("===================== start =====================")
+
+    # overall timeout value
+    overallTimeout = 300
+    # prefix of the files
+    if "target" in kwargs and kwargs["target"]:
+        evpFilePatt = kwargs["target"]
+    else:
+        prefixEVP = "/workflow."
+        # file pattern of evp files
+        evpFilePatt = panda_config.cache_dir + "/" + prefixEVP + "*"
+
+    from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+    taskBuffer.init(
+        panda_config.dbhost,
+        panda_config.dbpasswd,
+        nDBConnection=1,
+        useTimeout=True,
+        requester=requester_id,
+    )
+
+    test_mode = kwargs.get("test_mode", False)
+    dump_workflow = kwargs.get("dump_workflow", False)
+
+    # thread pool
+    class ThreadPool:
+        def __init__(self):
+            self.lock = threading.Lock()
+            self.list = []
+
+        def add(self, obj):
+            self.lock.acquire()
+            self.list.append(obj)
+            self.lock.release()
+
+        def remove(self, obj):
+            self.lock.acquire()
+            self.list.remove(obj)
+            self.lock.release()
+
+        def join(self):
+            self.lock.acquire()
+            thrlist = tuple(self.list)
+            self.lock.release()
+            for thr in thrlist:
+                thr.join()
+
+    # thread
+    class EvpThr(threading.Thread):
+        def __init__(self, task_buffer, lock, pool, file_name, to_delete, get_log):
+            threading.Thread.__init__(self)
+            self.lock = lock
+            self.pool = pool
+            self.fileName = file_name
+            self.to_delete = to_delete
+            self.get_log = get_log
+            self.pool.add(self)
+            self.processor = WorkflowProcessor(task_buffer=task_buffer, log_stream=_logger)
+
+        def run(self):
+            self.lock.acquire()
+            try:
+                self.processor.process(
+                    self.fileName,
+                    self.to_delete,
+                    test_mode,
+                    self.get_log,
+                    dump_workflow,
+                )
+            except Exception as e:
+                _logger.error(f"{str(e)} {traceback.format_exc()}")
+            self.pool.remove(self)
+            self.lock.release()
+
+    # get files
+    timeNow = naive_utcnow()
+    timeInt = naive_utcnow()
+    fileList = sorted(glob.glob(evpFilePatt))
+
+    # create thread pool and semaphore
+    adderLock = threading.Semaphore(1)
+    adderThreadPool = ThreadPool()
+
+    # add
+    while len(fileList) != 0:
+        # time limit to aviod too many copyArchve running at the sametime
+        if (naive_utcnow() - timeNow) > datetime.timedelta(minutes=overallTimeout):
+            _logger.debug("time over in main session")
+            break
+        # try to get Semaphore
+        adderLock.acquire()
+        # get fileList
+        if (naive_utcnow() - timeInt) > datetime.timedelta(minutes=15):
+            timeInt = naive_utcnow()
+            # get file
+            fileList = sorted(glob.glob(evpFilePatt))
+        # choose a file
+        fileName = fileList.pop(0)
+        # release lock
+        adderLock.release()
+        if not os.path.exists(fileName):
+            continue
+        try:
+            modTime = datetime.datetime(*(time.gmtime(os.path.getmtime(fileName))[:7]))
+            to_go = True
+            if test_mode:
+                _logger.debug(f"Testing : {fileName}")
+                to_delete = False
+            elif (timeNow - modTime) > datetime.timedelta(hours=2):
+                # last chance
+                _logger.debug(f"Last attempt : {fileName}")
+                to_delete = True
+            elif (timeInt - modTime) > datetime.timedelta(seconds=5):
+                # try
+                _logger.debug(f"Normal attempt : {fileName}")
+                to_delete = False
+            else:
+                _logger.debug(f"Wait {timeInt - modTime} : {fileName}")
+                to_go = False
+            if to_go:
+                thr = EvpThr(taskBuffer, adderLock, adderThreadPool, fileName, to_delete, False)
+                thr.start()
+        except Exception as e:
+            _logger.error(f"{str(e)} {traceback.format_exc()}")
+
+    # join all threads
+    adderThreadPool.join()
+
+    # stop taskBuffer if created inside this script
+    taskBuffer.cleanup(requester=requester_id)
+
+    _logger.debug("===================== end =====================")
+
+
+# run
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) > 1:
+        data = {"target": sys.argv[1], "test_mode": True, "dump_workflow": True}
+    else:
+        data = {}
+    main(**data)

pandaserver/daemons/scripts/recover_lost_files_daemon.py

@@ -0,0 +1,177 @@
+import datetime
+import fcntl
+import glob
+import json
+import os.path
+import sys
+import threading
+import time
+import traceback
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice import RecoverLostFilesCore
+
+# logger
+_logger = PandaLogger().getLogger("recover_lost_files")
+
+
+# main
+def main(tbuf=None, **kwargs):
+    _logger.debug("===================== start =====================")
+
+    # overall timeout value
+    overallTimeout = 300
+    # prefix of the files
+    prefixEVP = "recov."
+    # file pattern of evp files
+    evpFilePatt = panda_config.cache_dir + "/" + prefixEVP + "*"
+
+    from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+    taskBuffer.init(
+        panda_config.dbhost,
+        panda_config.dbpasswd,
+        nDBConnection=1,
+        useTimeout=True,
+        requester=requester_id,
+    )
+
+    # thread pool
+    class ThreadPool:
+        def __init__(self):
+            self.lock = threading.Lock()
+            self.list = []
+
+        def add(self, obj):
+            self.lock.acquire()
+            self.list.append(obj)
+            self.lock.release()
+
+        def remove(self, obj):
+            self.lock.acquire()
+            self.list.remove(obj)
+            self.lock.release()
+
+        def join(self):
+            self.lock.acquire()
+            thrlist = tuple(self.list)
+            self.lock.release()
+            for thr in thrlist:
+                thr.join()
+
+    # thread to ev-pd2p
+    class EvpThr(threading.Thread):
+        def __init__(self, lock, pool, tb_if, file_name, to_delete):
+            threading.Thread.__init__(self)
+            self.lock = lock
+            self.pool = pool
+            self.fileName = file_name
+            self.to_delete = to_delete
+            self.taskBuffer = tb_if
+            self.pool.add(self)
+
+        def run(self):
+            base_log = LogWrapper(_logger, self.fileName)
+            base_log.debug("start processing")
+            self.lock.acquire()
+            try:
+                with open(self.fileName) as f:
+                    try:
+                        # acquire an exclusive file lock
+                        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        base_log.debug("file lock acquired")
+                        ops = json.load(f)
+                        tag = ops.get("jediTaskID")
+                        if tag:
+                            tag = f"< jediTaskID={tag} >"
+                        else:
+                            tag = ops.get("dataset")
+                            tag = f"< dataset={tag} >"
+                        tmp_log = LogWrapper(_logger, tag)
+                        tmp_log.info(f"start {self.fileName}")
+                        s, o = RecoverLostFilesCore.main(self.taskBuffer, ops, tmp_log)
+                        tmp_log.info(f"status={s}. {o}")
+                        if s is not None or self.to_delete:
+                            tmp_log.debug(f"delete {self.fileName}")
+                            try:
+                                os.remove(self.fileName)
+                            except Exception:
+                                pass
+                    finally:
+                        # release the file lock
+                        base_log.debug("release file lock")
+                        try:
+                            fcntl.flock(f, fcntl.LOCK_EX)
+                        except Exception:
+                            pass
+            except FileNotFoundError:
+                pass
+            except Exception as e:
+                base_log.error(f"process failure with {str(e)} {traceback.format_exc()}")
+            self.pool.remove(self)
+            self.lock.release()
+            base_log.debug("end processing")
+
+    # get files
+    timeNow = naive_utcnow()
+    timeInt = naive_utcnow()
+    fileList = sorted(glob.glob(evpFilePatt))
+
+    # create thread pool and semaphore
+    adderLock = threading.Semaphore(1)
+    adderThreadPool = ThreadPool()
+
+    # add
+    while len(fileList) != 0:
+        # time limit to aviod too many copyArchve running at the sametime
+        if (naive_utcnow() - timeNow) > datetime.timedelta(minutes=overallTimeout):
+            _logger.debug("time over in main session")
+            break
+        # try to get Semaphore
+        adderLock.acquire()
+        # get fileList
+        if (naive_utcnow() - timeInt) > datetime.timedelta(minutes=15):
+            timeInt = naive_utcnow()
+            # get file
+            fileList = sorted(glob.glob(evpFilePatt))
+        # choose a file
+        fileName = fileList.pop(0)
+        # release lock
+        adderLock.release()
+        if not os.path.exists(fileName):
+            continue
+        try:
+            modTime = datetime.datetime(*(time.gmtime(os.path.getmtime(fileName))[:7]))
+            if (timeNow - modTime) > datetime.timedelta(hours=2):
+                # last chance
+                _logger.debug(f"Last attempt : {fileName}")
+                thr = EvpThr(adderLock, adderThreadPool, taskBuffer, fileName, False)
+                thr.start()
+            elif (timeInt - modTime) > datetime.timedelta(seconds=5):
+                # try
+                _logger.debug(f"Normal attempt : {fileName}")
+                thr = EvpThr(adderLock, adderThreadPool, taskBuffer, fileName, True)
+                thr.start()
+            else:
+                _logger.debug(f"Wait {timeInt - modTime} : {fileName}")
+        except Exception as e:
+            _logger.error(f"{str(e)} {traceback.format_exc()}")
+
+    # join all threads
+    adderThreadPool.join()
+
+    # stop taskBuffer if created inside this script
+    taskBuffer.cleanup(requester=requester_id)
+
+    _logger.debug("===================== end =====================")
+
+
+# run
+if __name__ == "__main__":
+    main()