panda-server-source 0.8.0__py3-none-any.whl

pandaserver/configurator/aux.py

@@ -0,0 +1,73 @@
+import json
+import os
+import time
+from urllib.parse import urlparse
+
+import requests
+
+from pandaserver.config import panda_config
+
+GB = 1024**3
+PROD_INPUT = "Production Input"
+PROD_OUTPUT = "Production Output"
+EXPRESS = "Express"
+FILES = "files"
+MBPS = "mbps"
+LATENCY = "latency"
+PACKETLOSS = "packetloss"
+DONE = "done"
+QUEUED = "queued"
+LATEST = "latest"
+H1 = "1h"
+H6 = "6h"
+D1 = "1d"
+W1 = "1w"
+TIMESTAMP = "timestamp"
+
+REQUESTS_TIMEOUT = 30
+
+
+def get_dump(url):
+    """
+    Retrieves a json file from the given URL and loads it into memory
+    """
+    # check if local file
+    p = urlparse(url)
+    if not p.scheme or p.scheme == "file":
+        try:
+            with open(p.path) as f:
+                return json.load(f)
+        except Exception:
+            return None
+    if panda_config.configurator_use_cert:
+        key_file = os.environ["X509_USER_PROXY"]
+        cert_file = os.environ["X509_USER_PROXY"]
+        ca_certs = os.environ["X509_CERT_DIR"]
+        cert = (cert_file, key_file)
+    else:
+        cert = None
+        ca_certs = False
+    for _ in range(1, 4):  # 3 retries
+        try:
+            r = requests.get(url, cert=cert, verify=ca_certs, timeout=REQUESTS_TIMEOUT)
+            if r.status_code == requests.codes.ok:
+                return r.json()
+        except Exception:
+            time.sleep(1)
+    return None
+
+
+def query_grafana_proxy(query, bearer_token):
+    headers = {
+        "Content-Type": "application/x-ndjson",
+        "Authorization": f"Bearer {bearer_token}",
+    }
+    grafana_proxy_url = "https://monit-grafana.cern.ch/api/datasources/proxy/10349/_msearch"
+    for _ in range(1, 4):  # 3 retries
+        try:
+            r = requests.post(grafana_proxy_url, data=query, headers=headers)
+            if r.status_code == requests.codes.ok:
+                return r.json()
+        except Exception:
+            time.sleep(1)
+    return None

pandaserver/daemons/master.py

@@ -0,0 +1,110 @@
+import argparse
+import grp
+import logging
+import os
+import pwd
+import signal
+import sys
+
+import daemon
+import lockfile
+from pandaserver.config import daemon_config, panda_config
+from pandaserver.daemons.utils import END_SIGNALS, DaemonMaster
+
+
+# get the logger
+def get_logger():
+    my_logger = logging.getLogger("PanDA-Daemon-Master")
+    # remove existing handlers
+    while my_logger.hasHandlers():
+        my_logger.removeHandler(my_logger.handlers[0])
+    # make new handler
+    _log_handler = logging.StreamHandler(sys.stdout)
+    _log_formatter = logging.Formatter("%(asctime)s %(name)-12s: %(levelname)-8s %(message)s")
+    _log_handler.setFormatter(_log_formatter)
+    # add new handler
+    my_logger.addHandler(_log_handler)
+    # debug log level
+    my_logger.setLevel(logging.DEBUG)
+    # return logger
+    return my_logger
+
+
+# kill the whole process group
+def kill_whole():
+    os.killpg(os.getpgrp(), signal.SIGKILL)
+
+
+# main function
+def main():
+    # whether to run daemons
+    if not getattr(daemon_config, "enable", False):
+        return
+    # get logger
+    main_log = get_logger()
+    # parse option
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-P",
+        "--pidfile",
+        action="store",
+        dest="pidfile",
+        default=None,
+        help="pid filename",
+    )
+    options = parser.parse_args()
+    if "PANDA_NO_ROOT" in os.environ:
+        uid = None
+        gid = None
+    else:
+        uname = getattr(daemon_config, "uname", "nobody")
+        gname = getattr(daemon_config, "gname", "nobody")
+        uid = pwd.getpwnam(uname).pw_uid
+        gid = grp.getgrnam(gname).gr_gid
+    n_workers = getattr(daemon_config, "n_proc", 1)
+    n_dbconn = getattr(daemon_config, "n_dbconn", 1)
+    worker_lifetime = getattr(daemon_config, "proc_lifetime", 28800)
+    use_tbif = getattr(daemon_config, "use_tbif", False)
+    main_log.info("main start")
+    # daemon context
+    dc = daemon.DaemonContext(
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        uid=uid,
+        gid=gid,
+        pidfile=lockfile.FileLock(options.pidfile),
+    )
+    with dc:
+        # get logger inside daemon context
+        tmp_log = get_logger()
+        # record in PID file
+        with open(options.pidfile, "w") as pid_file:
+            pid_file.write(f"{os.getpid()}")
+        # master object
+        master = DaemonMaster(
+            logger=tmp_log,
+            n_workers=n_workers,
+            n_dbconn=n_dbconn,
+            worker_lifetime=worker_lifetime,
+            use_tbif=use_tbif,
+        )
+
+        # function to end master when end signal caught
+        def end_master(sig, frame):
+            tmp_log.info(f"got end signal: {sig}")
+            master.stop()
+            kill_whole()
+
+        # set signal handler
+        for sig in END_SIGNALS:
+            signal.signal(sig, end_master)
+        # start master
+        master.run()
+    # get logger again
+    main_log = get_logger()
+    main_log.info("main end")
+
+
+# run
+if __name__ == "__main__":
+    main()

pandaserver/daemons/master_systemd.py

@@ -0,0 +1,79 @@
+import logging
+import os
+import signal
+import sys
+
+from pandaserver.config import daemon_config
+from pandaserver.daemons.utils import END_SIGNALS, DaemonMaster
+
+
+# get the logger
+def get_logger():
+    my_logger = logging.getLogger("PanDA-Daemon-Master")
+    # remove existing handlers
+    while my_logger.hasHandlers():
+        my_logger.removeHandler(my_logger.handlers[0])
+    # make new handler
+    _log_handler = logging.StreamHandler(sys.stdout)
+    _log_formatter = logging.Formatter("%(asctime)s %(name)-12s: %(levelname)-8s %(message)s")
+    _log_handler.setFormatter(_log_formatter)
+    # add new handler
+    my_logger.addHandler(_log_handler)
+    # debug log level
+    my_logger.setLevel(logging.DEBUG)
+    # return logger
+    return my_logger
+
+
+# kill the whole process group
+def kill_whole():
+    os.killpg(os.getpgrp(), signal.SIGKILL)
+
+
+# main function
+def main():
+    # whether to run daemons
+    if not getattr(daemon_config, "enable", False):
+        return
+
+    # get logger
+    main_log = get_logger()
+
+    n_workers = getattr(daemon_config, "n_proc", 1)
+    n_dbconn = getattr(daemon_config, "n_dbconn", 1)
+    worker_lifetime = getattr(daemon_config, "proc_lifetime", 28800)
+    use_tbif = getattr(daemon_config, "use_tbif", False)
+
+    main_log.info("main start")
+
+    # get logger inside daemon context
+    tmp_log = get_logger()
+
+    # master object
+    master = DaemonMaster(
+        logger=tmp_log,
+        n_workers=n_workers,
+        n_dbconn=n_dbconn,
+        worker_lifetime=worker_lifetime,
+        use_tbif=use_tbif,
+    )
+
+    # function to end master when end signal caught
+    def end_master(sig, frame):
+        tmp_log.info(f"got end signal: {sig}")
+        master.stop()
+        kill_whole()
+
+    # set signal handler
+    for sig in END_SIGNALS:
+        signal.signal(sig, end_master)
+
+    # start master
+    master.run()
+
+    main_log.info("main end")
+
+
+# run
+if __name__ == "__main__":
+    main()

pandaserver/daemons/scripts/add_main.py

@@ -0,0 +1,264 @@
+import datetime
+import multiprocessing
+import random
+import sys
+import time
+import traceback
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+from pandacommon.pandautils.thread_utils import GenericThread, WeightedLists
+
+from pandaserver.brokerage.SiteMapper import SiteMapper
+from pandaserver.config import panda_config
+from pandaserver.dataservice.adder_gen import AdderGen
+from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
+from pandaserver.taskbuffer.TaskBufferInterface import TaskBufferInterface
+
+# logger
+_logger = PandaLogger().getLogger("add_main")
+
+
+# main
+def main(argv=tuple(), tbuf=None, lock_pool=None, **kwargs):
+    requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+
+    prelock_pid = GenericThread().get_pid()
+    tmpLog = LogWrapper(_logger, f"<pid={prelock_pid}>")
+
+    tmpLog.debug("===================== start =====================")
+
+    # return value, true to run main again in next daemon loop
+    ret_val = True
+
+    # grace period
+    try:
+        gracePeriod = int(argv[1])
+    except Exception:
+        gracePeriod = 1
+
+    # lock interval in minutes
+    lock_interval = 10
+
+    # retry interval in minutes
+    retry_interval = 1
+
+    # last recovery time
+    last_recovery = naive_utcnow() + datetime.timedelta(seconds=random.randint(0, 30))
+
+    # instantiate TB
+    if tbuf is None:
+        from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+        taskBuffer.init(
+            panda_config.dbhost,
+            panda_config.dbpasswd,
+            nDBConnection=1,
+            useTimeout=True,
+            requester=requester_id,
+        )
+    else:
+        taskBuffer = tbuf
+
+    # instantiate sitemapper
+    aSiteMapper = SiteMapper(taskBuffer)
+
+    # thread for adder
+    class AdderThread(GenericThread):
+        def __init__(self, taskBuffer, aSiteMapper, job_output_reports, lock_pool):
+            GenericThread.__init__(self)
+            self.taskBuffer = taskBuffer
+            self.aSiteMapper = aSiteMapper
+            self.job_output_reports = job_output_reports
+            self.lock_pool = lock_pool
+
+        # main loop
+        def run(self):
+            # initialize
+            taskBuffer = self.taskBuffer
+            aSiteMapper = self.aSiteMapper
+            # get file list
+            timeNow = naive_utcnow()
+            timeInt = naive_utcnow()
+            # unique pid
+            GenericThread.__init__(self)
+            uniq_pid = self.get_pid()
+            # log pid
+            tmpLog.debug(f"pid={uniq_pid} : run")
+            # stats
+            n_processed = 0
+            # loop
+            while True:
+                # get report
+                one_jor = self.job_output_reports.pop()
+                if not one_jor:
+                    break
+                # lock
+                panda_id, job_status, attempt_nr, time_stamp = one_jor
+                token_str = f"pid={uniq_pid} : job={panda_id}.{attempt_nr}"
+                tmpLog.debug(f"{token_str} to lock timestamp={time_stamp}")
+                got_lock = taskBuffer.lockJobOutputReport(
+                    panda_id=panda_id,
+                    attempt_nr=attempt_nr,
+                    pid=uniq_pid,
+                    time_limit=lock_interval,
+                )
+                if not got_lock:
+                    tmpLog.debug(f"{token_str} skipped")
+                    continue
+                # add
+                try:
+                    modTime = time_stamp
+                    if (timeNow - modTime) > datetime.timedelta(hours=24):
+                        # last add
+                        tmpLog.debug(f"{token_str} last add st={job_status}")
+                        ignoreTmpError = False
+                    else:
+                        # usual add
+                        tmpLog.debug(f"{token_str} add st={job_status}")
+                        ignoreTmpError = True
+                    # get adder
+                    adder_gen = AdderGen(
+                        taskBuffer,
+                        panda_id,
+                        job_status,
+                        attempt_nr,
+                        ignore_tmp_error=ignoreTmpError,
+                        siteMapper=aSiteMapper,
+                        pid=uniq_pid,
+                        prelock_pid=uniq_pid,
+                        lock_offset=lock_interval - retry_interval,
+                        lock_pool=lock_pool,
+                    )
+                    n_processed += 1
+                    # execute
+                    adder_gen.run()
+                    tmpLog.debug(f"{token_str} done")
+                    del adder_gen
+                except Exception as e:
+                    tmpLog.error(f"pid={uniq_pid} : failed to run with {str(e)} {traceback.format_exc()}")
+            # stats
+            tmpLog.debug(f"pid={uniq_pid} : processed {n_processed}")
+
+        # launcher, run with multiprocessing
+        def proc_launch(self):
+            # run
+            self.process = multiprocessing.Process(target=self.run)
+            self.process.start()
+
+        # join of multiprocessing
+        def proc_join(self):
+            self.process.join()
+
+    # TaskBuffer with more connections behind TaskBufferInterface
+    tmpLog.debug("setup taskBufferIF")
+    n_connections = 4
+    _tbuf = TaskBuffer()
+    _tbuf.init(
+        panda_config.dbhost,
+        panda_config.dbpasswd,
+        nDBConnection=n_connections,
+        useTimeout=True,
+        requester=requester_id,
+    )
+    taskBufferIF = TaskBufferInterface()
+    taskBufferIF.launch(_tbuf)
+
+    # add files
+    tmpLog.debug("run Adder")
+
+    interval = 10
+    nLoop = 50
+    recover_dataset_update = False
+    for iLoop in range(nLoop):
+        tmpLog.debug(f"start iLoop={iLoop}/{nLoop}")
+        start_time = naive_utcnow()
+        adderThrList = []
+        nThr = 10
+
+        n_jors_per_batch = 200
+
+        jor_lists = WeightedLists(multiprocessing.Lock())
+
+        # get some job output reports
+        jor_list_others = taskBuffer.listJobOutputReport(
+            only_unlocked=True,
+            time_limit=lock_interval,
+            limit=random.randint(int(n_jors_per_batch * 0.5), int(n_jors_per_batch * 1.5)) * nThr,
+            grace_period=gracePeriod,
+            anti_labels=["user"],
+        )
+        jor_lists.add(3, jor_list_others)
+        jor_list_user = taskBuffer.listJobOutputReport(
+            only_unlocked=True,
+            time_limit=lock_interval,
+            limit=random.randint(int(n_jors_per_batch * 0.5), int(n_jors_per_batch * 1.5)) * nThr,
+            grace_period=gracePeriod,
+            labels=["user"],
+        )
+        jor_lists.add(7, jor_list_user)
+
+        # adder consumer processes
+        _n_thr_with_tbuf = 0
+        tbuf_list = []
+        tmpLog.debug(f"got {len(jor_lists)} job reports")
+        for i in range(nThr):
+            if i < _n_thr_with_tbuf:
+                tbuf = TaskBuffer()
+                tbuf_list.append(tbuf)
+                tbuf.init(
+                    panda_config.dbhost,
+                    panda_config.dbpasswd,
+                    nDBConnection=1,
+                    useTimeout=True,
+                    requester=requester_id,
+                )
+                thr = AdderThread(tbuf, aSiteMapper, jor_lists, lock_pool)
+            else:
+                thr = AdderThread(taskBufferIF.getInterface(), aSiteMapper, jor_lists, lock_pool)
+            adderThrList.append(thr)
+        # start all threads
+        for thr in adderThrList:
+            # thr.start()
+            thr.proc_launch()
+            time.sleep(0.25)
+
+        # join all threads
+        for thr in adderThrList:
+            # thr.join()
+            thr.proc_join()
+        [tbuf.cleanup(requester=requester_id) for tbuf in tbuf_list]
+        end_time = naive_utcnow()
+        sleep_time = interval - (end_time - start_time).seconds
+        if sleep_time > 0 and iLoop + 1 < nLoop:
+            sleep_time = random.randint(1, sleep_time)
+            tmpLog.debug(f"sleep {sleep_time} sec")
+            time.sleep(sleep_time)
+
+        # recovery
+        if naive_utcnow() - last_recovery > datetime.timedelta(minutes=2):
+            taskBuffer.async_update_datasets(None)
+            last_recovery = naive_utcnow()
+            recover_dataset_update = True
+
+    # recovery
+    if not recover_dataset_update:
+        taskBuffer.async_update_datasets(None)
+
+    # stop TaskBuffer IF
+    taskBufferIF.stop(requester=requester_id)
+
+    # stop taskBuffer if created inside this script
+    if tbuf is None:
+        taskBuffer.cleanup(requester=requester_id)
+
+    tmpLog.debug("===================== end =====================")
+
+    # return
+    return ret_val
+
+
+# run
+if __name__ == "__main__":
+    main(argv=sys.argv)