panda-server 0.3.4__tar.gz → 0.3.6__tar.gz

{panda_server-0.3.4 → panda_server-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: panda-server
-Version: 0.3.4
+Version: 0.3.6
 Summary: PanDA Server Package
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>
@@ -17,7 +17,7 @@ Requires-Dist: mod-wsgi
 Requires-Dist: numpy
 Requires-Dist: packaging
 Requires-Dist: panda-client-light>=1.5.55
-Requires-Dist: panda-common>=0.0.34
+Requires-Dist: panda-common>=0.1.1
 Requires-Dist: psutil>=5.4.8
 Requires-Dist: pyjwt
 Requires-Dist: pyopenssl

panda_server-0.3.6/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "0.3.6"

{panda_server-0.3.4 → panda_server-0.3.6}/pandaserver/daemons/scripts/add_sub.py

@@ -246,64 +246,6 @@ def main(argv=tuple(), tbuf=None, **kwargs):
         errStr = traceback.format_exc()
         tmp_log.error(errStr)
 
-    tmp_log.debug("Fork session")
-
-    # thread for fork
-    class ForkThr(threading.Thread):
-        def __init__(self, fileName):
-            threading.Thread.__init__(self)
-            self.fileName = fileName
-
-        def run(self):
-            if "VIRTUAL_ENV" in os.environ:
-                prefix = os.environ["VIRTUAL_ENV"]
-            else:
-                prefix = ""
-            setupStr = f"source {prefix}/etc/sysconfig/panda_server; "
-            runStr = f"{panda_config.native_python}/python -Wignore "
-            runStr += panda_config.pandaPython_dir + "/dataservice/forkSetupper.py -i "
-            runStr += self.fileName
-            if self.fileName.split("/")[-1].startswith("set.NULL."):
-                runStr += " -t"
-            comStr = setupStr + runStr
-            tmp_log.debug(comStr)
-            commands_get_status_output(comStr)
-
-    # get set.* files
-    filePatt = panda_config.logdir + "/" + "set.*"
-    fileList = glob.glob(filePatt)
-
-    # the max number of threads
-    maxThr = 10
-    nThr = 0
-
-    # loop over all files
-    forkThrList = []
-    timeNow = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
-    for tmpName in fileList:
-        if not os.path.exists(tmpName):
-            continue
-        try:
-            # takes care of only recent files
-            mod_time = datetime.datetime(*(time.gmtime(os.path.getmtime(tmpName))[:7]))
-            if datetime.timedelta(minutes=1) < (timeNow - mod_time) < datetime.timedelta(hours=1):
-                cSt, cOut = commands_get_status_output("ps aux | grep fork | grep -v PYTH")
-                # if no process is running for the file
-                if cSt == 0 and tmpName not in cOut:
-                    nThr += 1
-                    thr = ForkThr(tmpName)
-                    thr.start()
-                    forkThrList.append(thr)
-                    if nThr > maxThr:
-                        break
-        except Exception:
-            errType, errValue = sys.exc_info()[:2]
-            tmp_log.error(f"{errType} {errValue}")
-
-    # join fork threads
-    for thr in forkThrList:
-        thr.join()
-
     # stop taskBuffer if created inside this script
     if tbuf is None:
         taskBuffer.cleanup(requester=requester_id)

{panda_server-0.3.4 → panda_server-0.3.6}/pandaserver/daemons/scripts/copyArchive.py

@@ -6,15 +6,16 @@ import sys
 import time
 import traceback
 
-import pandaserver.userinterface.Client as Client
 import requests
 from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandacommon.pandautils.thread_utils import GenericThread
+from urllib3.exceptions import InsecureRequestWarning
+
+import pandaserver.userinterface.Client as Client
 from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.config import panda_config
 from pandaserver.jobdispatcher.Watcher import Watcher
 from pandaserver.taskbuffer import EventServiceUtils
-from urllib3.exceptions import InsecureRequestWarning
 
 # logger
 _logger = PandaLogger().getLogger("copyArchive")
@@ -629,7 +630,7 @@ def main(argv=tuple(), tbuf=None, **kwargs):
     varMap[":rFlag1"] = 2
     stDS, resDS = taskBuffer.querySQLS(sql, varMap)
     sqlSS = "SELECT laststart FROM ATLAS_PANDAMETA.siteData "
-    sqlSS += "WHERE site=:site AND flag=:flag AND hours=:hours AND laststart<:laststart "
+    sqlSS += "WHERE site=:site AND flag=:flag AND hours=:hours "
     sqlPI = "SELECT PandaID,eventService,attemptNr FROM ATLAS_PANDA.jobsActive4 "
     sqlPI += "WHERE prodSourceLabel=:prodSourceLabel AND jobStatus IN (:jobStatus1,:jobStatus2) "
     sqlPI += "AND (modificationTime<:timeLimit OR stateChangeTime<:timeLimit) "
@@ -644,9 +645,13 @@ def main(argv=tuple(), tbuf=None, **kwargs):
         varMap[":site"] = tmpSite
         varMap[":flag"] = "production"
         varMap[":hours"] = 3
-        varMap[":laststart"] = timeLimitSite
         stSS, resSS = taskBuffer.querySQLS(sqlSS, varMap)
-        if stSS is not None and len(resSS) > 0:
+        if resSS is not None and len(resSS) > 0:
+            last_start = resSS[0][0]
+        else:
+            last_start = None
+        site_status = siteMapper.getSite(tmpSite).status
+        if stSS is True and ((last_start is not None and last_start < timeLimitSite) or site_status in ["offline", "test"]):
             # get jobs
             varMap = {}
             varMap[":prodSourceLabel"] = "managed"
@@ -661,7 +666,7 @@ def main(argv=tuple(), tbuf=None, **kwargs):
             stPI, resPI = taskBuffer.querySQLS(sqlPI, varMap)
             jediJobs = []
             # reassign
-            _logger.debug(f"reassignJobs for JEDI at inactive site {tmpSite} laststart={resSS[0][0]}")
+            _logger.debug(f"reassignJobs for JEDI at inactive site {tmpSite} laststart={last_start} status={site_status}")
             if resPI is not None:
                 for pandaID, eventService, attemptNr in resPI:
                     if eventService in [EventServiceUtils.esMergeJobFlagNumber]:

{panda_server-0.3.4 → panda_server-0.3.6}/pandaserver/daemons/scripts/datasetManager.py

@@ -6,9 +6,10 @@ import threading
 import time
 import traceback
 
-import pandaserver.taskbuffer.ErrorCode
 from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandacommon.pandautils.thread_utils import GenericThread
+
+import pandaserver.taskbuffer.ErrorCode
 from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.config import panda_config
 from pandaserver.dataservice import DataServiceUtils
@@ -255,7 +256,7 @@ def main(tbuf=None, **kwargs):
 
     # close datasets
     _logger.debug("==== close datasets ====")
-    timeLimitU = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(minutes=30)
+    timeLimitU = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(minutes=1)
     timeLimitL = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(days=3)
     closeLock = threading.Semaphore(5)
     closeProxyLock = threading.Lock()

{panda_server-0.3.4 → panda_server-0.3.6}/pandaserver/dataservice/ddm.py

@@ -10,7 +10,6 @@ import traceback
 
 from typing import Dict, List
 
-from pandaserver.srvcore import CoreUtils
 from rucio.client import Client as RucioClient
 from rucio.common.exception import (
     DataIdentifierAlreadyExists,
@@ -21,6 +20,7 @@ from rucio.common.exception import (
     FileAlreadyExists,
     UnsupportedOperation,
 )
+from pandaserver.srvcore import CoreUtils
 
 
 # rucio
@@ -910,8 +910,8 @@ class RucioAPI:
                 if user_info is not None:
                     return_value = True
                     break
-        except Exception as e:
-            error_message = f"{str(e)}"
+        except Exception as error:
+            error_message = f"{str(error)}"
             user_info = error_message
         return return_value, user_info
 

panda_server-0.3.6/pandaserver/dataservice/setupper.py

@@ -0,0 +1,205 @@
+"""
+setup dataset
+
+"""
+
+import sys
+import threading
+import traceback
+from typing import List
+
+from pandacommon.pandalogger.LogWrapper import LogWrapper
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer import EventServiceUtils
+from pandaserver.taskbuffer.PickleJobSpec import PickleJobSpec
+
+# logger
+_logger = PandaLogger().getLogger("setupper")
+
+panda_config.setupPlugin()
+
+
+# main class
+class Setupper(threading.Thread):
+    """
+    Main class for setting up the dataset.
+    """
+    # constructor
+    def __init__(
+        self,
+        taskBuffer,
+        jobs: List[object],
+        resubmit: bool = False,
+        panda_ddm: bool = False,
+        ddm_attempt: int = 0,
+        only_ta: bool = False,
+        first_submission: bool = True,
+    ):
+        """
+        Constructor for the Setupper class.
+
+        :param taskBuffer: The buffer for tasks.
+        :param jobs: The jobs to be processed.
+        :param resubmit: A flag to indicate if the job is a resubmission. Defaults to False.
+        :param panda_ddm: A flag to indicate if PandaDDM is used. Defaults to False.
+        :param ddm_attempt: The number of attempts for DDM job. Defaults to 0.
+        :param only_ta: A flag to indicate if only task assignment should be run. Defaults to False.
+        :param first_submission: A flag to indicate if it's the first submission. Defaults to True.
+        """
+        threading.Thread.__init__(self)
+        self.jobs = jobs
+        self.task_buffer = taskBuffer
+        # resubmission or not
+        self.resubmit = resubmit
+        # use PandaDDM
+        self.panda_ddm = panda_ddm
+        # priority for ddm job
+        self.ddm_attempt = ddm_attempt
+        # run task assignment only
+        self.only_ta = only_ta
+        # first submission
+        self.first_submission = first_submission
+
+    # main
+    def run(self) -> None:
+        """
+        Main method for running the setup process.
+        """
+        try:
+            # make a message instance
+            tmp_log = LogWrapper(_logger, None)
+            # run main procedure in the same process
+            tmp_log.debug("main start")
+            tmp_log.debug(f"first_submission={self.first_submission}")
+            # make Specs pickleable
+            p_job_list = []
+            for job_spec in self.jobs:
+                p_job = PickleJobSpec()
+                p_job.update(job_spec)
+                p_job_list.append(p_job)
+            self.jobs = p_job_list
+            # group jobs per VO
+            vo_jobs_map = {}
+            tmp_log.debug(f"{len(self.jobs)} jobs in total")
+            for tmp_job in self.jobs:
+                # set VO=local for DDM free
+                if tmp_job.destinationSE == "local":
+                    tmp_vo = "local"
+                else:
+                    tmp_vo = tmp_job.VO
+                # make map
+                vo_jobs_map.setdefault(tmp_vo, [])
+                vo_jobs_map[tmp_vo].append(tmp_job)
+            # loop over all VOs
+            for tmp_vo in vo_jobs_map:
+                tmp_job_list = vo_jobs_map[tmp_vo]
+                tmp_log.debug(f"vo={tmp_vo} has {len(tmp_job_list)} jobs")
+                # get plugin
+                setupper_plugin_class = panda_config.getPlugin("setupper_plugins", tmp_vo)
+                if setupper_plugin_class is None:
+                    # use ATLAS plug-in by default
+                    from pandaserver.dataservice.setupper_atlas_plugin import SetupperAtlasPlugin
+                    setupper_plugin_class = SetupperAtlasPlugin
+                tmp_log.debug(f"plugin name -> {setupper_plugin_class.__name__}")
+                try:
+                    # make plugin
+                    setupper_plugin = setupper_plugin_class(
+                        self.task_buffer,
+                        self.jobs,
+                        tmp_log,
+                        resubmit=self.resubmit,
+                        panda_ddm=self.panda_ddm,
+                        ddm_attempt=self.ddm_attempt,
+                        only_ta=self.only_ta,
+                        first_submission=self.first_submission,
+                    )
+                    # run plugin
+                    tmp_log.debug("run plugin")
+                    setupper_plugin.run()
+                    # go forward if not Task Assignment
+                    if not self.only_ta:
+                        # update jobs
+                        tmp_log.debug("update jobs")
+                        self.update_jobs(setupper_plugin.jobs + setupper_plugin.jumbo_jobs, tmp_log)
+                        # execute post process
+                        tmp_log.debug("post execute plugin")
+                        setupper_plugin.post_run()
+                    tmp_log.debug("done plugin")
+                except Exception:
+                    error_type, error_value = sys.exc_info()[:2]
+                    tmp_log.error(f"plugin failed with {error_type}:{error_value}")
+            tmp_log.debug("main end")
+        except Exception as error:
+            tmp_log.error(f"master failed with {str(error)} {traceback.format_exc()}")
+
+    #  update jobs
+    def update_jobs(self, job_list: List[object], tmp_log: LogWrapper) -> None:
+        """
+        Updates the status of jobs.
+
+        :param job_list: The list of jobs to be updated.
+        :param log: The logger to be used for logging.
+        """
+        update_jobs = []
+        failed_jobs = []
+        activate_jobs = []
+        waiting_jobs = []
+        # sort jobs by status
+        for job in job_list:
+            # failed jobs
+            if job.jobStatus in ["failed", "cancelled"]:
+                failed_jobs.append(job)
+            # waiting
+            elif job.jobStatus == "waiting":
+                waiting_jobs.append(job)
+            # no input jobs
+            elif job.dispatchDBlock == "NULL":
+                activate_jobs.append(job)
+            # normal jobs
+            else:
+                # change status
+                job.jobStatus = "assigned"
+                update_jobs.append(job)
+
+        # trigger merge generation if all events are done
+        new_activate_jobs = []
+        n_finished = 0
+        for job in activate_jobs:
+            if job.notDiscardEvents() and job.allOkEvents() and not EventServiceUtils.isEventServiceMerge(job):
+                self.task_buffer.updateJobs([job])
+                # change status
+                job.jobStatus = "finished"
+                self.task_buffer.updateJobs([job], False)
+                n_finished += 1
+            else:
+                new_activate_jobs.append(job)
+        activate_jobs = new_activate_jobs
+        tmp_log.debug(f"# of finished jobs in activated : {n_finished}")
+        new_update_jobs = []
+        n_finished = 0
+        for job in update_jobs:
+            if job.notDiscardEvents() and job.allOkEvents() and not EventServiceUtils.isEventServiceMerge(job):
+                self.task_buffer.updateJobs([job], True)
+                # change status
+                job.jobStatus = "finished"
+                self.task_buffer.updateJobs([job], True)
+                n_finished += 1
+            else:
+                new_update_jobs.append(job)
+        update_jobs = new_update_jobs
+        tmp_log.debug(f"# of finished jobs in defined : {n_finished}")
+        # update DB
+        tmp_log.debug(f"# of activated jobs : {len(activate_jobs)}")
+        self.task_buffer.activateJobs(activate_jobs)
+        tmp_log.debug(f"# of updated jobs : {len(update_jobs)}")
+        self.task_buffer.updateJobs(update_jobs, True)
+        tmp_log.debug(f"# of failed jobs : {len(failed_jobs)}")
+        self.task_buffer.updateJobs(failed_jobs, True)
+        tmp_log.debug(f"# of waiting jobs : {len(waiting_jobs)}")
+        self.task_buffer.keepJobs(waiting_jobs)
+        # delete local values
+        del update_jobs
+        del failed_jobs
+        del activate_jobs
+        del waiting_jobs