panda-server-source 0.8.1__tar.gz → 1.0.1__tar.gz

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: panda-server-source
-Version: 0.8.1
+Version: 1.0.1
 Summary: Unified PanDA Server and JEDI Package - Source Only
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>
@@ -11,6 +11,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
+Requires-Dist: panda-client-light
+Requires-Dist: panda-common
 Description-Content-Type: text/markdown
 
 # Monorepo for the PanDA Server and JEDI

panda_server_source-1.0.1/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "1.0.1"

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedibrokerage/AtlasAnalJobBroker.py

@@ -47,6 +47,13 @@ class AtlasAnalJobBroker(JobBrokerBase):
             logger.error("Failed to load the WN architecture level map!!!")
             self.architecture_level_map = {}
 
+        # load the worker node GPU map
+        try:
+            self.wn_gpu_map = taskBufferIF.get_worker_node_gpu_map()
+        except BaseException:
+            logger.error("Failed to load the WN GPU map!!!")
+            self.wn_gpu_map = {}
+
     # main
     def doBrokerage(self, taskSpec, cloudName, inputChunk, taskParamMap):
         # make logger
@@ -834,7 +841,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
             newScanSiteList = []
             oldScanSiteList = copy.copy(scanSiteList)
             msg_map = {}
-            jsonCheck = AtlasBrokerUtils.JsonSoftwareCheck(self.siteMapper, self.sw_map, self.architecture_level_map)
+            jsonCheck = AtlasBrokerUtils.JsonSoftwareCheck(self.siteMapper, self.sw_map, self.architecture_level_map, self.wn_gpu_map)
             for tmpSiteName in scanSiteList:
                 tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
                 if tmpSiteSpec.isGPU() and not taskSpec.is_hpo_workflow():
@@ -1117,7 +1124,14 @@ class AtlasAnalJobBroker(JobBrokerBase):
             msg_map = {}
             for tmpSiteName in self.get_unified_sites(scanSiteList):
                 tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
-                # check at the site
+                # check direct access
+                if taskSpec.allowInputLAN() == "only" and not tmpSiteSpec.isDirectIO() and not tmpSiteSpec.always_use_direct_io() and not inputChunk.isMerging:
+                    tmp_msg = f"  skip site={tmpSiteName} since direct IO is disabled "
+                    tmp_msg += "criteria=-remoteio"
+                    msg_map[tmpSiteName] = tmp_msg
+                    continue
+
+                # check disk space
                 if tmpSiteSpec.maxwdir:
                     if CoreUtils.use_direct_io_for_job(taskSpec, tmpSiteSpec, inputChunk):
                         minDiskCount = minDiskCountR
@@ -1282,7 +1296,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                     continue
 
             ######################################
-            # selection for zero walltime: scouts, merges, and walltime-undefined jobs
+            # selection to skip short walltime: scouts, merges, and walltime-undefined jobs
             # must only go to sites with at least 24hr*10HS06s of available walltime
             if (
                 (not sitePreAssigned and inputChunk.useScout())
@@ -1318,16 +1332,16 @@ class AtlasAnalJobBroker(JobBrokerBase):
                     if siteMaxTime != 0 and siteMaxTime < minTimeForZeroWalltime:
                         tmpMsg = f"  skip site={tmpSiteName} due to site walltime {tmpSiteStr} (site upper limit) insufficient "
                         if inputChunk.useScout():
-                            tmpMsg += f"for scouts ({str_minTimeForZeroWalltime} at least) "
-                            tmpMsg += "criteria=-scoutwalltime"
+                            tmpMsg += f"for scout jobs (requireing {str_minTimeForZeroWalltime} at least) "
+                            tmpMsg += "criteria=-scout_walltime"
                         else:
-                            tmpMsg += f"for zero walltime ({str_minTimeForZeroWalltime} at least) "
-                            tmpMsg += "criteria=-zerowalltime"
+                            tmpMsg += f"for merge or walltime-undefined jobs (requiring {str_minTimeForZeroWalltime} at least) "
+                            tmpMsg += "criteria=-min_walltime"
                         msg_map[tmpSiteSpec.get_unified_name()] = tmpMsg
                         continue
                     newScanSiteList.append(tmpSiteName)
                 scanSiteList = newScanSiteList
-                self.add_summary_message(oldScanSiteList, scanSiteList, "zero walltime check", tmpLog, msg_map)
+                self.add_summary_message(oldScanSiteList, scanSiteList, "minimum walltime capacity check", tmpLog, msg_map)
                 if not scanSiteList:
                     self.dump_summary(tmpLog)
                     tmpLog.error("no candidates")

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedibrokerage/AtlasBrokerUtils.py

@@ -17,6 +17,7 @@ from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.DataServiceUtils import select_scope
 from pandaserver.taskbuffer import JobUtils, ProcessGroups, SiteSpec
+from pandaserver.taskbuffer.DdmSpec import DOWNTIME_STATUSES
 
 
 # get nuclei where data is available
@@ -932,6 +933,8 @@ def compare_version_string(version_string, comparison_string):
         return None
 
     operator = match.group(1).strip()
+    if operator == "=":
+        operator = "=="
     version_to_compare = match.group(2).strip()
 
     try:
@@ -959,10 +962,11 @@ def compare_version_string(version_string, comparison_string):
 # check SW with json
 class JsonSoftwareCheck:
     # constructor
-    def __init__(self, site_mapper, sw_map, wn_architecture_level_map):
+    def __init__(self, site_mapper, sw_map, wn_architecture_level_map, wn_gpu_map=None):
         self.siteMapper = site_mapper
         self.sw_map = sw_map
         self.wn_architecture_level_map = wn_architecture_level_map
+        self.wn_gpu_map = wn_gpu_map or {}
 
     # get lists
     def check(
@@ -1081,53 +1085,59 @@ class JsonSoftwareCheck:
 
                         # check GPU
                         if host_gpu_spec:
-                            # GPU not specified
+                            # CRIC is the GPU capability gate: skip queue if not GPU-capable per CRIC
                             if "gpu" not in architecture_map:
                                 continue
 
+                            # All attribute checks use WN GPU monitoring (MV_WORKER_NODE_GPU_SUMMARY)
+                            # which has richer per-host data (vram, architecture, driver version)
+                            wn_gpus = self.wn_gpu_map.get(tmp_site_name, [])
+
                             # check vendor
-                            if host_gpu_spec["vendor"] == "*":
-                                # task doesn't specify CPU vendor and PQ explicitly requests a specific vendor
-                                if "vendor" in architecture_map["gpu"] and "excl" in architecture_map["gpu"]["vendor"]:
-                                    continue
-                            else:
-                                # task specifies a vendor and PQ doesn't request any specific vendor
-                                if "vendor" not in architecture_map["gpu"]:
-                                    continue
-                                # task specifies a vendor and PQ doesn't accept any vendor or the specific vendor
-                                if "any" not in architecture_map["gpu"]["vendor"] and host_gpu_spec["vendor"] not in architecture_map["gpu"]["vendor"]:
+                            if host_gpu_spec["vendor"] != "*":
+                                if not wn_gpus or not any(g.get("vendor") and re.match(host_gpu_spec["vendor"], g["vendor"], re.IGNORECASE) for g in wn_gpus):
                                     continue
 
-                            # check model
-                            if host_gpu_spec["model"] == "*":
-                                if "model" in architecture_map["gpu"] and "excl" in architecture_map["gpu"]["model"]:
-                                    continue
-                            else:
+                            # check model (include or exclude pattern)
+                            if host_gpu_spec["model"] != "*":
                                 if isinstance(host_gpu_spec["model"], dict):
                                     model_pattern = host_gpu_spec["model"]["pattern"]
                                     model_excl = host_gpu_spec["model"].get("excl", False)
                                 else:
                                     model_pattern = host_gpu_spec["model"]
                                     model_excl = False
-                                if "model" not in architecture_map["gpu"] or (
-                                    "any" not in architecture_map["gpu"]["model"]
-                                    and any(re.match(model_pattern, m, re.IGNORECASE) for m in architecture_map["gpu"]["model"])
-                                    == model_excl
-                                ):
+                                if not wn_gpus:
+                                    continue
+                                matches = any(g.get("model") and re.match(model_pattern, g["model"], re.IGNORECASE) for g in wn_gpus)
+                                if matches == model_excl:
+                                    continue
+
+                            # check VRAM (in MB); supports operators: ==, >=, <=, >, <, != (e.g. ">=40960")
+                            if "vram" in host_gpu_spec:
+                                if not wn_gpus or not any(g.get("vram") and compare_version_string(str(g["vram"]), host_gpu_spec["vram"]) for g in wn_gpus):
                                     continue
 
-                            # check version
+                            # check GPU microarchitecture generation (e.g. Ampere, Hopper, Ada Lovelace)
+                            if "microarchitecture" in host_gpu_spec:
+                                req_arch = host_gpu_spec["microarchitecture"]
+                                if isinstance(req_arch, str):
+                                    req_arch = [req_arch]
+                                if not wn_gpus or not any(g.get("architecture") in req_arch for g in wn_gpus):
+                                    continue
+
+                            # check minimum CUDA version
                             if "version" in host_gpu_spec:
-                                if "version" not in architecture_map["gpu"]:
-                                    # PQ doesn't specify version
+                                if not wn_gpus or not any(
+                                    g.get("framework_version") and compare_version_string(g["framework_version"], host_gpu_spec["version"]) for g in wn_gpus
+                                ):
+                                    continue
+
+                            # check minimum GPU driver version (kernel driver, e.g. 575.57.08)
+                            if "driver_version" in host_gpu_spec:
+                                if not wn_gpus or not any(
+                                    g.get("driver_version") and compare_version_string(g["driver_version"], host_gpu_spec["driver_version"]) for g in wn_gpus
+                                ):
                                     continue
-                                elif "any" == architecture_map["gpu"]["version"]:
-                                    # PQ accepts any version
-                                    pass
-                                else:
-                                    # check version at PQ
-                                    if not compare_version_string(architecture_map["gpu"]["version"], host_gpu_spec["version"]):
-                                        continue
                     go_ahead = True
                 except Exception as e:
                     if log_stream:
@@ -1251,12 +1261,12 @@ def check_endpoints_with_blacklist(
             tmp_read_input_over_lan = tmp_input_endpoint["detailed_status"].get("read_lan")
             tmp_receive_input_over_wan = tmp_input_endpoint["detailed_status"].get("write_wan")
             # can read input from local
-            if tmp_read_input_over_lan not in ["OFF", "TEST"]:
+            if tmp_read_input_over_lan not in DOWNTIME_STATUSES:
                 read_input_over_lan = True
             # can receive input from remote to local
             if tmp_site_name not in sites_in_nucleus:
                 # satellite sites
-                if tmp_receive_input_over_wan not in ["OFF", "TEST"]:
+                if tmp_receive_input_over_wan not in DOWNTIME_STATUSES:
                     receive_input_over_wan = True
             else:
                 # NA for nucleus sites
@@ -1267,12 +1277,12 @@ def check_endpoints_with_blacklist(
             tmp_write_output_over_lan = tmp_output_endpoint["detailed_status"].get("write_lan")
             tmp_send_output_over_wan = tmp_output_endpoint["detailed_status"].get("read_wan")
             # can write output to local
-            if tmp_write_output_over_lan not in ["OFF", "TEST"]:
+            if tmp_write_output_over_lan not in DOWNTIME_STATUSES:
                 write_output_over_lan = True
             # can send output from local to remote
             if tmp_site_name not in sites_in_nucleus:
                 # satellite sites
-                if tmp_send_output_over_wan not in ["OFF", "TEST"]:
+                if tmp_send_output_over_wan not in DOWNTIME_STATUSES:
                     send_output_over_wan = True
             else:
                 # NA for nucleus sites

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedibrokerage/AtlasProdJobBroker.py

@@ -12,6 +12,7 @@ from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.DataServiceUtils import select_scope
 from pandaserver.srvcore import CoreUtils
 from pandaserver.taskbuffer import EventServiceUtils, JobUtils
+from pandaserver.taskbuffer.DdmSpec import DOWNTIME_STATUSES
 
 from . import AtlasBrokerUtils
 from .JobBrokerBase import JobBrokerBase
@@ -96,6 +97,13 @@ class AtlasProdJobBroker(JobBrokerBase):
             logger.error("Failed to load the WN architecture level map!!!")
             self.architecture_level_map = {}
 
+        # load the worker node GPU map
+        try:
+            self.wn_gpu_map = taskBufferIF.get_worker_node_gpu_map()
+        except BaseException:
+            logger.error("Failed to load the WN GPU map!!!")
+            self.wn_gpu_map = {}
+
     def convertMBpsToWeight(self, mbps):
         """
         Takes MBps value and converts to a weight between 1 and 2
@@ -255,7 +263,7 @@ class AtlasProdJobBroker(JobBrokerBase):
                 if not default_endpoint_out:
                     continue
                 receive_output_over_wan = default_endpoint_out["detailed_status"].get("write_wan")
-                if receive_output_over_wan in ["OFF", "TEST"]:
+                if receive_output_over_wan in DOWNTIME_STATUSES:
                     nucleus_with_storages_unwritable_over_wan[tmp_name] = receive_output_over_wan
 
         else:
@@ -754,7 +762,7 @@ class AtlasProdJobBroker(JobBrokerBase):
         resolved_platforms = {}
         preference_weight_map = {}
         if taskSpec.transHome is not None:
-            jsonCheck = AtlasBrokerUtils.JsonSoftwareCheck(self.siteMapper, self.sw_map, self.architecture_level_map)
+            jsonCheck = AtlasBrokerUtils.JsonSoftwareCheck(self.siteMapper, self.sw_map, self.architecture_level_map, self.wn_gpu_map)
             unified_site_list = self.get_unified_sites(scanSiteList)
 
             host_cpu_spec = taskSpec.get_host_cpu_spec()
@@ -1126,11 +1134,11 @@ class AtlasProdJobBroker(JobBrokerBase):
                 if siteMaxTime != 0 and siteMaxTime < minTimeForZeroWalltime:
                     tmp_msg = f"  skip site={tmpSiteName} due to site walltime {tmpSiteStr} (site upper limit) insufficient "
                     if inputChunk.useScout():
-                        tmp_msg += f"for scouts ({str_minTimeForZeroWalltime} at least) "
-                        tmp_msg += "criteria=-scoutwalltime"
+                        tmp_msg += f"for scout jobs (requireing {str_minTimeForZeroWalltime} at least) "
+                        tmp_msg += "criteria=-scout_walltime"
                     else:
-                        tmp_msg += f"for zero walltime ({str_minTimeForZeroWalltime} at least) "
-                        tmp_msg += "criteria=-zerowalltime"
+                        tmp_msg += f"for walltime-undefined jobs (requiring {str_minTimeForZeroWalltime} at least) "
+                        tmp_msg += "criteria=-min_walltime"
                     msg_map[tmpSiteSpec.get_unified_name()] = tmp_msg
                     continue
             # check min walltime at the site

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedibrokerage/AtlasProdTaskBroker.py

@@ -18,6 +18,7 @@ from pandajedi.jedicore.ThreadUtils import (
 )
 from pandajedi.jedirefine import RefinerUtils
 from pandaserver.dataservice import DataServiceUtils
+from pandaserver.taskbuffer.DdmSpec import DOWNTIME_STATUSES
 
 from . import AtlasBrokerUtils
 from .AtlasProdJobBroker import AtlasProdJobBroker
@@ -345,14 +346,14 @@ class AtlasProdTaskBrokerThread(WorkerThread):
                                     break
                                 # check blacklist
                                 read_wan_status = tmpEP["detailed_status"].get("read_wan")
-                                if read_wan_status in ["OFF", "TEST"]:
+                                if read_wan_status in DOWNTIME_STATUSES:
                                     tmpLog.info(
                                         f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has read_wan={read_wan_status} criteria=-source_blacklist"
                                     )
                                     to_skip = True
                                     break
                                 write_wan_status = tmpEP["detailed_status"].get("write_wan")
-                                if write_wan_status in ["OFF", "TEST"]:
+                                if write_wan_status in DOWNTIME_STATUSES:
                                     tmpLog.info(
                                         f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has write_wan={write_wan_status} criteria=-destination_blacklist"
                                     )

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedibrokerage/GenJobBroker.py

@@ -370,13 +370,13 @@ class GenJobBroker(JobBrokerBase):
         # append candidates
         newScanSiteList = []
         for siteCandidateSpec in candidateSpecList:
-            # append
-            inputChunk.addSiteCandidate(siteCandidateSpec)
+            # append only if all files are available at the site when file dispatch is disabled
             if panda_config.disable_file_dispatch and len(siteCandidateSpec.localDiskFiles) < total_num_files:
                 tmpLog.debug(
                     f"  skip {siteCandidateSpec.siteName} due to input file dispatch is disabled and not all files are available at the site ({len(siteCandidateSpec.localDiskFiles)} < {total_num_files})"
                 )
                 continue
+            inputChunk.addSiteCandidate(siteCandidateSpec)
             newScanSiteList.append(siteCandidateSpec.siteName)
             tmpLog.debug(f"  use {siteCandidateSpec.siteName} with weight={siteCandidateSpec.weight} nFiles={len(siteCandidateSpec.localDiskFiles)}")
         scanSiteList = newScanSiteList

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedicore/JediTaskBuffer.py

@@ -74,6 +74,7 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         order_by,
         maxFileRecords,
         skip_short_output,
+        lfn_constituent_map=None,
     ):
         with self.proxyPool.get() as proxy:
             return proxy.insertFilesForDataset_JEDI(
@@ -111,6 +112,7 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
                 order_by,
                 maxFileRecords,
                 skip_short_output,
+                lfn_constituent_map=lfn_constituent_map,
             )
 
     # get files from the JEDI contents table with jediTaskID and/or datasetID
@@ -198,11 +200,6 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.getTaskDatasetsWithID_JEDI(jediTaskID, pid, lockTask)
 
-    # get JEDI tasks with selection criteria
-    def getTaskIDsWithCriteria_JEDI(self, criteria, nTasks=50):
-        with self.proxyPool.get() as proxy:
-            return proxy.getTaskIDsWithCriteria_JEDI(criteria, nTasks)
-
     # get JEDI tasks to be finished
     def getTasksToBeFinished_JEDI(self, vo, prodSourceLabel, pid, nTasks=50, target_tasks=None):
         with self.proxyPool.get() as proxy:
@@ -255,9 +252,9 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
             )
 
     # get tasks to be processed
-    def checkWaitingTaskPrio_JEDI(self, vo, workQueue, prodSourceLabel, cloudName, resource_name=None):
+    def checkWaitingTaskPrio_JEDI(self, vo, workQueue, prodSourceLabel, cloudName, resource_name=None, pid=None):
         with self.proxyPool.get() as proxy:
-            return proxy.getTasksToBeProcessed_JEDI(None, vo, workQueue, prodSourceLabel, cloudName, isPeeking=True, resource_name=resource_name)
+            return proxy.getTasksToBeProcessed_JEDI(pid, vo, workQueue, prodSourceLabel, cloudName, isPeeking=True, resource_name=resource_name)
 
     # get job statistics with work queue
     def getJobStatisticsWithWorkQueue_JEDI(self, vo, prodSourceLabel, minPriority=None):
@@ -414,6 +411,7 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         unmergeDatasetSpecMap,
         uniqueTaskName,
         oldTaskStatus,
+        in_content_dataset_spec_list,
     ):
         with self.proxyPool.get() as proxy:
             return proxy.registerTaskInOneShot_JEDI(
@@ -429,6 +427,7 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
                 unmergeDatasetSpecMap,
                 uniqueTaskName,
                 oldTaskStatus,
+                in_content_dataset_spec_list,
             )
 
     # set tasks to be assigned
@@ -595,9 +594,9 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
             )
 
     # append input datasets for incremental execution
-    def appendDatasets_JEDI(self, jediTaskID, inMasterDatasetSpecList, inSecDatasetSpecList):
+    def appendDatasets_JEDI(self, jediTaskID, inMasterDatasetSpecList, inSecDatasetSpecList, in_content_dataset_specs):
         with self.proxyPool.get() as proxy:
-            return proxy.appendDatasets_JEDI(jediTaskID, inMasterDatasetSpecList, inSecDatasetSpecList)
+            return proxy.appendDatasets_JEDI(jediTaskID, inMasterDatasetSpecList, inSecDatasetSpecList, in_content_dataset_specs)
 
     # record retry history
     def recordRetryHistory_JEDI(self, jediTaskID, oldNewPandaIDs, relationType):
@@ -927,6 +926,11 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.get_tasks_inputdatasets_JEDI(vo)
 
+    # get dataset locality for a task and dataset
+    def get_dataset_locality(self, jedi_taskid, datasetid):
+        with self.proxyPool.get() as proxy:
+            return proxy.get_dataset_locality(jedi_taskid, datasetid)
+
     # update dataset locality
     def updateDatasetLocality_JEDI(self, jedi_taskid, datasetid, rse):
         with self.proxyPool.get() as proxy:
@@ -991,3 +995,8 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
     def get_task_failure_metrics(self, jedi_task_id):
         with self.proxyPool.get() as proxy:
             return proxy.get_task_failure_metrics(jedi_task_id)
+
+    # reset frozen time of a task to avoid being exausted
+    def reset_frozen_time_for_task(self, task_id):
+        with self.proxyPool.get() as proxy:
+            return proxy.reset_frozen_time_for_task(task_id)

panda_server_source-1.0.1/pandajedi/jedidog/AsyncRequestWatchDog.py

@@ -0,0 +1,27 @@
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandaserver.asyncprocess import processor
+
+from .WatchDogBase import WatchDogBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# watchdog that processes async requests on JEDI machines
+class AsyncRequestWatchDog(WatchDogBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        WatchDogBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main
+    def doAction(self):
+        tmpLog = MsgWrapper(logger)
+        tmpLog.debug("start")
+        try:
+            processor.run(service_name="jedi", tbuf=self.taskBufferIF)
+        except Exception as e:
+            tmpLog.error(f"failed to process async requests with {e}")
+            return self.SC_FAILED
+        tmpLog.debug("done")
+        return self.SC_SUCCEEDED

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jedidog/AtlasAnalWatchDog.py

@@ -8,6 +8,7 @@ from pandacommon.pandalogger.PandaLogger import PandaLogger
 
 from pandajedi.jedicore.MsgWrapper import MsgWrapper
 from pandaserver.dataservice.activator import Activator
+from pandaserver.taskbuffer.JediDatasetSpec import JediDatasetSpec
 
 from .TypicalWatchDogBase import TypicalWatchDogBase
 
@@ -27,6 +28,8 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
         orig_tmp_log = MsgWrapper(logger)
         try:
             orig_tmp_log.debug("start")
+            # site mapper
+            self.site_mapper = self.taskBufferIF.get_site_mapper()
             # handle waiting jobs
             self.doForWaitingJobs()
             # throttle tasks if so many prestaging requests
@@ -718,6 +721,11 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
 
     # periodic task action
     def do_periodic_action(self):
+        """
+        Perform periodic action on tasks
+          * extend lifetime of output containers for analysis tasks
+          * reset frozen time of pending tasks if input dataset is available only at RSEs in downtime, to avoid unnecessary exhaustion
+        """
         tmp_log = MsgWrapper(logger, " do_periodic_action label=user")
         tmp_log.debug("start")
         try:
@@ -731,7 +739,7 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
             # get tasks
             task_list = self.taskBufferIF.get_tasks_for_periodic_action(self.vo, self.prodSourceLabel)
             for task_id in task_list:
-                # get datasets
+                # get output datasets
                 _, tmp_datasets = self.taskBufferIF.getDatasetsWithJediTaskID_JEDI(task_id, ["output"])
                 done_containers = set()
                 for dataset_spec in tmp_datasets:
@@ -744,5 +752,24 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
                         dataset_spec.containerName,
                         {"type=.+": {"lifetime": lifetime}, "(SCRATCH|USER)DISK": {"lifetime": lifetime}},
                     )
+                # get input datasets
+                _, tmp_datasets = self.taskBufferIF.getDatasetsWithJediTaskID_JEDI(task_id, ["input", JediDatasetSpec.get_constituent_input_type()])
+                for dataset_spec in tmp_datasets:
+                    # get locations
+                    rses = self.taskBufferIF.get_dataset_locality(task_id, dataset_spec.datasetID)
+                    # check if all locations are in downtime
+                    if rses:
+                        all_in_downtime = True
+                        for rse in rses:
+                            if self.site_mapper.is_readable_locally(rse):
+                                all_in_downtime = False
+                                break
+                        if all_in_downtime:
+                            tmp_log.debug(
+                                f"reset frozen time for taskID={task_id} since all locations {rses} of input dataset {dataset_spec.datasetName} are in downtime"
+                            )
+                            self.taskBufferIF.reset_frozen_time_for_task(task_id)
+                            break
+
         except Exception as e:
             tmp_log.error(f"failed with {str(e)}{traceback.format_exc()}")

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jediorder/ContentsFeeder.py

@@ -13,6 +13,7 @@ from pandajedi.jedicore import Interaction
 from pandajedi.jedicore.MsgWrapper import MsgWrapper
 from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool, WorkerThread
 from pandajedi.jedirefine import RefinerUtils
+from pandaserver.taskbuffer.JediDatasetSpec import JediDatasetSpec
 
 from .JediKnight import JediKnight
 
@@ -125,10 +126,16 @@ class ContentsFeederThread(WorkerThread):
             datasetsIdxConsistency = []
 
             # get task
-            tmpStat, taskSpec = self.taskBufferIF.getTaskWithID_JEDI(jediTaskID, False, real_run, self.pid, 10, clearError=True)
+            tmpStat, taskSpec = self.taskBufferIF.getTaskWithID_JEDI(jediTaskID, False, real_run, self.pid, 10)
             if not tmpStat or taskSpec is None:
                 self.logger.debug(f"failed to get taskSpec for jediTaskID={jediTaskID}")
                 continue
+            # get constituent datasets grouped by their master input datasetID
+            constituent_by_master = {}
+            _, c_datasets = self.taskBufferIF.getDatasetsWithJediTaskID_JEDI(jediTaskID, [JediDatasetSpec.get_constituent_input_type()])
+            if c_datasets:
+                for c_ds in c_datasets:
+                    constituent_by_master.setdefault(c_ds.masterID, []).append((c_ds.datasetID, c_ds.datasetName))
 
             # make logger
             try:
@@ -531,6 +538,16 @@ class ContentsFeederThread(WorkerThread):
                                 orderBy = taskSpec.order_input_by()
                             else:
                                 orderBy = None
+                            # build LFN -> constituent datasetID map for this input dataset
+                            lfn_constituent_map = {}
+                            if datasetSpec.datasetID in constituent_by_master:
+                                for c_id, c_name in constituent_by_master[datasetSpec.datasetID]:
+                                    try:
+                                        c_files = ddmIF.getFilesInDataset(c_name)
+                                        for f_data in c_files.values():
+                                            lfn_constituent_map[str(f_data["lfn"])] = c_id
+                                    except Exception:
+                                        tmpLog.warning(f"failed to get files for constituent dataset {c_name}")
                             # feed files to the contents table
                             tmpLog.debug("update contents")
                             res_dict = self.taskBufferIF.insertFilesForDataset_JEDI(
@@ -568,6 +585,7 @@ class ContentsFeederThread(WorkerThread):
                                 orderBy,
                                 maxFileRecords,
                                 skip_short_output,
+                                lfn_constituent_map=lfn_constituent_map,
                             )
                             retDB = res_dict["ret_val"]
                             missingFileList = res_dict["missingFileList"]
@@ -627,14 +645,15 @@ class ContentsFeederThread(WorkerThread):
                                 setFrozenTime = False
                                 skip_secondaries = True
                     tmpLog.debug("end loop")
-            # task holdup by workflow if no master inputs are ready
-            if not taskOnHold and not taskBroken and allUpdated and nFilesMasterReady == 0 and checkedMaster and taskSpec.is_workflow_holdup():
+            # task holdup by workflow if not enough master inputs are ready and parent steps are not all done
+            insufficient_ready_files = nFilesMasterReady == 0 or (nFilesPerJob and nFilesMasterReady < nFilesPerJob)
+            if not taskOnHold and not taskBroken and allUpdated and insufficient_ready_files and checkedMaster and taskSpec.is_workflow_holdup():
                 # hold up by the workflow
                 taskOnHold = True
                 tmpLog.debug("task to hold up by workflow")
             # no master input
             if not taskOnHold and not taskBroken and allUpdated and nFilesMaster == 0 and checkedMaster:
-                tmpErrStr = "no master input files. input dataset is empty"
+                tmpErrStr = "primary input dataset is empty or all files are excluded "
                 if master_offset:
                     tmpErrStr += f" with offset={master_offset}"
                 tmpLog.error(tmpErrStr)
@@ -688,7 +707,7 @@ class ContentsFeederThread(WorkerThread):
                     allRet = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(jediTaskID, taskSpec, pid=self.pid, setFrozenTime=setFrozenTime)
                 elif allUpdated:
                     # all OK
-                    allRet, newTaskStatus = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(
+                    _, newTaskStatus = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(
                         jediTaskID, getTaskStatus=True, pid=self.pid, useWorldCloud=taskSpec.useWorldCloud()
                     )
                     tmpMsg = f"set task_status={newTaskStatus}"

{panda_server_source-0.8.1 → panda_server_source-1.0.1}/pandajedi/jediorder/JediMaster.py

@@ -1,4 +1,3 @@
-import datetime
 import grp
 import multiprocessing
 import optparse