panda-server-source 0.8.0__tar.gz → 1.0.0__tar.gz

panda_server_source-1.0.0/.gitignore

@@ -0,0 +1,41 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Distribution / packaging
+*.whl
+*.tar.gz
+
+# Testing
+.tox/
+.coverage
+.coverage.*
+htmlcov/
+.pytest_cache/
+*.log
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Claude
+.claude/

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/Dockerfile

@@ -11,6 +11,8 @@ RUN /opt/panda/bin/pip install --no-cache-dir .[postgres,oracle,mcp]
 RUN /opt/panda/bin/pip install --no-cache-dir rucio-clients
 RUN /opt/panda/bin/pip install --no-cache-dir "git+https://github.com/PanDAWMS/panda-cacheschedconfig.git"
 RUN ln -s /opt/panda/lib/python*/site-packages/mod_wsgi/server/mod_wsgi*.so /etc/httpd/modules/mod_wsgi.so
+# Remove the default OS ssl.conf which references a non-existent localhost.crt and conflicts with panda's SSL config
+RUN rm -f /etc/httpd/conf.d/ssl.conf
 
 WORKDIR /
 RUN rm -rf /tmp/src

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: panda-server-source
-Version: 0.8.0
+Version: 1.0.0
 Summary: Unified PanDA Server and JEDI Package - Source Only
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>
@@ -11,6 +11,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
+Requires-Dist: panda-client-light
+Requires-Dist: panda-common
 Description-Content-Type: text/markdown
 
 # Monorepo for the PanDA Server and JEDI

panda_server_source-1.0.0/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "1.0.0"

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedibrokerage/AtlasAnalJobBroker.py

@@ -161,7 +161,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 totalJobStat = self.taskBufferIF.countJobsPerTarget_JEDI(taskSpec.origUserName, True)
             self.set_task_common("totalJobStat", totalJobStat)
         # check total to cap
-        if totalJobStat:
+        if totalJobStat and not inputChunk.isMerging:
             if taskSpec.workingGroup:
                 gdp_token_jobs = "CAP_RUNNING_GROUP_JOBS"
                 gdp_token_cores = "CAP_RUNNING_GROUP_CORES"
@@ -756,6 +756,30 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 retVal = retTmpError
                 continue
 
+            ######################################
+            # selection with IO intensity
+            if taskSpec.ioIntensity and not inputChunk.isMerging:
+                newScanSiteList = []
+                oldScanSiteList = copy.copy(scanSiteList)
+                msg_map = {}
+                for tmpSiteName in scanSiteList:
+                    tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
+                    max_io_intensity = tmpSiteSpec.get_max_io_intensity()
+                    if max_io_intensity is None or taskSpec.ioIntensity <= max_io_intensity:
+                        newScanSiteList.append(tmpSiteName)
+                    else:
+                        msg_map[tmpSiteSpec.get_unified_name()] = (
+                            f"  skip site={tmpSiteSpec.get_unified_name()} since ioIntensity={taskSpec.ioIntensity} "
+                            f"is larger than site max_io_intensity={max_io_intensity} criteria=-max_io_intensity"
+                        )
+                scanSiteList = newScanSiteList
+                self.add_summary_message(oldScanSiteList, scanSiteList, "max IO intensity check", tmpLog, msg_map)
+                if not scanSiteList:
+                    self.dump_summary(tmpLog)
+                    tmpLog.error("no candidates")
+                    retVal = retTmpError
+                    continue
+
             ######################################
             # selection for VP
             if taskSpec.avoid_vp() or avoidVP or not checkDataLocality:
@@ -1093,7 +1117,14 @@ class AtlasAnalJobBroker(JobBrokerBase):
             msg_map = {}
             for tmpSiteName in self.get_unified_sites(scanSiteList):
                 tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
-                # check at the site
+                # check direct access
+                if taskSpec.allowInputLAN() == "only" and not tmpSiteSpec.isDirectIO() and not tmpSiteSpec.always_use_direct_io() and not inputChunk.isMerging:
+                    tmp_msg = f"  skip site={tmpSiteName} since direct IO is disabled "
+                    tmp_msg += "criteria=-remoteio"
+                    msg_map[tmpSiteName] = tmp_msg
+                    continue
+
+                # check disk space
                 if tmpSiteSpec.maxwdir:
                     if CoreUtils.use_direct_io_for_job(taskSpec, tmpSiteSpec, inputChunk):
                         minDiskCount = minDiskCountR

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedibrokerage/AtlasProdJobBroker.py

@@ -1406,6 +1406,30 @@ class AtlasProdJobBroker(JobBrokerBase):
             taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
             return retTmpError
 
+        ######################################
+        # selection with IO intensity
+        if taskSpec.ioIntensity and not inputChunk.isMerging:
+            newScanSiteList = []
+            oldScanSiteList = copy.copy(scanSiteList)
+            msg_map = {}
+            for tmpSiteName in scanSiteList:
+                tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
+                max_io_intensity = tmpSiteSpec.get_max_io_intensity()
+                if max_io_intensity is None or taskSpec.ioIntensity <= max_io_intensity:
+                    newScanSiteList.append(tmpSiteName)
+                else:
+                    msg_map[tmpSiteSpec.get_unified_name()] = (
+                        f"  skip site={tmpSiteSpec.get_unified_name()} since ioIntensity={taskSpec.ioIntensity} "
+                        f"is larger than site max_io_intensity={max_io_intensity} criteria=-max_io_intensity"
+                    )
+            scanSiteList = newScanSiteList
+            self.add_summary_message(oldScanSiteList, scanSiteList, "max IO intensity check", tmpLog, msg_map)
+            if not scanSiteList:
+                self.dump_summary(tmpLog)
+                tmpLog.error("no candidates")
+                taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
+                return retTmpError
+
         ######################################
         # selection for full chain
         if nucleusSpec:
@@ -1679,6 +1703,8 @@ class AtlasProdJobBroker(JobBrokerBase):
             tmpLog.error("failed to get job statistics with priority")
             taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
             return retTmpError
+        # get resource types per site
+        site_resource_type_map = self.taskBufferIF.get_distinct_resource_types_per_site(taskSpec.jediTaskID)
         workerStat = self.taskBufferIF.ups_load_worker_stats()
         upsQueues = set(self.taskBufferIF.ups_get_queues())
         tmpLog.info(f"calculate weight and check cap for {len(scanSiteList)} candidates")
@@ -1767,31 +1793,37 @@ class AtlasProdJobBroker(JobBrokerBase):
                 useAssigned = True
             # stat with resource type
             RT_Cap = 2
-            if tmpSiteName in tmpStatMapRT and taskSpec.resource_type in tmpStatMapRT[tmpSiteName]:
-                useCapRT = True
-                tmpRTrunning = tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("running", 0)
-                tmpRTrunning = max(tmpRTrunning, nRunning)
-                tmpRTqueue = tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("defined", 0)
-                if useAssigned:
-                    tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("assigned", 0)
-                tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("activated", 0)
-                tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("starting", 0)
-            else:
-                useCapRT = False
-                tmpRTqueue = 0
-                tmpRTrunning = 0
+            useCapRT = False
+            tmpRTqueue = 0
+            tmpRTrunning = 0
+            resource_type_str = taskSpec.resource_type
+            if tmpSiteName in tmpStatMapRT:
+                # loop over all resource types for the site since task and job may have different resource types
+                tmp_resource_types = site_resource_type_map.get(tmpSiteName, [taskSpec.resource_type])
+                resource_type_str = ",".join(tmp_resource_types)
+                for tmp_resource_type in tmp_resource_types:
+                    if tmp_resource_type in tmpStatMapRT[tmpSiteName]:
+                        useCapRT = True
+                        tmpRTrunning += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("running", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("defined", 0)
+                        if useAssigned:
+                            tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("assigned", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("activated", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("starting", 0)
+                if useCapRT:
+                    tmpRTrunning = max(tmpRTrunning, nRunning)
             if totalSize == 0 or totalSize - siteSizeMap[tmpSiteName] <= 0:
                 weight = float(nRunning + 1) / float(nActivated + nStarting + nDefined + 10)
                 weightStr = (
                     f"nRun={nRunning} nAct={nActivated} nStart={nStarting} nDef={nDefined} nPilot={nPilot}{corrNumPilotStr} "
-                    f"totalSizeMB={int(totalSize / 1024 / 1024)} totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} "
+                    f"totalSizeMB={int(totalSize / 1024 / 1024)} totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} resource_type={resource_type_str} "
                 )
             else:
                 weight = float(nRunning + 1) / float(nActivated + nAssigned + nStarting + nDefined + 10) / manyAssigned
                 weightStr = (
                     f"nRun={nRunning} nAct={nActivated} nAss={nAssigned} nStart={nStarting} nDef={nDefined} manyAss={manyAssigned} "
                     f"nPilot={nPilot}{corrNumPilotStr} totalSizeMB={int(totalSize / 1024 / 1024)} "
-                    f"totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} "
+                    f"totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} resource_type={resource_type_str} "
                 )
 
             # reduce weights by taking data availability into account
@@ -1954,7 +1986,7 @@ class AtlasProdJobBroker(JobBrokerBase):
                     ngMsg += f"nDefined_rt+nActivated_rt+nAssigned_rt+nStarting_rt={tmpRTqueue} "
                 else:
                     ngMsg += f"nDefined_rt+nActivated_rt+nStarting_rt={tmpRTqueue} (nAssigned_rt ignored due to data locally available) "
-                ngMsg += "with gshare+resource_type is greater than max({cutOffValue},{RT_Cap}*nRun_rt={RT_Cap}*{tmpRTrunning}) criteria=-cap_rt"
+                ngMsg += f"with gshare+resource_type({resource_type_str}) is greater than max({cutOffValue},{RT_Cap}*nRun_rt={RT_Cap}*{tmpRTrunning}) criteria=-cap_rt"
             elif (
                 nRunning + nActivated + nAssigned + nStarting + nDefined == 0
                 and taskSpec.currentPriority <= self.max_prio_for_bootstrap

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedibrokerage/AtlasProdTaskBroker.py

@@ -354,7 +354,7 @@ class AtlasProdTaskBrokerThread(WorkerThread):
                                 write_wan_status = tmpEP["detailed_status"].get("write_wan")
                                 if write_wan_status in ["OFF", "TEST"]:
                                     tmpLog.info(
-                                        f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has write_wan={read_wan_status} criteria=-destination_blacklist"
+                                        f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has write_wan={write_wan_status} criteria=-destination_blacklist"
                                     )
                                     to_skip = True
                                     break

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedibrokerage/GenJobBroker.py

@@ -7,6 +7,7 @@ from pandajedi.jedicore import Interaction
 from pandajedi.jedicore.MsgWrapper import MsgWrapper
 from pandajedi.jedicore.SiteCandidate import SiteCandidate
 from pandajedi.jedirefine import RefinerUtils
+from pandaserver.config import panda_config
 from pandaserver.srvcore import CoreUtils
 
 from . import AtlasBrokerUtils
@@ -277,6 +278,7 @@ class GenJobBroker(JobBrokerBase):
         ######################################
         # get list of available files
         availableFileMap = {}
+        total_num_files = 0
         for datasetSpec in inputChunk.getDatasets():
             try:
                 # get list of site to be scanned
@@ -312,6 +314,7 @@ class GenJobBroker(JobBrokerBase):
                 if tmpAvFileMap is None:
                     raise Interaction.JEDITemporaryError("ddmIF.getAvailableFiles failed")
                 availableFileMap[datasetSpec.datasetName] = tmpAvFileMap
+                total_num_files += len(datasetSpec.Files)
             except Exception as e:
                 tmpLog.error(f"failed to get available files with {e}")
                 taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
@@ -367,7 +370,12 @@ class GenJobBroker(JobBrokerBase):
         # append candidates
         newScanSiteList = []
         for siteCandidateSpec in candidateSpecList:
-            # append
+            # append only if all files are available at the site when file dispatch is disabled
+            if panda_config.disable_file_dispatch and len(siteCandidateSpec.localDiskFiles) < total_num_files:
+                tmpLog.debug(
+                    f"  skip {siteCandidateSpec.siteName} due to input file dispatch is disabled and not all files are available at the site ({len(siteCandidateSpec.localDiskFiles)} < {total_num_files})"
+                )
+                continue
             inputChunk.addSiteCandidate(siteCandidateSpec)
             newScanSiteList.append(siteCandidateSpec.siteName)
             tmpLog.debug(f"  use {siteCandidateSpec.siteName} with weight={siteCandidateSpec.weight} nFiles={len(siteCandidateSpec.localDiskFiles)}")

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedicore/JediTaskBuffer.py

@@ -198,11 +198,6 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.getTaskDatasetsWithID_JEDI(jediTaskID, pid, lockTask)
 
-    # get JEDI tasks with selection criteria
-    def getTaskIDsWithCriteria_JEDI(self, criteria, nTasks=50):
-        with self.proxyPool.get() as proxy:
-            return proxy.getTaskIDsWithCriteria_JEDI(criteria, nTasks)
-
     # get JEDI tasks to be finished
     def getTasksToBeFinished_JEDI(self, vo, prodSourceLabel, pid, nTasks=50, target_tasks=None):
         with self.proxyPool.get() as proxy:
@@ -677,6 +672,11 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.throttleTask_JEDI(jediTaskID, waitTime, errorDialog)
 
+    # throttle tasks of a user or working group and get the list of throttled tasks
+    def throttle_tasks_with_uid(self, vo: str, prod_source_label: str, wait_time: int, dialog: str, uid: str, is_user: bool = True):
+        with self.proxyPool.get() as proxy:
+            return proxy.throttle_tasks_with_uid(vo, prod_source_label, wait_time, dialog, uid, is_user)
+
     # release throttled tasks
     def releaseThrottledTasks_JEDI(self, vo, prodSourceLabel):
         with self.proxyPool.get() as proxy:
@@ -757,12 +757,16 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.getFailureCountsForTask_JEDI(jediTaskID, timeWindow)
 
-        # count the number of queued jobs per user or working group
-
+    # count the number of queued jobs per user or working group
     def countJobsPerTarget_JEDI(self, target, is_user):
         with self.proxyPool.get() as proxy:
             return proxy.countJobsPerTarget_JEDI(target, is_user)
 
+    # count the number of jobs and cores per user or working group in VO and production source label
+    def count_jobs_per_uid_JEDI(self, vo, prod_source_label):
+        with self.proxyPool.get() as proxy:
+            return proxy.count_jobs_per_uid_JEDI(vo, prod_source_label)
+
     # get old merge job PandaIDs
     def getOldMergeJobPandaIDs_JEDI(self, jediTaskID, pandaID):
         with self.proxyPool.get() as proxy:

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedidog/AtlasAnalWatchDog.py

@@ -149,7 +149,8 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
                 maxTransfer = 1024
             # throttle interval
             thrInterval = 120
-            # loop over all users
+            # loop over all user tasks with data motion
+            user_motion_to_keep_throttled = {}
             for userName, userDict in dispUserTasks.items():
                 # loop over all transfer types
                 for transferType, maxSize in [("prestaging", maxPrestaging), ("transfer", maxTransfer)]:
@@ -166,17 +167,116 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
                                 tmpLog.debug(f"action=throttle_{transferType} jediTaskID={taskID} for user={userName}")
                                 errDiag = f"throttled since transferring large data volume in total={userTotal}GB > limit={maxSize}GB type={transferType}"
                                 self.taskBufferIF.throttleTask_JEDI(taskID, thrInterval, errDiag)
-                        # remove the user from the list
-                        if userName in thrUserTasks and transferType in thrUserTasks[userName]:
-                            del thrUserTasks[userName][transferType]
-            # release users
+                        # keep throttled
+                        user_motion_to_keep_throttled.setdefault(userName, set()).add(transferType)
+            # release tasks related to data motion
             for userName, taskData in thrUserTasks.items():
                 for transferType, taskIDs in taskData.items():
+                    if transferType not in ["prestaging", "transfer"]:
+                        continue
+                    if userName in user_motion_to_keep_throttled and transferType in user_motion_to_keep_throttled[userName]:
+                        continue
                     tmpLog.debug(f"user={userName} release throttled tasks with {transferType}")
                     # unthrottle tasks
                     for taskID in taskIDs:
                         tmpLog.debug(f"action=release_{transferType} jediTaskID={taskID} for user={userName}")
                         self.taskBufferIF.release_task_on_hold(taskID, "throttled")
+
+            # throttle users with too many running/queued jobs
+            queue_factor = 2
+            throttle_threshold = 0.95
+            release_threshold = 0.85
+            gdp_token_group_jobs = "CAP_RUNNING_GROUP_JOBS"
+            gdp_token_group_cores = "CAP_RUNNING_GROUP_CORES"
+            max_num_run_group_jobs = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_group_jobs)
+            max_num_run_group_cores = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_group_cores)
+            gdp_token_user_jobs = "CAP_RUNNING_USER_JOBS"
+            gdp_token_user_cores = "CAP_RUNNING_USER_CORES"
+            max_num_run_user_jobs = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_user_jobs)
+            max_num_run_user_cores = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_user_cores)
+            # get job count per uid
+            job_count_per_uid = self.taskBufferIF.count_jobs_per_uid_JEDI(self.vo, self.prodSourceLabel)
+            users_to_keep_throttled = set()
+            for uid, count_dict in job_count_per_uid.items():
+                to_throttle = True
+                to_release = True
+                msg = None
+                if count_dict["is_user"]:
+                    # user
+                    if max_num_run_user_jobs:
+                        if count_dict["nRunJobs"] > max_num_run_user_jobs * throttle_threshold:
+                            msg = f"""throttle due to too many running jobs {count_dict["nRunJobs"]} > {gdp_token_user_jobs}x{throttle_threshold} type=jobs"""
+                        elif count_dict["nQueuedJobs"] > max_num_run_user_jobs * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued jobs {count_dict["nQueuedJobs"]} > {gdp_token_user_jobs}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None and max_num_run_user_cores:
+                        if count_dict["nRunCores"] > max_num_run_user_cores:
+                            msg = (
+                                f"""throttle due to too many running cores {count_dict["nRunCores"]} > {gdp_token_user_cores}x{throttle_threshold} type=jobs"""
+                            )
+                        elif count_dict["nQueuedCores"] > max_num_run_user_cores * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued cores {count_dict["nQueuedCores"]} > {gdp_token_user_cores}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None:
+                        to_throttle = False
+                    # check if can be released
+                    if max_num_run_user_jobs:
+                        if (
+                            count_dict["nRunJobs"] > max_num_run_user_jobs * release_threshold
+                            or count_dict["nQueuedJobs"] > max_num_run_user_jobs * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                    if max_num_run_user_cores:
+                        if (
+                            count_dict["nRunCores"] > max_num_run_user_cores * release_threshold
+                            or count_dict["nQueuedCores"] > max_num_run_user_cores * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                else:
+                    # group
+                    if max_num_run_group_jobs:
+                        if count_dict["nRunJobs"] > max_num_run_group_jobs:
+                            msg = f"""throttle due to too many running jobs {count_dict["nRunJobs"]} > {gdp_token_group_jobs}x{throttle_threshold} type=jobs"""
+                        elif count_dict["nQueuedJobs"] > max_num_run_group_jobs * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued jobs {count_dict["nQueuedJobs"]} > {gdp_token_group_jobs}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None and max_num_run_group_cores:
+                        if count_dict["nRunCores"] > max_num_run_group_cores:
+                            msg = (
+                                f"""throttle due to too many running cores {count_dict["nRunCores"]} > {gdp_token_group_cores}x{throttle_threshold} type=jobs"""
+                            )
+                        elif count_dict["nQueuedCores"] > max_num_run_group_cores * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued cores {count_dict["nQueuedCores"]} > {gdp_token_group_cores}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None:
+                        to_throttle = False
+                    # check if can be released
+                    if max_num_run_group_jobs:
+                        if (
+                            count_dict["nRunJobs"] > max_num_run_group_jobs * release_threshold
+                            or count_dict["nQueuedJobs"] > max_num_run_group_jobs * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                    if max_num_run_group_cores:
+                        if (
+                            count_dict["nRunCores"] > max_num_run_group_cores * release_threshold
+                            or count_dict["nQueuedCores"] > max_num_run_group_cores * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                if to_throttle:
+                    tmpLog.debug(f"uid={uid} {msg}")
+                    throttled_tasks = self.taskBufferIF.throttle_tasks_with_uid(self.vo, self.prodSourceLabel, thrInterval, msg, uid, count_dict["is_user"])
+                    tmpLog.debug(f"throttled tasks: {throttled_tasks}")
+                if to_throttle or not to_release:
+                    users_to_keep_throttled.add(uid)
+            # unthrottle users which are currently throttled but no more exceed the limit
+            for userName, taskData in thrUserTasks.items():
+                for transferType, taskIDs in taskData.items():
+                    if transferType not in ["jobs"]:
+                        continue
+                    if userName in users_to_keep_throttled:
+                        continue
+                    tmpLog.debug(f"user={userName} release throttled tasks since no more excess of queued and running job counts")
+                    # unthrottle tasks
+                    for taskID in taskIDs:
+                        tmpLog.debug(f"action=release_{transferType} jediTaskID={taskID} for user={userName}")
+                        self.taskBufferIF.release_task_on_hold(taskID, "throttled")
             tmpLog.debug("done")
         except Exception:
             errtype, errvalue = sys.exc_info()[:2]

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jediorder/ContentsFeeder.py

@@ -125,7 +125,7 @@ class ContentsFeederThread(WorkerThread):
             datasetsIdxConsistency = []
 
             # get task
-            tmpStat, taskSpec = self.taskBufferIF.getTaskWithID_JEDI(jediTaskID, False, real_run, self.pid, 10, clearError=True)
+            tmpStat, taskSpec = self.taskBufferIF.getTaskWithID_JEDI(jediTaskID, False, real_run, self.pid, 10)
             if not tmpStat or taskSpec is None:
                 self.logger.debug(f"failed to get taskSpec for jediTaskID={jediTaskID}")
                 continue
@@ -627,14 +627,15 @@ class ContentsFeederThread(WorkerThread):
                                 setFrozenTime = False
                                 skip_secondaries = True
                     tmpLog.debug("end loop")
-            # task holdup by workflow if no master inputs are ready
-            if not taskOnHold and not taskBroken and allUpdated and nFilesMasterReady == 0 and checkedMaster and taskSpec.is_workflow_holdup():
+            # task holdup by workflow if not enough master inputs are ready and parent steps are not all done
+            insufficient_ready_files = nFilesMasterReady == 0 or (nFilesPerJob and nFilesMasterReady < nFilesPerJob)
+            if not taskOnHold and not taskBroken and allUpdated and insufficient_ready_files and checkedMaster and taskSpec.is_workflow_holdup():
                 # hold up by the workflow
                 taskOnHold = True
                 tmpLog.debug("task to hold up by workflow")
             # no master input
             if not taskOnHold and not taskBroken and allUpdated and nFilesMaster == 0 and checkedMaster:
-                tmpErrStr = "no master input files. input dataset is empty"
+                tmpErrStr = "primary input dataset is empty or all files are excluded "
                 if master_offset:
                     tmpErrStr += f" with offset={master_offset}"
                 tmpLog.error(tmpErrStr)
@@ -688,7 +689,7 @@ class ContentsFeederThread(WorkerThread):
                     allRet = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(jediTaskID, taskSpec, pid=self.pid, setFrozenTime=setFrozenTime)
                 elif allUpdated:
                     # all OK
-                    allRet, newTaskStatus = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(
+                    _, newTaskStatus = self.taskBufferIF.updateTaskStatusByContFeeder_JEDI(
                         jediTaskID, getTaskStatus=True, pid=self.pid, useWorldCloud=taskSpec.useWorldCloud()
                     )
                     tmpMsg = f"set task_status={newTaskStatus}"

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jediorder/JobGenerator.py

@@ -1318,7 +1318,7 @@ class JobGeneratorThread(WorkerThread):
                                 jobSpec.transferType = siteCandidate.remoteProtocol
                                 jobSpec.sourceSite = siteCandidate.remoteSource
                             elif not inputChunk.isMerging and not taskSpec.useLocalIO() and (taskSpec.allowInputLAN() is not None and siteSpec.isDirectIO()):
-                                jobSpec.transferType = "direct"
+                                jobSpec.transferType = taskParamMap.get("transferType", "direct")
                             # collect old PandaIDs
                             if tmpFileSpec.PandaID is not None and tmpFileSpec.PandaID not in subOldPandaIDs:
                                 subOldPandaIDs.append(tmpFileSpec.PandaID)
@@ -1503,6 +1503,8 @@ class JobGeneratorThread(WorkerThread):
                     # on-site merging
                     if taskSpec.on_site_merging():
                         jobSpec.set_on_site_merging()
+                    if inputChunk.isMerging:
+                        jobSpec.set_pmerge_job_flag()
                     # set task queued time
                     jobSpec.set_task_queued_time(task_queued_time)
                     # extract middle name
@@ -1985,6 +1987,7 @@ class JobGeneratorThread(WorkerThread):
             specialHandling = specialHandling[:-1]
             if specialHandling != "":
                 jobSpec.specialHandling = specialHandling
+            jobSpec.set_build_job_flag()
             jobSpec.set_task_queued_time(taskSpec.get_queued_time())
             libDsFileNameBase = libDsName + ".$JEDIFILEID"
             # make lib.tgz

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jediorder/JobSplitter.py

@@ -210,6 +210,8 @@ class JobSplitter:
                     maxWalltime = taskSpec.getMaxWalltime()
                 if maxWalltime is None:
                     maxWalltime = siteSpec.maxtime
+                elif maxWalltime > siteSpec.maxtime:
+                    maxWalltime = siteSpec.maxtime
                 # core count
                 if siteSpec.coreCount:
                     coreCount = siteSpec.coreCount

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedipprocess/PostProcessorBase.py

@@ -8,6 +8,7 @@ from pandacommon.pandautils.PandaUtils import naive_utcnow
 from pandajedi.jedicore import Interaction
 from pandaserver.config import panda_config
 from pandaserver.taskbuffer import EventServiceUtils
+from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
 
 # port for SMTP server
 smtpPortList = [25, 587]
@@ -60,7 +61,7 @@ class PostProcessorBase(object):
     def doBasicPostProcess(self, taskSpec, tmpLog):
         # update task status
         taskSpec.lockedBy = None
-        taskSpec.status = self.getFinalTaskStatus(taskSpec)
+        taskSpec.status = self.getFinalTaskStatus(taskSpec, update_error_dialog=True)
         if taskSpec.status == "failed":
             # set dialog for preprocess
             if taskSpec.usePrePro() and not taskSpec.checkPreProcessed():
@@ -177,7 +178,17 @@ class PostProcessorBase(object):
         return nFiles, nFilesFinished, totalInputEvents, totalOkEvents, taskCompleteness
 
     # get final task status
-    def getFinalTaskStatus(self, taskSpec, checkParent=True, checkGoal=False):
+    def getFinalTaskStatus(self, taskSpec: JediTaskSpec, checkParent: bool = True, checkGoal: bool = False, update_error_dialog: bool = False):
+        """
+        Get final task status based on the number of files and events, and the status of parent task if needed.
+
+        Args:
+        - taskSpec: the task specification object
+        - checkParent: if True, check the status of parent task and set failed if the parent task is failed, broken, or aborted. If False, ignore the parent status.
+        - checkGoal: if True, return True if the goal is reached and False otherwise. If False, return the final task status.
+        - update_error_dialog: if True, update error dialog for failed tasks
+
+        """
         # count nFiles and nEvents
         nFiles, nFilesFinished, totalInputEvents, totalOkEvents, taskCompleteness = self.getTaskCompleteness(taskSpec)
         # set new task status
@@ -189,12 +200,16 @@ class PostProcessorBase(object):
             status = "paused"
         elif self.failOnZeroOkFile and nFiles == nFilesFinished == 0:
             status = "failed"
+            if update_error_dialog:
+                taskSpec.setErrDiag("failed due to no input", True)
         elif nFiles == nFilesFinished:
             # check parent status
             if checkParent and taskSpec.parent_tid not in [None, taskSpec.jediTaskID]:
                 parent_status = self.taskBufferIF.getTaskStatus_JEDI(taskSpec.parent_tid)
                 if parent_status in ["failed", "broken", "aborted"]:
                     status = "failed"
+                    if update_error_dialog:
+                        taskSpec.setErrDiag(f"failed since the parent task is {parent_status} although all files were processed", True)
                 elif parent_status != "done":
                     status = "finished"
                 else:
@@ -212,6 +227,8 @@ class PostProcessorBase(object):
                 status = "done"
         elif nFilesFinished == 0:
             status = "failed"
+            if update_error_dialog:
+                taskSpec.setErrDiag("failed since no file was successfully processed", True)
         else:
             status = "finished"
         # task goal
@@ -227,6 +244,8 @@ class PostProcessorBase(object):
         ):
             if taskCompleteness < taskGoal:
                 status = "failed"
+                if update_error_dialog:
+                    taskSpec.setErrDiag(f"failed since the task completeness {taskCompleteness:.1f} is less than the goal {taskGoal:.1f}", True)
         # HPO tasks always go to finished
         if taskSpec.is_hpo_workflow():
             event_stat = self.taskBufferIF.get_event_statistics(taskSpec.jediTaskID)
@@ -242,7 +261,7 @@ class PostProcessorBase(object):
         return status
 
     # pre-check
-    def doPreCheck(self, taskSpec, tmpLog):
+    def doPreCheck(self, taskSpec: JediTaskSpec, tmpLog):
         # send task to exhausted
         if (
             taskSpec.useExhausted()

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedirefine/AtlasAnalTaskRefiner.py

@@ -241,11 +241,11 @@ class AtlasAnalTaskRefiner(TaskRefinerBase):
             self.taskSpec.reqID = tmpJobID
             # site limitation
             if "excludedSite" in taskParamMap and "includedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("incexc")
+                self.taskSpec.setLimitedSites("with_allow_and_denylist")
             elif "excludedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("exc")
+                self.taskSpec.setLimitedSites("with_denylist")
             elif "includedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("inc")
+                self.taskSpec.setLimitedSites("with_allowlist")
             # input prestaging
             if self.taskSpec.inputPreStaging():
                 # set first contents feed flag

{panda_server_source-0.8.0 → panda_server_source-1.0.0}/pandajedi/jedisetup/AtlasTaskSetupper.py

@@ -197,7 +197,7 @@ class AtlasTaskSetupper(TaskSetupperBase):
                                                         targetName,
                                                         locForDouble,
                                                         copies=1,
-                                                        owner=userName,
+                                                        owner=None,
                                                         lifetime=lifetime,
                                                         activity=activity,
                                                         grouping="NONE",