panda-server-source 0.8.0__tar.gz → 0.8.1__tar.gz

panda_server_source-0.8.1/.gitignore

@@ -0,0 +1,41 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Distribution / packaging
+*.whl
+*.tar.gz
+
+# Testing
+.tox/
+.coverage
+.coverage.*
+htmlcov/
+.pytest_cache/
+*.log
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Claude
+.claude/

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/Dockerfile

@@ -11,6 +11,8 @@ RUN /opt/panda/bin/pip install --no-cache-dir .[postgres,oracle,mcp]
 RUN /opt/panda/bin/pip install --no-cache-dir rucio-clients
 RUN /opt/panda/bin/pip install --no-cache-dir "git+https://github.com/PanDAWMS/panda-cacheschedconfig.git"
 RUN ln -s /opt/panda/lib/python*/site-packages/mod_wsgi/server/mod_wsgi*.so /etc/httpd/modules/mod_wsgi.so
+# Remove the default OS ssl.conf which references a non-existent localhost.crt and conflicts with panda's SSL config
+RUN rm -f /etc/httpd/conf.d/ssl.conf
 
 WORKDIR /
 RUN rm -rf /tmp/src

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: panda-server-source
-Version: 0.8.0
+Version: 0.8.1
 Summary: Unified PanDA Server and JEDI Package - Source Only
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>

panda_server_source-0.8.1/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "0.8.1"

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedibrokerage/AtlasAnalJobBroker.py

@@ -161,7 +161,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 totalJobStat = self.taskBufferIF.countJobsPerTarget_JEDI(taskSpec.origUserName, True)
             self.set_task_common("totalJobStat", totalJobStat)
         # check total to cap
-        if totalJobStat:
+        if totalJobStat and not inputChunk.isMerging:
             if taskSpec.workingGroup:
                 gdp_token_jobs = "CAP_RUNNING_GROUP_JOBS"
                 gdp_token_cores = "CAP_RUNNING_GROUP_CORES"
@@ -756,6 +756,30 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 retVal = retTmpError
                 continue
 
+            ######################################
+            # selection with IO intensity
+            if taskSpec.ioIntensity and not inputChunk.isMerging:
+                newScanSiteList = []
+                oldScanSiteList = copy.copy(scanSiteList)
+                msg_map = {}
+                for tmpSiteName in scanSiteList:
+                    tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
+                    max_io_intensity = tmpSiteSpec.get_max_io_intensity()
+                    if max_io_intensity is None or taskSpec.ioIntensity <= max_io_intensity:
+                        newScanSiteList.append(tmpSiteName)
+                    else:
+                        msg_map[tmpSiteSpec.get_unified_name()] = (
+                            f"  skip site={tmpSiteSpec.get_unified_name()} since ioIntensity={taskSpec.ioIntensity} "
+                            f"is larger than site max_io_intensity={max_io_intensity} criteria=-max_io_intensity"
+                        )
+                scanSiteList = newScanSiteList
+                self.add_summary_message(oldScanSiteList, scanSiteList, "max IO intensity check", tmpLog, msg_map)
+                if not scanSiteList:
+                    self.dump_summary(tmpLog)
+                    tmpLog.error("no candidates")
+                    retVal = retTmpError
+                    continue
+
             ######################################
             # selection for VP
             if taskSpec.avoid_vp() or avoidVP or not checkDataLocality:

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedibrokerage/AtlasProdJobBroker.py

@@ -1406,6 +1406,30 @@ class AtlasProdJobBroker(JobBrokerBase):
             taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
             return retTmpError
 
+        ######################################
+        # selection with IO intensity
+        if taskSpec.ioIntensity and not inputChunk.isMerging:
+            newScanSiteList = []
+            oldScanSiteList = copy.copy(scanSiteList)
+            msg_map = {}
+            for tmpSiteName in scanSiteList:
+                tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
+                max_io_intensity = tmpSiteSpec.get_max_io_intensity()
+                if max_io_intensity is None or taskSpec.ioIntensity <= max_io_intensity:
+                    newScanSiteList.append(tmpSiteName)
+                else:
+                    msg_map[tmpSiteSpec.get_unified_name()] = (
+                        f"  skip site={tmpSiteSpec.get_unified_name()} since ioIntensity={taskSpec.ioIntensity} "
+                        f"is larger than site max_io_intensity={max_io_intensity} criteria=-max_io_intensity"
+                    )
+            scanSiteList = newScanSiteList
+            self.add_summary_message(oldScanSiteList, scanSiteList, "max IO intensity check", tmpLog, msg_map)
+            if not scanSiteList:
+                self.dump_summary(tmpLog)
+                tmpLog.error("no candidates")
+                taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
+                return retTmpError
+
         ######################################
         # selection for full chain
         if nucleusSpec:
@@ -1679,6 +1703,8 @@ class AtlasProdJobBroker(JobBrokerBase):
             tmpLog.error("failed to get job statistics with priority")
             taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
             return retTmpError
+        # get resource types per site
+        site_resource_type_map = self.taskBufferIF.get_distinct_resource_types_per_site(taskSpec.jediTaskID)
         workerStat = self.taskBufferIF.ups_load_worker_stats()
         upsQueues = set(self.taskBufferIF.ups_get_queues())
         tmpLog.info(f"calculate weight and check cap for {len(scanSiteList)} candidates")
@@ -1767,31 +1793,37 @@ class AtlasProdJobBroker(JobBrokerBase):
                 useAssigned = True
             # stat with resource type
             RT_Cap = 2
-            if tmpSiteName in tmpStatMapRT and taskSpec.resource_type in tmpStatMapRT[tmpSiteName]:
-                useCapRT = True
-                tmpRTrunning = tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("running", 0)
-                tmpRTrunning = max(tmpRTrunning, nRunning)
-                tmpRTqueue = tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("defined", 0)
-                if useAssigned:
-                    tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("assigned", 0)
-                tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("activated", 0)
-                tmpRTqueue += tmpStatMapRT[tmpSiteName][taskSpec.resource_type].get("starting", 0)
-            else:
-                useCapRT = False
-                tmpRTqueue = 0
-                tmpRTrunning = 0
+            useCapRT = False
+            tmpRTqueue = 0
+            tmpRTrunning = 0
+            resource_type_str = taskSpec.resource_type
+            if tmpSiteName in tmpStatMapRT:
+                # loop over all resource types for the site since task and job may have different resource types
+                tmp_resource_types = site_resource_type_map.get(tmpSiteName, [taskSpec.resource_type])
+                resource_type_str = ",".join(tmp_resource_types)
+                for tmp_resource_type in tmp_resource_types:
+                    if tmp_resource_type in tmpStatMapRT[tmpSiteName]:
+                        useCapRT = True
+                        tmpRTrunning += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("running", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("defined", 0)
+                        if useAssigned:
+                            tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("assigned", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("activated", 0)
+                        tmpRTqueue += tmpStatMapRT[tmpSiteName][tmp_resource_type].get("starting", 0)
+                if useCapRT:
+                    tmpRTrunning = max(tmpRTrunning, nRunning)
             if totalSize == 0 or totalSize - siteSizeMap[tmpSiteName] <= 0:
                 weight = float(nRunning + 1) / float(nActivated + nStarting + nDefined + 10)
                 weightStr = (
                     f"nRun={nRunning} nAct={nActivated} nStart={nStarting} nDef={nDefined} nPilot={nPilot}{corrNumPilotStr} "
-                    f"totalSizeMB={int(totalSize / 1024 / 1024)} totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} "
+                    f"totalSizeMB={int(totalSize / 1024 / 1024)} totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} resource_type={resource_type_str} "
                 )
             else:
                 weight = float(nRunning + 1) / float(nActivated + nAssigned + nStarting + nDefined + 10) / manyAssigned
                 weightStr = (
                     f"nRun={nRunning} nAct={nActivated} nAss={nAssigned} nStart={nStarting} nDef={nDefined} manyAss={manyAssigned} "
                     f"nPilot={nPilot}{corrNumPilotStr} totalSizeMB={int(totalSize / 1024 / 1024)} "
-                    f"totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} "
+                    f"totalNumFiles={maxNumFiles} nRun_rt={tmpRTrunning} nQueued_rt={tmpRTqueue} resource_type={resource_type_str} "
                 )
 
             # reduce weights by taking data availability into account
@@ -1954,7 +1986,7 @@ class AtlasProdJobBroker(JobBrokerBase):
                     ngMsg += f"nDefined_rt+nActivated_rt+nAssigned_rt+nStarting_rt={tmpRTqueue} "
                 else:
                     ngMsg += f"nDefined_rt+nActivated_rt+nStarting_rt={tmpRTqueue} (nAssigned_rt ignored due to data locally available) "
-                ngMsg += "with gshare+resource_type is greater than max({cutOffValue},{RT_Cap}*nRun_rt={RT_Cap}*{tmpRTrunning}) criteria=-cap_rt"
+                ngMsg += f"with gshare+resource_type({resource_type_str}) is greater than max({cutOffValue},{RT_Cap}*nRun_rt={RT_Cap}*{tmpRTrunning}) criteria=-cap_rt"
             elif (
                 nRunning + nActivated + nAssigned + nStarting + nDefined == 0
                 and taskSpec.currentPriority <= self.max_prio_for_bootstrap

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedibrokerage/AtlasProdTaskBroker.py

@@ -354,7 +354,7 @@ class AtlasProdTaskBrokerThread(WorkerThread):
                                 write_wan_status = tmpEP["detailed_status"].get("write_wan")
                                 if write_wan_status in ["OFF", "TEST"]:
                                     tmpLog.info(
-                                        f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has write_wan={read_wan_status} criteria=-destination_blacklist"
+                                        f"  skip nucleus={tmpNucleus} since {tmp_ddm_endpoint_name} has write_wan={write_wan_status} criteria=-destination_blacklist"
                                     )
                                     to_skip = True
                                     break

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedibrokerage/GenJobBroker.py

@@ -7,6 +7,7 @@ from pandajedi.jedicore import Interaction
 from pandajedi.jedicore.MsgWrapper import MsgWrapper
 from pandajedi.jedicore.SiteCandidate import SiteCandidate
 from pandajedi.jedirefine import RefinerUtils
+from pandaserver.config import panda_config
 from pandaserver.srvcore import CoreUtils
 
 from . import AtlasBrokerUtils
@@ -277,6 +278,7 @@ class GenJobBroker(JobBrokerBase):
         ######################################
         # get list of available files
         availableFileMap = {}
+        total_num_files = 0
         for datasetSpec in inputChunk.getDatasets():
             try:
                 # get list of site to be scanned
@@ -312,6 +314,7 @@ class GenJobBroker(JobBrokerBase):
                 if tmpAvFileMap is None:
                     raise Interaction.JEDITemporaryError("ddmIF.getAvailableFiles failed")
                 availableFileMap[datasetSpec.datasetName] = tmpAvFileMap
+                total_num_files += len(datasetSpec.Files)
             except Exception as e:
                 tmpLog.error(f"failed to get available files with {e}")
                 taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
@@ -369,6 +372,11 @@ class GenJobBroker(JobBrokerBase):
         for siteCandidateSpec in candidateSpecList:
             # append
             inputChunk.addSiteCandidate(siteCandidateSpec)
+            if panda_config.disable_file_dispatch and len(siteCandidateSpec.localDiskFiles) < total_num_files:
+                tmpLog.debug(
+                    f"  skip {siteCandidateSpec.siteName} due to input file dispatch is disabled and not all files are available at the site ({len(siteCandidateSpec.localDiskFiles)} < {total_num_files})"
+                )
+                continue
             newScanSiteList.append(siteCandidateSpec.siteName)
             tmpLog.debug(f"  use {siteCandidateSpec.siteName} with weight={siteCandidateSpec.weight} nFiles={len(siteCandidateSpec.localDiskFiles)}")
         scanSiteList = newScanSiteList

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedicore/JediTaskBuffer.py

@@ -677,6 +677,11 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.throttleTask_JEDI(jediTaskID, waitTime, errorDialog)
 
+    # throttle tasks of a user or working group and get the list of throttled tasks
+    def throttle_tasks_with_uid(self, vo: str, prod_source_label: str, wait_time: int, dialog: str, uid: str, is_user: bool = True):
+        with self.proxyPool.get() as proxy:
+            return proxy.throttle_tasks_with_uid(vo, prod_source_label, wait_time, dialog, uid, is_user)
+
     # release throttled tasks
     def releaseThrottledTasks_JEDI(self, vo, prodSourceLabel):
         with self.proxyPool.get() as proxy:
@@ -757,12 +762,16 @@ class JediTaskBuffer(TaskBuffer.TaskBuffer, CommandReceiveInterface):
         with self.proxyPool.get() as proxy:
             return proxy.getFailureCountsForTask_JEDI(jediTaskID, timeWindow)
 
-        # count the number of queued jobs per user or working group
-
+    # count the number of queued jobs per user or working group
     def countJobsPerTarget_JEDI(self, target, is_user):
         with self.proxyPool.get() as proxy:
             return proxy.countJobsPerTarget_JEDI(target, is_user)
 
+    # count the number of jobs and cores per user or working group in VO and production source label
+    def count_jobs_per_uid_JEDI(self, vo, prod_source_label):
+        with self.proxyPool.get() as proxy:
+            return proxy.count_jobs_per_uid_JEDI(vo, prod_source_label)
+
     # get old merge job PandaIDs
     def getOldMergeJobPandaIDs_JEDI(self, jediTaskID, pandaID):
         with self.proxyPool.get() as proxy:

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedidog/AtlasAnalWatchDog.py

@@ -149,7 +149,8 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
                 maxTransfer = 1024
             # throttle interval
             thrInterval = 120
-            # loop over all users
+            # loop over all user tasks with data motion
+            user_motion_to_keep_throttled = {}
             for userName, userDict in dispUserTasks.items():
                 # loop over all transfer types
                 for transferType, maxSize in [("prestaging", maxPrestaging), ("transfer", maxTransfer)]:
@@ -166,17 +167,116 @@ class AtlasAnalWatchDog(TypicalWatchDogBase):
                                 tmpLog.debug(f"action=throttle_{transferType} jediTaskID={taskID} for user={userName}")
                                 errDiag = f"throttled since transferring large data volume in total={userTotal}GB > limit={maxSize}GB type={transferType}"
                                 self.taskBufferIF.throttleTask_JEDI(taskID, thrInterval, errDiag)
-                        # remove the user from the list
-                        if userName in thrUserTasks and transferType in thrUserTasks[userName]:
-                            del thrUserTasks[userName][transferType]
-            # release users
+                        # keep throttled
+                        user_motion_to_keep_throttled.setdefault(userName, set()).add(transferType)
+            # release tasks related to data motion
             for userName, taskData in thrUserTasks.items():
                 for transferType, taskIDs in taskData.items():
+                    if transferType not in ["prestaging", "transfer"]:
+                        continue
+                    if userName in user_motion_to_keep_throttled and transferType in user_motion_to_keep_throttled[userName]:
+                        continue
                     tmpLog.debug(f"user={userName} release throttled tasks with {transferType}")
                     # unthrottle tasks
                     for taskID in taskIDs:
                         tmpLog.debug(f"action=release_{transferType} jediTaskID={taskID} for user={userName}")
                         self.taskBufferIF.release_task_on_hold(taskID, "throttled")
+
+            # throttle users with too many running/queued jobs
+            queue_factor = 2
+            throttle_threshold = 0.95
+            release_threshold = 0.85
+            gdp_token_group_jobs = "CAP_RUNNING_GROUP_JOBS"
+            gdp_token_group_cores = "CAP_RUNNING_GROUP_CORES"
+            max_num_run_group_jobs = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_group_jobs)
+            max_num_run_group_cores = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_group_cores)
+            gdp_token_user_jobs = "CAP_RUNNING_USER_JOBS"
+            gdp_token_user_cores = "CAP_RUNNING_USER_CORES"
+            max_num_run_user_jobs = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_user_jobs)
+            max_num_run_user_cores = self.taskBufferIF.getConfigValue("prio_mgr", gdp_token_user_cores)
+            # get job count per uid
+            job_count_per_uid = self.taskBufferIF.count_jobs_per_uid_JEDI(self.vo, self.prodSourceLabel)
+            users_to_keep_throttled = set()
+            for uid, count_dict in job_count_per_uid.items():
+                to_throttle = True
+                to_release = True
+                msg = None
+                if count_dict["is_user"]:
+                    # user
+                    if max_num_run_user_jobs:
+                        if count_dict["nRunJobs"] > max_num_run_user_jobs * throttle_threshold:
+                            msg = f"""throttle due to too many running jobs {count_dict["nRunJobs"]} > {gdp_token_user_jobs}x{throttle_threshold} type=jobs"""
+                        elif count_dict["nQueuedJobs"] > max_num_run_user_jobs * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued jobs {count_dict["nQueuedJobs"]} > {gdp_token_user_jobs}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None and max_num_run_user_cores:
+                        if count_dict["nRunCores"] > max_num_run_user_cores:
+                            msg = (
+                                f"""throttle due to too many running cores {count_dict["nRunCores"]} > {gdp_token_user_cores}x{throttle_threshold} type=jobs"""
+                            )
+                        elif count_dict["nQueuedCores"] > max_num_run_user_cores * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued cores {count_dict["nQueuedCores"]} > {gdp_token_user_cores}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None:
+                        to_throttle = False
+                    # check if can be released
+                    if max_num_run_user_jobs:
+                        if (
+                            count_dict["nRunJobs"] > max_num_run_user_jobs * release_threshold
+                            or count_dict["nQueuedJobs"] > max_num_run_user_jobs * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                    if max_num_run_user_cores:
+                        if (
+                            count_dict["nRunCores"] > max_num_run_user_cores * release_threshold
+                            or count_dict["nQueuedCores"] > max_num_run_user_cores * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                else:
+                    # group
+                    if max_num_run_group_jobs:
+                        if count_dict["nRunJobs"] > max_num_run_group_jobs:
+                            msg = f"""throttle due to too many running jobs {count_dict["nRunJobs"]} > {gdp_token_group_jobs}x{throttle_threshold} type=jobs"""
+                        elif count_dict["nQueuedJobs"] > max_num_run_group_jobs * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued jobs {count_dict["nQueuedJobs"]} > {gdp_token_group_jobs}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None and max_num_run_group_cores:
+                        if count_dict["nRunCores"] > max_num_run_group_cores:
+                            msg = (
+                                f"""throttle due to too many running cores {count_dict["nRunCores"]} > {gdp_token_group_cores}x{throttle_threshold} type=jobs"""
+                            )
+                        elif count_dict["nQueuedCores"] > max_num_run_group_cores * queue_factor * throttle_threshold:
+                            msg = f"""throttle due to too many queued cores {count_dict["nQueuedCores"]} > {gdp_token_group_cores}x{throttle_threshold}x{queue_factor} type=jobs"""
+                    if msg is None:
+                        to_throttle = False
+                    # check if can be released
+                    if max_num_run_group_jobs:
+                        if (
+                            count_dict["nRunJobs"] > max_num_run_group_jobs * release_threshold
+                            or count_dict["nQueuedJobs"] > max_num_run_group_jobs * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                    if max_num_run_group_cores:
+                        if (
+                            count_dict["nRunCores"] > max_num_run_group_cores * release_threshold
+                            or count_dict["nQueuedCores"] > max_num_run_group_cores * queue_factor * release_threshold
+                        ):
+                            to_release = False
+                if to_throttle:
+                    tmpLog.debug(f"uid={uid} {msg}")
+                    throttled_tasks = self.taskBufferIF.throttle_tasks_with_uid(self.vo, self.prodSourceLabel, thrInterval, msg, uid, count_dict["is_user"])
+                    tmpLog.debug(f"throttled tasks: {throttled_tasks}")
+                if to_throttle or not to_release:
+                    users_to_keep_throttled.add(uid)
+            # unthrottle users which are currently throttled but no more exceed the limit
+            for userName, taskData in thrUserTasks.items():
+                for transferType, taskIDs in taskData.items():
+                    if transferType not in ["jobs"]:
+                        continue
+                    if userName in users_to_keep_throttled:
+                        continue
+                    tmpLog.debug(f"user={userName} release throttled tasks since no more excess of queued and running job counts")
+                    # unthrottle tasks
+                    for taskID in taskIDs:
+                        tmpLog.debug(f"action=release_{transferType} jediTaskID={taskID} for user={userName}")
+                        self.taskBufferIF.release_task_on_hold(taskID, "throttled")
             tmpLog.debug("done")
         except Exception:
             errtype, errvalue = sys.exc_info()[:2]

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jediorder/JobGenerator.py

@@ -1503,6 +1503,8 @@ class JobGeneratorThread(WorkerThread):
                     # on-site merging
                     if taskSpec.on_site_merging():
                         jobSpec.set_on_site_merging()
+                    if inputChunk.isMerging:
+                        jobSpec.set_pmerge_job_flag()
                     # set task queued time
                     jobSpec.set_task_queued_time(task_queued_time)
                     # extract middle name
@@ -1985,6 +1987,7 @@ class JobGeneratorThread(WorkerThread):
             specialHandling = specialHandling[:-1]
             if specialHandling != "":
                 jobSpec.specialHandling = specialHandling
+            jobSpec.set_build_job_flag()
             jobSpec.set_task_queued_time(taskSpec.get_queued_time())
             libDsFileNameBase = libDsName + ".$JEDIFILEID"
             # make lib.tgz

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandajedi/jedirefine/AtlasAnalTaskRefiner.py

@@ -241,11 +241,11 @@ class AtlasAnalTaskRefiner(TaskRefinerBase):
             self.taskSpec.reqID = tmpJobID
             # site limitation
             if "excludedSite" in taskParamMap and "includedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("incexc")
+                self.taskSpec.setLimitedSites("with_allow_and_denylist")
             elif "excludedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("exc")
+                self.taskSpec.setLimitedSites("with_denylist")
             elif "includedSite" in taskParamMap:
-                self.taskSpec.setLimitedSites("inc")
+                self.taskSpec.setLimitedSites("with_allowlist")
             # input prestaging
             if self.taskSpec.inputPreStaging():
                 # set first contents feed flag

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandaserver/api/v1/data_carousel_api.py

@@ -50,7 +50,7 @@ def change_staging_destination(req: PandaRequest, request_id: int | None = None,
     Change destination of staging
 
     The current active staging request will be cancelled, and a new request will be created with the newly selected destination RSE, excluding the original destination.
-    The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
+    The requests can be specified by request_id or dataset (if both exist, request_id is taken).
     Requires a secure connection production role.
 
     API details:
@@ -210,7 +210,7 @@ def force_to_staging(req: PandaRequest, request_id: int | None = None, dataset:
 
     The request will skip the queue and go to staging immediately (will submit DDM rules).
     Only effective on queued requests.
-    The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
+    The requests can be specified by request_id or dataset (if both exist, request_id is taken).
     Requires a secure connection production role.
 
     API details:
@@ -262,3 +262,63 @@ def force_to_staging(req: PandaRequest, request_id: int | None = None, dataset:
     tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
 
     return generate_response(success, message, data)
+
+
+@request_validation(_logger, secure=True, production=True, request_method="POST")
+def retire_unused(req: PandaRequest, request_id: int | None = None, dataset: str | None = None) -> dict:
+    """
+    Retire unused staging request
+
+    If the request is done and has no related tasks, it can be retired to clean up the DDM rules and replicas.
+    The requests can be specified by request_id or dataset (if both exist, request_id is taken).
+    Requires a secure connection production role.
+
+    API details:
+        HTTP Method: POST
+        Path: /v1/data_carousel/retire_unused
+
+    Args:
+        req(PandaRequest): internally generated request object
+        request_id (int|None): request_id of the staging request, e.g. `123`
+        dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
+
+    Returns:
+        dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
+    """
+    tmp_logger = LogWrapper(_logger, f"retire_unused request_id={request_id} dataset={dataset}")
+    tmp_logger.debug("Start")
+    success, message, data = False, "", None
+    time_start = naive_utcnow()
+
+    dc_req_spec = None
+    if request_id is not None:
+        # specified by request_id
+        dc_req_spec = global_dcif.get_request_by_id(request_id)
+    elif request_id is None and dataset is not None:
+        # specified by dataset
+        dc_req_spec = global_dcif.get_request_by_dataset(dataset)
+
+    if dc_req_spec is not None:
+        is_ok, dc_req_spec, err_msg = global_dcif.retire_unused_request(dc_req_spec)
+        if not is_ok:
+            err_msg = f"failed to retire request_id={dc_req_spec.request_id} : {err_msg}"
+            tmp_logger.error(err_msg)
+            success, message = False, err_msg
+        else:
+            success = True
+            message = f"retired successfully"
+            data = {
+                "request_id": dc_req_spec.request_id,
+                "dataset": dc_req_spec.dataset,
+                "status": dc_req_spec.status,
+                "ddm_rule_id": dc_req_spec.ddm_rule_id,
+            }
+    else:
+        err_msg = f"failed to get corresponding request"
+        tmp_logger.error(err_msg)
+        success, message = False, err_msg
+
+    time_delta = naive_utcnow() - time_start
+    tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
+
+    return generate_response(success, message, data)

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandaserver/api/v1/task_api.py

@@ -1417,57 +1417,6 @@ def get_parent_detailed_info(req: PandaRequest, task_id: int) -> Dict[str, Any]:
     return generate_response(True, data=task_info)
 
 
-@request_validation(_logger, secure=True, request_method="GET")
-def get_scout_job_descriptions(req: PandaRequest, task_id: int) -> Dict:
-    """
-    Get scout job descriptions for a task.
-
-    Resolves all PanDA job IDs associated with a JEDI task, retrieves their job
-    descriptions including archive lookup, and returns only scout jobs. Scout
-    jobs are identified by the ``sj`` token in the comma-separated
-    ``specialHandling`` field.
-    Requires a secure connection.
-
-    API details:
-        HTTP Method: GET
-        Path: /v1/task/get_scout_job_descriptions
-
-    Args:
-        req(PandaRequest): internally generated request object containing the env variables.
-        task_id (int): JEDI task ID.
-
-    Returns:
-        dict: The system response `{"success": success, "message": message, "data": data}`.
-              When successful, the data field contains a list of scout job descriptions.
-              If no jobs are found or no scout jobs exist, returns success=True with empty list.
-    """
-    tmp_logger = LogWrapper(_logger, f"get_scout_job_descriptions task_id={task_id}")
-    tmp_logger.debug("Start")
-
-    try:
-        task_id = int(task_id)
-    except Exception:
-        tmp_logger.error("Failed due to invalid task_id")
-        return generate_response(False, message=MESSAGE_TASK_ID)
-
-    job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id, scout_only=True)
-    if not job_ids:
-        tmp_logger.debug("No scout jobs found for task")
-        return generate_response(True, data=[])
-
-    tmp_logger.debug(f"Found {len(job_ids)} scout job IDs for task")
-
-    max_ids = 5500
-    if len(job_ids) > max_ids:
-        tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
-        job_ids = job_ids[:max_ids]
-
-    scout_jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
-
-    tmp_logger.debug(f"Done with {len(scout_jobs)} scout jobs")
-    return generate_response(True, data=scout_jobs)
-
-
 @request_validation(_logger, secure=True, request_method="GET")
 def get_job_descriptions(req: PandaRequest, task_id: int, unsuccessful_only: bool = False) -> Dict:
     """
@@ -1514,7 +1463,7 @@ def get_job_descriptions(req: PandaRequest, task_id: int, unsuccessful_only: boo
         tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
         job_ids = job_ids[:max_ids]
 
-    jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
+    jobs = [job.to_dict_advanced(add_extra_info=True) for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
 
     tmp_logger.debug(f"Done with {len(jobs)} jobs")
     return generate_response(True, data=jobs)

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandaserver/api/v1/tests/task_api_tests.py

@@ -542,26 +542,6 @@ class TestTaskAPI(unittest.TestCase):
         self.assertEqual(output, expected_response)
 
 
-    def test_get_scout_job_descriptions(self):
-        # def get_scout_job_descriptions(req: PandaRequest, task_id: int) -> Dict:
-        url = f"{api_url_ssl}/task/get_scout_job_descriptions"
-        print(f"Testing URL: {url}")
-        data = {"task_id": JEDI_TASK_ID}
-        status, output = self.http_client.get(url, data)
-        print(output)
-        output["status"] = status
-
-        if JEDI_TASK_ID == -1:
-            # Fake task has no scout jobs, returns empty list
-            output["message"] = ""
-            expected_response = {"status": 0, "success": True, "data": [], "message": ""}
-            self.assertEqual(output, expected_response)
-        else:
-            # Real task: success with a list of scout job descriptions
-            self.assertEqual(status, 0)
-            self.assertTrue(output["success"])
-            self.assertIsInstance(output["data"], list)
-
     def test_get_job_descriptions(self):
         # def get_job_descriptions(req: PandaRequest, task_id: int, unsuccessful_only: bool = False) -> Dict:
         url = f"{api_url_ssl}/task/get_job_descriptions"

{panda_server_source-0.8.0 → panda_server_source-0.8.1}/pandaserver/brokerage/SiteMapper.py

@@ -285,12 +285,8 @@ class SiteMapper:
         if cloud == WORLD_CLOUD:
             return self.worldCloudSpec
 
-        # return some default sites
-        default_cloud = {
-            "source": "default",
-            "sites": self.cloudSpec["US"]["sites"],
-        }
-        return default_cloud
+        # return all sites (WORLD) as fallback for unknown clouds
+        return self.worldCloudSpec
 
     # accessor for cloud
     def checkCloud(self, cloud):