panda-server-source 1.0.2__tar.gz → 1.0.4__tar.gz

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/Dockerfile.base

@@ -1,6 +1,6 @@
 # Base Platform Image for PanDA services
 #
-# NOTE: Modify the image tag in ./github/workflows/base-platform-docker-image.yml when OS or Python version is changed
+# NOTE: Modify the image tag in .github/workflows/base-platform-docker-publish.yml when OS or Python version is changed
 
 ARG PYTHON_VERSION=3.11.6
 

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: panda-server-source
-Version: 1.0.2
+Version: 1.0.4
 Summary: Unified PanDA Server and JEDI Package - Source Only
 Project-URL: Homepage, https://panda-wms.readthedocs.io/en/latest/
 Author-email: PanDA Team <panda-support@cern.ch>

panda_server_source-1.0.4/PandaPkgInfo.py

@@ -0,0 +1 @@
+release_version = "1.0.4"

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/pandajedi/jedibrokerage/AtlasAnalJobBroker.py

@@ -23,6 +23,7 @@ logger = PandaLogger().getLogger(__name__.split(".")[-1])
 
 APP = "jedi"
 COMPONENT = "jobbroker"
+ANALYSIS_COMPONENT = "anal_jobbroker"
 VO = "atlas"
 
 
@@ -66,6 +67,9 @@ class AtlasAnalJobBroker(JobBrokerBase):
         # return for failure
         retFatal = self.SC_FATAL, inputChunk
         retTmpError = self.SC_FAILED, inputChunk
+        # return when the task is only waiting for something it cannot control, so that
+        # it is not penalized with the pending timeout while it waits
+        retWaiting = self.SC_WAITING, inputChunk
         # new maxwdir
         newMaxwdir = {}
         # get primary site candidates
@@ -150,12 +154,12 @@ class AtlasAnalJobBroker(JobBrokerBase):
             # not use MCORE
             useMP = "unuse"
         # get statistics of failures
-        timeWindowForFC = self.taskBufferIF.getConfigValue("anal_jobbroker", "TW_DONE_JOB_STAT", "jedi", taskSpec.vo)
+        timeWindowForFC = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "TW_DONE_JOB_STAT", "jedi", taskSpec.vo)
         if timeWindowForFC is None:
             timeWindowForFC = 6
 
         # get minimum bad jobs to skip PQ
-        minBadJobsToSkipPQ = self.taskBufferIF.getConfigValue("anal_jobbroker", "MIN_BAD_JOBS_TO_SKIP_PQ", "jedi", taskSpec.vo)
+        minBadJobsToSkipPQ = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "MIN_BAD_JOBS_TO_SKIP_PQ", "jedi", taskSpec.vo)
         if minBadJobsToSkipPQ is None:
             minBadJobsToSkipPQ = 5
 
@@ -183,20 +187,20 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 if totalJobStat["nRunJobs"] > maxNumRunJobs:
                     tmpLog.error(f"throttle to generate jobs due to too many running jobs {totalJobStat['nRunJobs']} > {gdp_token_jobs}")
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
                 elif totalJobStat["nQueuedJobs"] > maxFactor * maxNumRunJobs:
                     tmpLog.error(f"throttle to generate jobs due to too many queued jobs {totalJobStat['nQueuedJobs']} > {maxFactor}x{gdp_token_jobs}")
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
             if maxNumRunCores:
                 if totalJobStat["nRunCores"] > maxNumRunCores:
                     tmpLog.error(f"throttle to generate jobs due to too many running cores {totalJobStat['nRunCores']} > {gdp_token_cores}")
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
                 elif totalJobStat["nQueuedCores"] > maxFactor * maxNumRunCores:
                     tmpLog.error(f"throttle to generate jobs due to too many queued cores {totalJobStat['nQueuedCores']} > {maxFactor}x{gdp_token_cores}")
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
 
         # get gshare usage
         ret_val, gshare_usage_dict = AtlasBrokerUtils.getGShareUsage(tbIF=self.taskBufferIF, gshare=taskSpec.gshare)
@@ -250,38 +254,40 @@ class AtlasAnalJobBroker(JobBrokerBase):
         user_analyis_throttle_intensity_A = 1.0
 
         # parameters about Analysis Stabilizer
-        base_queue_length_per_pq = self.taskBufferIF.getConfigValue("anal_jobbroker", "BASE_QUEUE_LENGTH_PER_PQ", "jedi", taskSpec.vo)
+        base_queue_length_per_pq = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "BASE_QUEUE_LENGTH_PER_PQ", "jedi", taskSpec.vo)
         if base_queue_length_per_pq is None:
             base_queue_length_per_pq = 100
-        base_expected_wait_hour_on_pq = self.taskBufferIF.getConfigValue("anal_jobbroker", "BASE_EXPECTED_WAIT_HOUR_ON_PQ", "jedi", taskSpec.vo)
+        base_expected_wait_hour_on_pq = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "BASE_EXPECTED_WAIT_HOUR_ON_PQ", "jedi", taskSpec.vo)
         if base_expected_wait_hour_on_pq is None:
             base_expected_wait_hour_on_pq = 8
-        base_default_queue_length_per_pq_user = self.taskBufferIF.getConfigValue("anal_jobbroker", "BASE_DEFAULT_QUEUE_LENGTH_PER_PQ_USER", "jedi", taskSpec.vo)
+        base_default_queue_length_per_pq_user = self.taskBufferIF.getConfigValue(
+            ANALYSIS_COMPONENT, "BASE_DEFAULT_QUEUE_LENGTH_PER_PQ_USER", "jedi", taskSpec.vo
+        )
         if base_default_queue_length_per_pq_user is None:
             base_default_queue_length_per_pq_user = 5
-        base_queue_ratio_on_pq = self.taskBufferIF.getConfigValue("anal_jobbroker", "BASE_QUEUE_RATIO_ON_PQ", "jedi", taskSpec.vo)
+        base_queue_ratio_on_pq = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "BASE_QUEUE_RATIO_ON_PQ", "jedi", taskSpec.vo)
         if base_queue_ratio_on_pq is None:
             base_queue_ratio_on_pq = 0.05
-        static_max_queue_running_ratio = self.taskBufferIF.getConfigValue("anal_jobbroker", "STATIC_MAX_QUEUE_RUNNING_RATIO", "jedi", taskSpec.vo)
+        static_max_queue_running_ratio = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "STATIC_MAX_QUEUE_RUNNING_RATIO", "jedi", taskSpec.vo)
         if static_max_queue_running_ratio is None:
             static_max_queue_running_ratio = 2.0
-        max_expected_wait_hour = self.taskBufferIF.getConfigValue("anal_jobbroker", "MAX_EXPECTED_WAIT_HOUR", "jedi", taskSpec.vo)
+        max_expected_wait_hour = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "MAX_EXPECTED_WAIT_HOUR", "jedi", taskSpec.vo)
         if max_expected_wait_hour is None:
             max_expected_wait_hour = 12.0
 
-        max_missing_input_files = self.taskBufferIF.getConfigValue("jobbroker", "MAX_MISSING_INPUT_FILES", "jedi", taskSpec.vo)
+        max_missing_input_files = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "MAX_MISSING_INPUT_FILES", "jedi", taskSpec.vo)
         if max_missing_input_files is None:
             max_missing_input_files = 10
-        min_input_completeness = self.taskBufferIF.getConfigValue("jobbroker", "MIN_INPUT_COMPLETENESS", "jedi", taskSpec.vo)
+        min_input_completeness = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "MIN_INPUT_COMPLETENESS", "jedi", taskSpec.vo)
         if min_input_completeness is None:
             min_input_completeness = 90
 
         # minimum brokerage weight
         min_weight_param = f"MIN_WEIGHT_{taskSpec.prodSourceLabel}_{taskSpec.gshare}"
-        min_weight = self.taskBufferIF.getConfigValue("jobbroker", min_weight_param, "jedi", taskSpec.vo)
+        min_weight = self.taskBufferIF.getConfigValue(COMPONENT, min_weight_param, "jedi", taskSpec.vo)
         if min_weight is None:
             min_weight_param = f"MIN_WEIGHT_{taskSpec.prodSourceLabel}"
-            min_weight = self.taskBufferIF.getConfigValue("jobbroker", min_weight_param, "jedi", taskSpec.vo)
+            min_weight = self.taskBufferIF.getConfigValue(COMPONENT, min_weight_param, "jedi", taskSpec.vo)
         if min_weight is None:
             min_weight = 0
 
@@ -304,7 +310,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                         )
                     )
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
                 elif task_class_value == 0 and usage_percent > user_analyis_to_throttle_threshold_perc_B:
                     # B-tasks to throttle
                     tmpLog.error(
@@ -313,7 +319,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                         )
                     )
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
                 elif (
                     task_class_value == 1 and usage_percent * usage_slot_ratio_A * user_analyis_throttle_intensity_A > user_analyis_to_throttle_threshold_perc_A
                 ):
@@ -324,7 +330,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                         )
                     )
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
-                    return retTmpError
+                    return retWaiting
             except Exception as e:
                 tmpLog.error(f"got error when checking low-ranked tasks to throttle; skipped : {e}")
 
@@ -362,11 +368,11 @@ class AtlasAnalJobBroker(JobBrokerBase):
 
         # IO intensity cutoff in kB/sec to allow input transfers
         io_intensity_key = "IO_INTENSITY_CUTOFF_USER"
-        io_intensity_cutoff = self.taskBufferIF.getConfigValue("anal_jobbroker", io_intensity_key, "jedi", taskSpec.vo)
+        io_intensity_cutoff = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, io_intensity_key, "jedi", taskSpec.vo)
 
         # timelimit for data locality check
         loc_check_timeout_key = "DATA_CHECK_TIMEOUT_USER"
-        loc_check_timeout_val = self.taskBufferIF.getConfigValue("anal_jobbroker", loc_check_timeout_key, "jedi", taskSpec.vo)
+        loc_check_timeout_val = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, loc_check_timeout_key, "jedi", taskSpec.vo)
 
         # check input datasets
         element_map = dict()
@@ -1175,8 +1181,17 @@ class AtlasAnalJobBroker(JobBrokerBase):
             newScanSiteList = []
             oldScanSiteList = copy.copy(scanSiteList)
             msg_map = {}
+            # free space
+            diskThreshold_in = self.taskBufferIF.getConfigValue(COMPONENT, "STORAGE_MIN_FREE_SIZE", "jedi", taskSpec.vo)
+            if not diskThreshold_in:
+                # default to 200GB if not set in config
+                diskThreshold_in = 200
+            diskThreshold_out = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "STORAGE_MIN_FREE_SIZE_OUTPUT", "jedi", taskSpec.vo)
+            if not diskThreshold_out:
+                # default to 200GB if not set in config
+                diskThreshold_out = 200
             for tmpSiteName in self.get_unified_sites(scanSiteList):
-                # check endpoint
+                # check output endpoint
                 tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
                 scope_input, scope_output = select_scope(tmpSiteSpec, JobUtils.ANALY_PS, JobUtils.ANALY_PS)
                 if scope_output not in tmpSiteSpec.ddm_endpoints_output:
@@ -1184,18 +1199,27 @@ class AtlasAnalJobBroker(JobBrokerBase):
                     continue
                 tmp_output_endpoint = tmpSiteSpec.ddm_endpoints_output[scope_output].getEndPoint(tmpSiteSpec.ddm_output[scope_output])
                 if tmp_output_endpoint is not None:
-                    # free space must be >= 200GB
-                    diskThreshold = 200
-                    tmpSpaceSize = 0
-                    if tmp_output_endpoint["space_expired"] is not None:
-                        tmpSpaceSize += tmp_output_endpoint["space_expired"]
-                    if tmp_output_endpoint["space_free"] is not None:
-                        tmpSpaceSize += tmp_output_endpoint["space_free"]
-                    if (
-                        tmpSpaceSize < diskThreshold and "skip_RSE_check" not in tmpSiteSpec.catchall
-                    ):  # skip_RSE_check: exceptional bypass of RSEs without storage reporting
-                        msg_map[tmpSiteName] = f"  skip site={tmpSiteName} due to disk shortage in SE {tmpSpaceSize} < {diskThreshold}GB criteria=-disk"
+                    tmp_space_size = tmp_output_endpoint["space_usable"]
+                    if tmp_space_size < diskThreshold_out:
+                        msg_map[tmpSiteName] = (
+                            f"  skip site={tmpSiteName} due to output disk shortage in SE {tmp_space_size} < {diskThreshold_out}GB criteria=-disk"
+                        )
                         continue
+                # check input endpoint to transfer input when data locality is not required
+                if not checkDataLocality:
+                    if scope_input not in tmpSiteSpec.ddm_endpoints_input:
+                        msg_map[tmpSiteName] = f"  skip site={tmpSiteName} since {scope_input} input endpoint undefined criteria=-disk"
+                        continue
+                    # check only when input and output endpoints are different
+                    if tmpSiteSpec.ddm_output[scope_output] != tmpSiteSpec.ddm_input[scope_input]:
+                        tmp_input_endpoint = tmpSiteSpec.ddm_endpoints_input[scope_input].getEndPoint(tmpSiteSpec.ddm_input[scope_input])
+                        if tmp_input_endpoint is not None:
+                            tmp_space_size = tmp_input_endpoint["space_usable"]
+                            if tmp_space_size < diskThreshold_in:
+                                msg_map[tmpSiteName] = (
+                                    f"  skip site={tmpSiteName} due to input disk shortage in SE {tmp_space_size} < {diskThreshold_in}GB criteria=-disk"
+                                )
+                                continue
                 # check if blacklisted
                 tmp_msg = AtlasBrokerUtils.check_endpoints_with_blacklist(tmpSiteSpec, scope_input, scope_output, sites_in_nucleus, remote_source_available)
                 if tmp_msg is not None:
@@ -1332,7 +1356,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                     if siteMaxTime != 0 and siteMaxTime < minTimeForZeroWalltime:
                         tmpMsg = f"  skip site={tmpSiteName} due to site walltime {tmpSiteStr} (site upper limit) insufficient "
                         if inputChunk.useScout():
-                            tmpMsg += f"for scout jobs (requireing {str_minTimeForZeroWalltime} at least) "
+                            tmpMsg += f"for scout jobs (requiring {str_minTimeForZeroWalltime} at least) "
                             tmpMsg += "criteria=-scout_walltime"
                         else:
                             tmpMsg += f"for merge or walltime-undefined jobs (requiring {str_minTimeForZeroWalltime} at least) "
@@ -1525,10 +1549,10 @@ class AtlasAnalJobBroker(JobBrokerBase):
                 msg_map = {}
                 msgList = []
                 msgListVP = []
-                minQueue = self.taskBufferIF.getConfigValue("anal_jobbroker", "OVERLOAD_MIN_QUEUE", "jedi", taskSpec.vo)
+                minQueue = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "OVERLOAD_MIN_QUEUE", "jedi", taskSpec.vo)
                 if minQueue is None:
                     minQueue = 20
-                ratioOffset = self.taskBufferIF.getConfigValue("anal_jobbroker", "OVERLOAD_RATIO_OFFSET", "jedi", taskSpec.vo)
+                ratioOffset = self.taskBufferIF.getConfigValue(ANALYSIS_COMPONENT, "OVERLOAD_RATIO_OFFSET", "jedi", taskSpec.vo)
                 if ratioOffset is None:
                     ratioOffset = 1.2
                 grandRatio = AtlasBrokerUtils.get_total_nq_nr_ratio(jobStatPrioMap, taskSpec.gshare)
@@ -1617,7 +1641,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                         continue
                     allowed_queue_length_from_wait_time = base_expected_wait_hour_on_pq * to_running_rate
                     if nQueue_pq_in_gshare < allowed_queue_length_from_wait_time:
-                        # not statisfy since overall waiting time of the site is not long enough
+                        # not satisfy since overall waiting time of the site is not long enough
                         tmpLog.debug(
                             "not throttle on {0} since nQ({1}) < {2:.3f} = toRunningRate({3:.3f} /hr) * base wait time ({4} hr)".format(
                                 tmpSiteName, nQueue_pq_in_gshare, allowed_queue_length_from_wait_time, to_running_rate, base_expected_wait_hour_on_pq
@@ -2055,7 +2079,7 @@ class AtlasAnalJobBroker(JobBrokerBase):
                     nbw_main = n_jobs_to_submit
                     if taskSpec.gshare in ["User Analysis", "Express Analysis"]:
                         nbw_main = main_weight_site_class_dict[bw_map["class"]]
-                    # secondary weight proportional to remaing queue length
+                    # secondary weight proportional to remaining queue length
                     nbw_sec = 1
                     if taskSpec.gshare in ["User Analysis", "Express Analysis"]:
                         _nbw_numer = max(bw_map["rem_q_len"] - nbw_main / site_class_n_site_dict[bw_map["class"]], nbw_main * 0.001)

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/pandajedi/jedibrokerage/AtlasBrokerUtils.py

@@ -8,7 +8,6 @@ import time
 import traceback
 from typing import Any
 
-from packaging import version
 from pandacommon.pandautils.PandaUtils import naive_utcnow
 
 from pandajedi.jedicore import Interaction
@@ -16,6 +15,10 @@ from pandajedi.jediddm.DDMInterface import DDMInterface
 from pandaserver.brokerage.SiteMapper import SiteMapper
 from pandaserver.dataservice import DataServiceUtils
 from pandaserver.dataservice.DataServiceUtils import select_scope
+from pandaserver.srvcore.hardware_matching import (
+    compare_version_string,
+    match_gpu_spec,
+)
 from pandaserver.taskbuffer import JobUtils, ProcessGroups, SiteSpec
 from pandaserver.taskbuffer.DdmSpec import DOWNTIME_STATUSES
 
@@ -916,49 +919,6 @@ def getAnalySitesClass(tbIF, fresher_than_minutes_ago=60):
     return ret_val, ret_map
 
 
-def compare_version_string(version_string, comparison_string):
-    """
-    Compares a version string with another string composed of a comparison operator and a version string.
-
-    Args:
-        version_string (str): The version string to compare.
-        comparison_string (str): The string containing the comparison operator and version string (e.g., ">=2.0").
-
-    Returns:
-        bool or None: True if the version string satisfies the comparison, False if it doesn't,
-                       or None if the comparison string is invalid.
-    """
-    match = re.match(r"([=><]+)(.+)", comparison_string)
-    if not match:
-        return None
-
-    operator = match.group(1).strip()
-    if operator == "=":
-        operator = "=="
-    version_to_compare = match.group(2).strip()
-
-    try:
-        version1 = version.parse(version_string)
-        version2 = version.parse(version_to_compare)
-    except version.InvalidVersion:
-        return None
-
-    if operator == "==":
-        return version1 == version2
-    elif operator == "!=":
-        return version1 != version2
-    elif operator == ">=":
-        return version1 >= version2
-    elif operator == "<=":
-        return version1 <= version2
-    elif operator == ">":
-        return version1 > version2
-    elif operator == "<":
-        return version1 < version2
-    else:
-        return None
-
-
 # check SW with json
 class JsonSoftwareCheck:
     # constructor
@@ -1090,54 +1050,12 @@ class JsonSoftwareCheck:
                                 continue
 
                             # All attribute checks use WN GPU monitoring (MV_WORKER_NODE_GPU_SUMMARY)
-                            # which has richer per-host data (vram, architecture, driver version)
-                            wn_gpus = self.wn_gpu_map.get(tmp_site_name, [])
-
-                            # check vendor
-                            if host_gpu_spec["vendor"] != "*":
-                                if not wn_gpus or not any(g.get("vendor") and re.match(host_gpu_spec["vendor"], g["vendor"], re.IGNORECASE) for g in wn_gpus):
-                                    continue
-
-                            # check model (include or exclude pattern)
-                            if host_gpu_spec["model"] != "*":
-                                if isinstance(host_gpu_spec["model"], dict):
-                                    model_pattern = host_gpu_spec["model"]["pattern"]
-                                    model_excl = host_gpu_spec["model"].get("excl", False)
-                                else:
-                                    model_pattern = host_gpu_spec["model"]
-                                    model_excl = False
-                                if not wn_gpus:
-                                    continue
-                                matches = any(g.get("model") and re.match(model_pattern, g["model"], re.IGNORECASE) for g in wn_gpus)
-                                if matches == model_excl:
-                                    continue
-
-                            # check VRAM (in MB); supports operators: ==, >=, <=, >, <, != (e.g. ">=40960")
-                            if "vram" in host_gpu_spec:
-                                if not wn_gpus or not any(g.get("vram") and compare_version_string(str(g["vram"]), host_gpu_spec["vram"]) for g in wn_gpus):
-                                    continue
-
-                            # check GPU microarchitecture generation (e.g. Ampere, Hopper, Ada Lovelace)
-                            if "microarchitecture" in host_gpu_spec:
-                                req_arch = host_gpu_spec["microarchitecture"]
-                                if isinstance(req_arch, str):
-                                    req_arch = [req_arch]
-                                if not wn_gpus or not any(g.get("architecture") in req_arch for g in wn_gpus):
-                                    continue
-
-                            # check minimum CUDA version
-                            if "version" in host_gpu_spec:
-                                if not wn_gpus or not any(
-                                    g.get("framework_version") and compare_version_string(g["framework_version"], host_gpu_spec["version"]) for g in wn_gpus
-                                ):
-                                    continue
-
-                            # check minimum GPU driver version (kernel driver, e.g. 575.57.08)
-                            if "driver_version" in host_gpu_spec:
-                                if not wn_gpus or not any(
-                                    g.get("driver_version") and compare_version_string(g["driver_version"], host_gpu_spec["driver_version"]) for g in wn_gpus
-                                ):
-                                    continue
+                            # which has richer per-host data (vram, architecture, driver version).
+                            # The minimum-requirement attributes are checked against all GPU entries of
+                            # the queue, to prevent brokering to mixed sites where some nodes fall below
+                            # the requirement
+                            if not match_gpu_spec(host_gpu_spec, self.wn_gpu_map.get(tmp_site_name, [])):
+                                continue
                     go_ahead = True
                 except Exception as e:
                     if log_stream:
@@ -1209,6 +1127,52 @@ class JsonSoftwareCheck:
         return ok_sites, no_auto_sites, preference_weight_map
 
 
+# pattern to extract the architecture alternation of a cmt_config, e.g. (x86_64|aarch64)-el9-gcc15-opt
+ARCH_ALTERNATION_IN_CMT_CONFIG = re.compile(r"^\(([A-Za-z0-9_.+-]+(?:\|[A-Za-z0-9_.+-]+)*)\)([A-Za-z0-9_.+-]+)$")
+
+
+# get CPU architectures of a queue
+def get_queue_cpu_architectures(queue_name: str, sw_map: dict) -> list:
+    """
+    get CPU architectures of a queue
+    :param queue_name: queue name
+    :param sw_map: software map
+    :return: list of CPU architectures. empty if unavailable
+    """
+    arch_list = []
+    for arch_spec in sw_map.get(queue_name, {}).get("architectures", []):
+        if arch_spec.get("type") == "cpu":
+            arch_list += arch_spec.get("arch", [])
+    return arch_list
+
+
+# resolve the architecture of cmt_config
+def resolve_arch_in_cmt_config(queue_name: str, cmt_config: str, sw_map: dict) -> str | None:
+    """
+    resolve the architecture alternation of a cmt_config with the queue's CPU architectures,
+    e.g. (x86_64|aarch64)-el9-gcc15-opt to x86_64-el9-gcc15-opt at an x86_64 queue.
+    Only a leading alternation followed by a literal remainder is supported
+    :param queue_name: queue name
+    :param cmt_config: cmt config to resolve
+    :param sw_map: software map
+    :return: resolved cmt config or None if unresolvable
+    """
+    match = ARCH_ALTERNATION_IN_CMT_CONFIG.search(cmt_config)
+    if not match:
+        return None
+    # return None if the queue doesn't describe CPU architectures
+    queue_arch_list = get_queue_cpu_architectures(queue_name, sw_map)
+    if not queue_arch_list:
+        return None
+    # use the first architecture supported by the queue
+    remainder = match.group(2)
+    for arch in match.group(1).split("|"):
+        if "any" in queue_arch_list or arch in queue_arch_list:
+            return arch + remainder
+    # return None if no architecture is supported by the queue
+    return None
+
+
 # resolve cmt_config
 def resolve_cmt_config(queue_name: str, cmt_config: str, base_platform, sw_map: dict) -> str | None:
     """
@@ -1222,22 +1186,35 @@ def resolve_cmt_config(queue_name: str, cmt_config: str, base_platform, sw_map:
     # return None if queue_name is unavailable
     if queue_name not in sw_map:
         return None
+    cmt_config_list = sw_map[queue_name].get("cmtconfigs", [])
     # return None if cmt_config is valid
-    if cmt_config in sw_map[queue_name]["cmtconfigs"]:
+    if cmt_config in cmt_config_list:
         return None
+    resolved_cmt_config = None
     # check if cmt_config matches with any of the queue's cmt_configs
-    for tmp_cmt_config in sw_map[queue_name]["cmtconfigs"]:
-        if re.search("^" + cmt_config + "$", tmp_cmt_config):
-            if base_platform:
-                # add base_platform if necessary
-                tmp_cmt_config = tmp_cmt_config + "@" + base_platform
-            return tmp_cmt_config
+    try:
+        for tmp_cmt_config in cmt_config_list:
+            if re.search("^" + cmt_config + "$", tmp_cmt_config):
+                resolved_cmt_config = tmp_cmt_config
+                break
+    except re.error:
+        # ignore malformed cmt_config
+        return None
+    # resolve the architecture if the queue has no matching cmt_config
+    if resolved_cmt_config is None:
+        resolved_cmt_config = resolve_arch_in_cmt_config(queue_name, cmt_config, sw_map)
     # return None if cmt_config is unavailable
-    return None
+    if resolved_cmt_config is None:
+        return None
+    if base_platform:
+        # add base_platform if necessary
+        resolved_cmt_config = resolved_cmt_config + "@" + base_platform
+    return resolved_cmt_config
 
 
 def check_endpoints_with_blacklist(
-    site_spec: SiteSpec.SiteSpec, scope_input: str, scope_output: str, sites_in_nucleus: list, remote_source_available: bool
+    site_spec: SiteSpec.SiteSpec, scope_input: str, scope_output: str, sites_in_nucleus: list, remote_source_available: bool,
+    storage_type: str = None, complete_replica_locations: set = None
 ) -> str | None:
     """
     Check if site's endpoints are in the blacklist
@@ -1247,6 +1224,8 @@ def check_endpoints_with_blacklist(
     :param scope_output: output scope
     :param sites_in_nucleus: list of sites in nucleus
     :param remote_source_available: if remote source is available
+    :param storage_type: type of storage
+    :param complete_replica_locations: set of locations with complete replicas for all datasets
 
     :return: description of blacklisted reason or None
     """
@@ -1258,22 +1237,26 @@ def check_endpoints_with_blacklist(
     tmp_site_name = site_spec.sitename
     if scope_input in site_spec.ddm_endpoints_input:
         for tmp_input_endpoint in site_spec.ddm_endpoints_input[scope_input].all.values():
+            if storage_type and tmp_input_endpoint["type"] != storage_type:
+                continue
             tmp_read_input_over_lan = tmp_input_endpoint["detailed_status"].get("read_lan")
             tmp_receive_input_over_wan = tmp_input_endpoint["detailed_status"].get("write_wan")
             # can read input from local
             if tmp_read_input_over_lan not in DOWNTIME_STATUSES:
                 read_input_over_lan = True
             # can receive input from remote to local
-            if tmp_site_name not in sites_in_nucleus:
+            if tmp_site_name not in sites_in_nucleus or (complete_replica_locations and tmp_input_endpoint["ddm_endpoint_name"] not in complete_replica_locations):
                 # satellite sites
                 if tmp_receive_input_over_wan not in DOWNTIME_STATUSES:
                     receive_input_over_wan = True
             else:
-                # NA for nucleus sites
+                # NA for nucleus sites when input datasets are already available
                 receive_input_over_wan = True
                 remote_source_available = True
     if scope_output in site_spec.ddm_endpoints_output:
         for tmp_output_endpoint in site_spec.ddm_endpoints_output[scope_output].all.values():
+            if storage_type and tmp_output_endpoint["type"] != storage_type:
+                continue
             tmp_write_output_over_lan = tmp_output_endpoint["detailed_status"].get("write_lan")
             tmp_send_output_over_wan = tmp_output_endpoint["detailed_status"].get("read_wan")
             # can write output to local

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/pandajedi/jedibrokerage/AtlasProdJobBroker.py

@@ -316,7 +316,8 @@ class AtlasProdJobBroker(JobBrokerBase):
         ######################################
         # check dataset completeness
         remote_source_available = True
-        if inputChunk.getDatasets() and not taskSpec.inputPreStaging():
+        complete_replica_locations = None
+        if inputChunk.getDatasets():
             for datasetSpec in inputChunk.getDatasets():
                 datasetName = datasetSpec.datasetName
                 # skip distributed datasets
@@ -347,6 +348,14 @@ class AtlasProdJobBroker(JobBrokerBase):
                     tmpLog.error(f"failed to get available storage endpoints with {datasetName}")
                     taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
                     return retTmpError
+                # get the list of locations with complete replicas for all datasets
+                if complete_replica_locations is None:
+                    complete_replica_locations = set(tmp_list_of_complete_replica_locations)
+                else:
+                    complete_replica_locations = complete_replica_locations.intersection(set(tmp_list_of_complete_replica_locations))
+                if taskSpec.inputPreStaging():
+                    tmpLog.debug(f"completeness check disabled for {datasetName} since it is being pre-staged")
+                    continue
                 # pending if the dataset is incomplete or missing at online endpoints
                 if not tmp_complete_disk_ok and not tmp_complete_tape_ok:
                     err_msg = f"{datasetName} is "
@@ -984,6 +993,10 @@ class AtlasProdJobBroker(JobBrokerBase):
         msg_map = {}
         tmpLog.set_message_slot()
         newSkippedTmp = dict()
+        diskThreshold = self.taskBufferIF.getConfigValue(COMPONENT, "STORAGE_MIN_FREE_SIZE", "jedi", taskSpec.vo)
+        if not diskThreshold:
+            # set default threshold to 200GB if not configured
+            diskThreshold = 200
         for tmpSiteName in self.get_unified_sites(scanSiteList):
             tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
             scope_input, scope_output = select_scope(tmpSiteSpec, JobUtils.PROD_PS, JobUtils.PROD_PS)
@@ -992,22 +1005,17 @@ class AtlasProdJobBroker(JobBrokerBase):
             tmp_msg = None
             # check free size on output endpoint
             if tmp_default_output_endpoint is not None:
-                tmpSpaceSize = 0
-                if tmp_default_output_endpoint["space_free"] is not None:
-                    tmpSpaceSize += tmp_default_output_endpoint["space_free"]
-                if tmp_default_output_endpoint["space_expired"] is not None:
-                    tmpSpaceSize += tmp_default_output_endpoint["space_expired"]
-                diskThreshold = 200
-
-                # skip_RSE_check: exceptional bypass of RSEs without storage reporting
-                if tmpSpaceSize < diskThreshold and "skip_RSE_check" not in tmpSiteSpec.catchall:
+                tmpSpaceSize = tmp_default_output_endpoint["space_usable"]
+                if tmpSpaceSize < diskThreshold:
                     tmp_msg = (
                         f"  skip site={tmpSiteName} due to disk shortage at "
                         f"{tmpSiteSpec.ddm_output[scope_output]} {tmpSpaceSize}GB < {diskThreshold}GB criteria=-disk"
                     )
             # check if blacklisted
             if not tmp_msg:
-                tmp_msg = AtlasBrokerUtils.check_endpoints_with_blacklist(tmpSiteSpec, scope_input, scope_output, sites_in_nucleus, remote_source_available)
+                tmp_msg = AtlasBrokerUtils.check_endpoints_with_blacklist(
+                    tmpSiteSpec, scope_input, scope_output, sites_in_nucleus, remote_source_available, "DATADISK", complete_replica_locations
+                )
             if tmp_msg is not None:
                 newSkippedTmp[tmpSiteName] = tmp_msg
                 msg_map[tmpSiteName] = tmp_msg
@@ -1134,7 +1142,7 @@ class AtlasProdJobBroker(JobBrokerBase):
                 if siteMaxTime != 0 and siteMaxTime < minTimeForZeroWalltime:
                     tmp_msg = f"  skip site={tmpSiteName} due to site walltime {tmpSiteStr} (site upper limit) insufficient "
                     if inputChunk.useScout():
-                        tmp_msg += f"for scout jobs (requireing {str_minTimeForZeroWalltime} at least) "
+                        tmp_msg += f"for scout jobs (requiring {str_minTimeForZeroWalltime} at least) "
                         tmp_msg += "criteria=-scout_walltime"
                     else:
                         tmp_msg += f"for walltime-undefined jobs (requiring {str_minTimeForZeroWalltime} at least) "
@@ -1789,8 +1797,11 @@ class AtlasProdJobBroker(JobBrokerBase):
                 tmpLog.debug(f"using nStarting+nRunning at {tmpPseudoSiteName} to set nRunning={nRunning} due to numSlot={numStandby}")
             else:
                 # the number of standby jobs is defined
+                old_nRunning = nRunning
                 nRunning = max(int(numStandby / tmpSiteSpec.coreCount), nRunning)
-                tmpLog.debug(f"using numSlots={numStandby}/coreCount at {tmpPseudoSiteName} to set nRunning={nRunning}")
+                tmpLog.debug(
+                    f"using max(int(numStandby={numStandby}/coreCount={tmpSiteSpec.coreCount}), nRunning={old_nRunning}) at {tmpPseudoSiteName} to set nRunning={nRunning}"
+                )
             manyAssigned = float(nAssigned + 1) / float(nActivated + 1)
             manyAssigned = min(2.0, manyAssigned)
             manyAssigned = max(1.0, manyAssigned)

{panda_server_source-1.0.2 → panda_server_source-1.0.4}/pandajedi/jedibrokerage/AtlasProdTaskBroker.py

@@ -94,7 +94,7 @@ class AtlasProdTaskBroker(TaskBrokerBase):
 class AtlasProdTaskBrokerThread(WorkerThread):
     # constructor
     def __init__(self, inputList, threadPool, taskbufferIF, ddmIF, fullRW, prioRW, workQueue):
-        # initialize woker with no semaphore
+        # initialize worker with no semaphore
         WorkerThread.__init__(self, None, threadPool, logger)
         # attributres
         self.inputList = inputList
@@ -360,18 +360,14 @@ class AtlasProdTaskBrokerThread(WorkerThread):
                                     to_skip = True
                                     break
                                 # check space
-                                tmpSpaceSize = 0
-                                if tmpEP["space_free"]:
-                                    tmpSpaceSize += tmpEP["space_free"]
-                                if tmpEP["space_expired"]:
-                                    tmpSpaceSize += tmpEP["space_expired"]
+                                tmpSpaceSize = tmpEP["space_usable"]
                                 tmpSpaceToUse = 0
                                 if tmpNucleus in self.fullRW:
                                     # 0.25GB per cpuTime/corePower/day
                                     tmpSpaceToUse = int(self.fullRW[tmpNucleus] / 10 / 24 / 3600 * 0.25)
                                 if tmpSpaceSize - tmpSpaceToUse < diskThreshold:
                                     tmpLog.info(
-                                        "  skip nucleus={0} since disk shortage (free {1} TB - reserved {2} TB < thr {3} TB) at endpoint {4} criteria=-space".format(
+                                        "  skip nucleus={0} since disk shortage (usable {1} TB - projected demand {2} TB < thr {3} TB) at endpoint {4} criteria=-space".format(
                                             tmpNucleus, tmpSpaceSize // 1024, tmpSpaceToUse // 1024, diskThreshold // 1024, tmpEP["ddm_endpoint_name"]
                                         )
                                     )