panda-server-source 0.8.0__py3-none-any.whl

pandaserver/taskbuffer/WorkQueue.py

@@ -0,0 +1,227 @@
+"""
+work queue specification
+
+"""
+
+import re
+
+from pandaserver.taskbuffer.GlobalShares import Share
+
+RESOURCE = "Resource"
+ACTIVE_FUNCTIONS = [RESOURCE]
+
+
+class WorkQueue(object):
+    # attributes
+    _attributes = (
+        "queue_id",
+        "queue_name",
+        "queue_type",
+        "VO",
+        "queue_share",
+        "queue_order",
+        "criteria",
+        "variables",
+        "partitionID",
+        "stretchable",
+        "status",
+        "queue_function",
+    )
+
+    # parameters for selection criteria
+    _paramsForSelection = ("prodSourceLabel", "workingGroup", "processingType", "coreCount", "site", "eventService", "splitRule", "campaign")
+
+    # correspondence with Global Shares attributes and parameters
+    _attributes_gs_conversion_dic = {"name": "queue_name", "value": "queue_share", "prodsourcelabel": "queue_type", "queue_id": "queue_id", "vo": "VO"}
+
+    _params_gs_conversion_dic = {
+        "prodsourcelabel": "prodSourceLabel",
+        "workinggroup": "workingGroup",
+        "campaign": "campaign",
+        "processingtype": "processingType",
+    }
+
+    def __init__(self):
+        """
+        Constructor
+        """
+        # install attributes
+        for attr in self._attributes:
+            setattr(self, attr, None)
+
+        # global share is by default false
+        self.is_global_share = False
+        # throttled is set to True by default. Some Global Shares will overwrite it to False
+        self.throttled = True
+
+    def __str__(self):
+        """
+        String representation of a workqueue
+        :return: string with the representation of the work queue
+        """
+        return str(self.queue_name)
+
+    def dump(self):
+        """
+        Creates a human-friendly string with the work queue information
+        :return: string representation of the work queue
+        """
+        dump_str = f"id:{self.queue_id} order:{self.queue_order} name:{self.queue_name} share:{self.queue_share} "
+
+        # normal queue
+        if self.is_global_share:
+            dump_str += f"gs_name:{self.queue_name} (global share)"
+        else:
+            dump_str += f"criteria:{self.criteria} var:{str(self.variables)} eval:{self.evalString}"
+
+        return dump_str
+
+    def getID(self):
+        """
+        get ID
+        :return: returns a list with the ID of the work queue
+        """
+        return self.queue_id
+
+    def pack(self, values):
+        """
+        Packs tuple into the object
+        :param values: list with the values in the order declared in the attributes section
+        :return: nothing
+        """
+        for i, attr in enumerate(self._attributes):
+            val = values[i]
+            setattr(self, attr, val)
+
+        # disallow negative share
+        if self.queue_share is not None and self.queue_share < 0:
+            self.queue_share = 0
+
+        # convert variables string to a map of bind-variables
+        tmp_map = {}
+        try:
+            for item in self.variables.split(","):
+                # look for key: value
+                item = item.strip()
+                items = item.split(":")
+                if len(items) != 2:
+                    continue
+                # add
+                tmp_map[f":{items[0]}"] = items[1]
+        except Exception:
+            pass
+        # assign map
+        self.variables = tmp_map
+        # make a python statement for eval
+        if self.criteria in ["", None]:
+            # catch all
+            self.evalString = "True"
+        else:
+            tmp_eval_str = self.criteria
+            # replace IN/OR/AND to in/or/and
+            tmp_eval_str = re.sub(" IN ", " in ", tmp_eval_str, re.I)
+            tmp_eval_str = re.sub(" OR ", " or ", tmp_eval_str, re.I)
+            tmp_eval_str = re.sub(" AND ", " and ", tmp_eval_str, re.I)
+            # replace = to ==
+            tmp_eval_str = tmp_eval_str.replace("=", "==")
+            # replace LIKE
+            tmp_eval_str = re.sub("(?P<var>[^ \(]+)\s+LIKE\s+(?P<pat>[^ \(]+)", "re.search(\g<pat>,\g<var>,re.I) is not None", tmp_eval_str, re.I)
+            # NULL
+            tmp_eval_str = re.sub(" IS NULL", "==None", tmp_eval_str)
+            tmp_eval_str = re.sub(" IS NOT NULL", "!=None", tmp_eval_str)
+            # replace NOT to not
+            tmp_eval_str = re.sub(" NOT ", " not ", tmp_eval_str, re.I)
+            # fomat cases
+            for tmp_param in self._paramsForSelection:
+                tmp_eval_str = re.sub(tmp_param, tmp_param, tmp_eval_str, re.I)
+            # replace bind-variables
+            for tmp_key, tmp_val in self.variables.items():
+                if "%" in tmp_val:
+                    # wildcard
+                    tmp_val = tmp_val.replace("%", ".*")
+                    tmp_val = f"'^{tmp_val}$'"
+                else:
+                    # normal variable
+                    tmp_val = f"'{tmp_val}'"
+                tmp_eval_str = tmp_eval_str.replace(tmp_key, tmp_val)
+            # assign
+            self.evalString = tmp_eval_str
+
+    def pack_gs(self, gshare):
+        """
+        Packs tuple into the object
+        :param gshare: global share
+        :return: nothing
+        """
+
+        # the object becomes a global share wq
+        self.is_global_share = True
+        try:
+            tmp_map = {}
+            for i, attr in enumerate(gshare._attributes):
+                # global share attributes can be mapped to a wq attribute(1), to a wq param(2), or to none of both
+                # 1. if the gs attribute is mapped to a wq attribute, do a get and a set
+                if attr in self._attributes_gs_conversion_dic:
+                    attr_wq = self._attributes_gs_conversion_dic[attr]
+                    val = getattr(gshare, attr)
+                    setattr(self, attr_wq, val)
+                # 2. if the gs attribute is mapped to a wq param, add it to the bind variables dictionary
+                # Probably we don't need this, we just care about matching the gs name
+                if attr in self._params_gs_conversion_dic:
+                    param_wq = self._params_gs_conversion_dic[attr]
+                    val = getattr(gshare, attr)
+                    tmp_map[f":{param_wq}"] = val
+
+                # 3. Special case for throttled. This is defined additionally, since it's not present in WQs
+                if attr == "throttled" and gshare.throttled == "N":
+                    self.throttled = False
+
+            self.variables = tmp_map
+        except Exception:
+            pass
+
+    # evaluate in python
+    def evaluate(self, param_map):
+        # only active queues are evaluated
+        if self.isActive():
+            # normal queue
+            # expand parameters to local namespace
+            for tmp_param_key, tmp_param_val in param_map.items():
+                if isinstance(tmp_param_val, str):
+                    # add quotes for string
+                    exec(f'{tmp_param_key}="{tmp_param_val}"', globals())
+                else:
+                    exec(f"{tmp_param_key}={tmp_param_val}", globals())
+            # add default parameters if missing
+            for tmp_param in self._paramsForSelection:
+                if tmp_param not in param_map:
+                    exec(f"{tmp_param}=None", globals())
+            # evaluate
+            exec(f"ret_var = {self.evalString}", globals())
+            return self, ret_var
+
+        # return False
+        return self, False
+
+    # check if active
+    def isActive(self):
+        if self.status != "inactive":  # and self.queue_function in ACTIVE_FUNCTIONS:
+            return True
+        return False
+
+    # check if its eligible after global share alignment
+    def isAligned(self):
+        if self.queue_function == RESOURCE or self.is_global_share:
+            return True
+        return False
+
+    # return column names for INSERT
+    def column_names(cls):
+        ret = ""
+        for attr in cls._attributes:
+            if ret != "":
+                ret += ","
+            ret += attr
+        return ret
+
+    column_names = classmethod(column_names)

pandaserver/taskbuffer/WorkQueueMapper.py

@@ -0,0 +1,203 @@
+"""
+mapper to map task/job to a work queue
+
+"""
+
+import re
+
+from .WorkQueue import WorkQueue
+
+
+class WorkQueueMapper:
+    # constructor
+    def __init__(self):
+        # Initialize maps
+        self.work_queue_map = {}
+        self.work_queue_global_dic_by_name = {}
+        self.work_queue_global_dic_by_id = {}
+
+    def getSqlQuery(self):
+        """
+        Generates the SQL to get all work queues
+        """
+        sql = f"SELECT {WorkQueue.column_names()} FROM ATLAS_PANDA.JEDI_Work_Queue"
+
+        return sql
+
+    def makeMap(self, work_queues, global_leave_shares):
+        """
+        Creates the mapping with work queues and global shares
+        :param work_queues: work queues
+        :param global_leave_shares: global leave shares
+        :return
+        """
+
+        # 1. add all workqueues to the map
+        for wq in work_queues:
+            # pack
+            work_queue = WorkQueue()
+            work_queue.pack(wq)
+
+            # skip inactive queues
+            if not work_queue.isActive():
+                continue
+
+            # add VO
+            if work_queue.VO not in self.work_queue_map:
+                self.work_queue_map[work_queue.VO] = {}
+
+            # add type
+            if work_queue.queue_type not in self.work_queue_map[work_queue.VO]:
+                self.work_queue_map[work_queue.VO][work_queue.queue_type] = []
+
+            self.work_queue_map[work_queue.VO][work_queue.queue_type].append(work_queue)
+            self.work_queue_global_dic_by_name[work_queue.queue_name] = work_queue
+            self.work_queue_global_dic_by_id[work_queue.queue_id] = work_queue
+
+        # sort the queue list by order
+        for vo in self.work_queue_map:
+            for type in self.work_queue_map[vo]:
+                # make ordered map
+                ordered_map = {}
+                queue_map = self.work_queue_map[vo][type]
+                for wq in queue_map:
+                    if wq.queue_order not in ordered_map:
+                        ordered_map[wq.queue_order] = []
+                    # append
+                    ordered_map[wq.queue_order].append(wq)
+                # make sorted list
+                ordered_list = list(ordered_map.keys())
+                ordered_list.sort(key=lambda x: (x is None, x))
+                new_list = []
+                for order_val in ordered_list:
+                    new_list += ordered_map[order_val]
+                # set new list
+                self.work_queue_map[vo][type] = new_list
+
+        # 2. add all the global shares
+        for gs in global_leave_shares:
+            work_queue_gs = WorkQueue()
+            work_queue_gs.pack_gs(gs)
+
+            if work_queue_gs.VO is None:
+                vo = "atlas"
+            else:
+                vo = work_queue_gs.VO
+
+            if vo not in self.work_queue_map:
+                self.work_queue_map[vo] = {}
+
+            if work_queue_gs.queue_type not in self.work_queue_map[vo]:
+                self.work_queue_map[vo][work_queue_gs.queue_type] = []
+
+            self.work_queue_map[vo][work_queue_gs.queue_type].append(work_queue_gs)
+            self.work_queue_global_dic_by_name[work_queue_gs.queue_name] = work_queue_gs
+            self.work_queue_global_dic_by_id[work_queue_gs.queue_id] = work_queue_gs
+
+        # return
+        return
+
+    def dump(self):
+        """
+        Creates a human-friendly string showing the work queue mappings
+        :return: string representation of the work queue mappings
+        """
+        dump_str = "WorkQueue mapping\n"
+        for VO in self.work_queue_map:
+            dump_str += f"  VO={VO}\n"
+            for type in self.work_queue_map[VO]:
+                dump_str += f"    type={type}\n"
+                for workQueue in self.work_queue_map[VO][type]:
+                    dump_str += f"    {workQueue.dump()}\n"
+        # return
+        return dump_str
+
+    def getQueueWithSelParams(self, vo, type, **param_map):
+        """
+        Used for tagging of work queues in task refiner. Get a work queue based on the selection parameters
+        :param vo: vo
+        :param type: type (in practice equivalent to prodsourcelabel)
+        :param param_map: parameter selection map
+        :return: work queue object and explanation in case no queue was found
+        """
+        ret_str = ""
+        if vo not in self.work_queue_map:
+            ret_str = f"queues for vo={vo} are undefined"
+        elif type not in self.work_queue_map[vo]:
+            # check type
+            ret_str = f"queues for type={type} are undefined in vo={vo}"
+        else:
+            for wq in self.work_queue_map[vo][type]:
+                # don't return global share IDs for work queues
+                if wq.is_global_share:
+                    continue
+
+                # evaluate
+                try:
+                    ret_queue, result = wq.evaluate(param_map)
+                    if result:
+                        return ret_queue, ret_str
+                except Exception:
+                    ret_str += f"{wq.queue_name},"
+
+            ret_str = ret_str[:-1]
+            if ret_str != "":
+                new_ret_str = f"eval with VO={vo} "
+                for tmp_param_key, tmp_param_val in param_map.items():
+                    new_ret_str += "{0}={1} failed for {0}".format(tmp_param_key, tmp_param_val, ret_str)
+                ret_str = new_ret_str
+
+        # no queue matched to selection parameters
+        return None, ret_str
+
+    def getQueueByName(self, vo, type, queue_name):
+        """
+        # get queue by name
+        :param queue_name: name of the queue
+        :param vo: vo
+        :param type: type
+        :return: queue object or None if not found
+        """
+        if vo in self.work_queue_map and type in self.work_queue_map[vo]:
+            for wq in self.work_queue_map[vo][type]:
+                if wq.queue_name == queue_name:
+                    return wq
+        return None
+
+    # get queue with ID
+    def getQueueWithIDGshare(self, queue_id, gshare_name):
+        # 1. Check for a Resource queue
+        if queue_id in self.work_queue_global_dic_by_id and self.work_queue_global_dic_by_id[queue_id].queue_function == "Resource":
+            return self.work_queue_global_dic_by_id[queue_id]
+
+        # 2. If it wasn't a resource queue, return the global share work queue
+        if gshare_name in self.work_queue_global_dic_by_name:
+            return self.work_queue_global_dic_by_name[gshare_name]
+
+        # not found
+        return None
+
+    # get queue list with VO and type
+    def getAlignedQueueList(self, vo, queue_type):
+        """
+        NOTE: Returns ONLY resource queues and global shares (old non-resource queues are skipped)
+        """
+        ret_list = []
+
+        if vo in self.work_queue_map:
+            # if queue type was specified
+            if queue_type not in ["", None, "any"]:
+                for map_queue_type in self.work_queue_map[vo]:
+                    if re.match(map_queue_type, queue_type):
+                        for tmp_wq in self.work_queue_map[vo][map_queue_type]:
+                            if tmp_wq.isAligned():
+                                ret_list.append(tmp_wq)
+
+            # include all queue types
+            else:
+                for tmp_type, tmp_wq_list in self.work_queue_map[vo].items():
+                    for tmp_wq in tmp_wq_list:
+                        if tmp_wq.isAligned():
+                            ret_list.append(tmp_wq)
+
+        return ret_list

pandaserver/taskbuffer/WorkerSpec.py

@@ -0,0 +1,143 @@
+"""
+worker specification
+
+"""
+
+import datetime
+
+
+class WorkerSpec(object):
+    # attributes
+    _attributes = (
+        "harvesterID",
+        "workerID",
+        "batchID",
+        "queueName",
+        "status",
+        "computingSite",
+        "nCore",
+        "nodeID",
+        "submitTime",
+        "startTime",
+        "endTime",
+        "lastUpdate",
+        "stdOut",
+        "stdErr",
+        "batchLog",
+        "jdl",
+        "resourceType",
+        "nativeExitCode",
+        "nativeStatus",
+        "diagMessage",
+        "nJobs",
+        "computingElement",
+        "submissionHost",
+        "harvesterHost",
+        "errorCode",
+        "jobType",
+        "minRamCount",
+    )
+    # slots
+    __slots__ = _attributes + ("_changedAttrs",)
+    # attributes which have 0 by default
+    _zeroAttrs = ()
+    # catchall resouce type
+    RT_catchall = "ANY"
+
+    # constructor
+    def __init__(self):
+        # install attributes
+        for attr in self._attributes:
+            object.__setattr__(self, attr, None)
+        # map of changed attributes
+        object.__setattr__(self, "_changedAttrs", {})
+
+    # override __setattr__ to collect the changed attributes
+    def __setattr__(self, name, value):
+        oldVal = getattr(self, name)
+        # convert string to datetime
+        if isinstance(value, str) and value.startswith("datetime/"):
+            value = datetime.datetime.strptime(value.split("/")[-1], "%Y-%m-%d %H:%M:%S.%f")
+        object.__setattr__(self, name, value)
+        # collect changed attributes
+        if oldVal != value:
+            self._changedAttrs[name] = value
+
+    # reset changed attribute list
+    def resetChangedList(self):
+        self._oldPandaID = self.PandaID
+        object.__setattr__(self, "_changedAttrs", {})
+
+    # return map of values
+    def valuesMap(self, onlyChanged=False):
+        ret = {}
+        for attr in self._attributes:
+            if onlyChanged and attr not in self._changedAttrs:
+                continue
+            val = getattr(self, attr)
+            if val is None:
+                if attr in self._zeroAttrs:
+                    val = 0
+            ret[f":{attr}"] = val
+        return ret
+
+    # pack tuple into FileSpec
+    def pack(self, values):
+        for i in range(len(self._attributes)):
+            attr = self._attributes[i]
+            val = values[i]
+            object.__setattr__(self, attr, val)
+
+    # return column names for INSERT
+    def columnNames(cls, prefix=None):
+        ret = ""
+        for attr in cls._attributes:
+            if prefix is not None:
+                ret += f"{prefix}."
+            ret += f"{attr},"
+        ret = ret[:-1]
+        return ret
+
+    columnNames = classmethod(columnNames)
+
+    # return expression of bind variables for INSERT
+    def bindValuesExpression(cls):
+        from pandaserver.config import panda_config
+
+        ret = "VALUES("
+        for attr in cls._attributes:
+            ret += f":{attr},"
+        ret = ret[:-1]
+        ret += ")"
+        return ret
+
+    bindValuesExpression = classmethod(bindValuesExpression)
+
+    # return an expression of bind variables for UPDATE to update only changed attributes
+    def bindUpdateChangesExpression(self):
+        ret = ""
+        for attr in self._attributes:
+            if attr not in self._changedAttrs:
+                continue
+            ret += "{0}=:{0},".format(attr)
+        ret = ret[:-1]
+        return ret
+
+    # return state values to be pickled
+    def __getstate__(self):
+        state = []
+        for attr in self._attributes:
+            val = getattr(self, attr)
+            state.append(val)
+        state.append(self._changedAttrs)
+        return state
+
+    # restore state from the unpickled state values
+    def __setstate__(self, state):
+        i = 0
+        for attr in self._attributes:
+            if i >= len(state) - 1:
+                break
+            object.__setattr__(self, attr, state[i])
+            i += 1
+        object.__setattr__(self, "_changedAttrs", state[-1])