panda-server-source 0.8.0__py3-none-any.whl

pandaserver/taskbuffer/GlobalShares.py

@@ -0,0 +1,244 @@
+# Definitions
+EXECUTING = "executing"
+QUEUED = "queued"
+PLEDGED = "pledged"
+
+
+class Node(object):
+    def __init__(self):
+        self.children = []
+
+    def add_child(self, node):
+        self.children.append(node)
+
+    def get_leaves(self, leaves=[]):
+        # If the node has no leaves, return the node in a list
+        if not self.children:
+            leaves.append(self)
+            return leaves
+
+        # Recursively get to the bottom
+        for child in self.children:
+            child.get_leaves(leaves)
+
+        return leaves
+
+
+class Share(Node):
+    """
+    Implement the share node
+    """
+
+    _attributes = (
+        "name",
+        "value",
+        "parent",
+        "prodsourcelabel",
+        "workinggroup",
+        "campaign",
+        "processingtype",
+        "transpath",
+        "vo",
+        "rtype",
+        "queue_id",
+        "throttled",
+    )
+
+    def __str__(self, level=0):
+        """
+        Print the tree structure
+        """
+        ret = "{0} name: {1}, value: {2}\n".format("\t" * level, self.name, self.value)
+        for child in self.children:
+            ret += child.__str__(level + 1)
+        return ret
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __mul__(self, other):
+        """
+        If I multiply a share object by a number, multiply the value field
+        """
+        self.value *= other
+        return self.value
+
+    def __rmul__(self, other):
+        return self.__mul__
+
+    def __imul__(self, other):
+        return self.__mul__
+
+    def __init__(
+        self,
+        name,
+        value,
+        parent,
+        prodsourcelabel,
+        workinggroup,
+        campaign,
+        processingtype,
+        transpath,
+        rtype,
+        vo,
+        queue_id,
+        throttled,
+    ):
+        # Create default attributes
+        for attr in self._attributes:
+            setattr(self, attr, None)
+
+        Node.__init__(self)
+        self.name = name
+        self.value = value
+        self.parent = parent
+        self.prodsourcelabel = prodsourcelabel
+        self.workinggroup = workinggroup
+        self.campaign = campaign
+        self.processingtype = processingtype
+        self.transpath = transpath
+        self.rtype = rtype
+        self.vo = vo
+        self.queue_id = queue_id
+        self.throttled = throttled
+
+    def pretty_print_hs_distribution(self, hs_distribution, level=0):
+        try:
+            executing = hs_distribution[self.name][EXECUTING] / 1000.0
+        except Exception:
+            executing = 0
+
+        try:
+            target = hs_distribution[self.name][PLEDGED] / 1000.0
+        except Exception:
+            target = 0
+
+        try:
+            queued = hs_distribution[self.name][QUEUED] / 1000.0
+        except Exception:
+            queued = 0
+
+        ret = "{0} name: {1}, values: {2:.1f}k|{3:.1f}k|{4:.1f}k\n".format("\t" * level, self.name, executing, target, queued)
+        for child in self.children:
+            ret += child.pretty_print_hs_distribution(hs_distribution, level + 1)
+        return ret
+
+    def normalize(self, multiplier=100, divider=100):
+        """
+        Will run down the branch and normalize values beneath
+        """
+        self.value *= multiplier * 1.0 / divider
+        if not self.children:
+            return
+
+        divider = 0
+        for child in self.children:
+            divider += child.value
+
+        multiplier = self.value
+
+        for child in self.children:
+            child.normalize(multiplier=multiplier, divider=divider)
+
+        return
+
+    def sort_branch_by_current_hs_distribution(self, hs_distribution):
+        """
+        Runs down the branch in order of under-pledging. It returns a list of sorted leave shares
+        """
+        sorted_shares = []
+
+        # If the node has no leaves, return the node in a list
+        if not self.children:
+            sorted_shares = [self]
+            return sorted_shares
+
+        # If the node has leaves, sort the children
+        children_sorted = []
+        for child1 in self.children:
+            loop_index = 0
+            insert_index = len(children_sorted)  # insert at the end, if not deemed otherwise
+
+            # Calculate under-pledging
+            try:
+                child1_under_pledge = hs_distribution[child1.name][EXECUTING] * 1.0 / hs_distribution[child1.name][PLEDGED]
+            except ZeroDivisionError:
+                child1_under_pledge = 10**6  # Initialize to a large default number
+
+            for child2 in children_sorted:
+                try:
+                    # Calculate under-pledging
+                    child2_under_pledge = hs_distribution[child2.name][EXECUTING] * 1.0 / hs_distribution[child2.name][PLEDGED]
+                except ZeroDivisionError:
+                    child2_under_pledge = 10**6  # Initialize to a large default number
+                except KeyError:
+                    continue  # Does not exist
+
+                if child1_under_pledge < child2_under_pledge:
+                    insert_index = loop_index
+                    break
+
+                loop_index += 1
+
+            # Insert the child into the list
+            children_sorted.insert(insert_index, child1)
+
+        # Go recursively and sort the grand* children
+        for child in children_sorted:
+            sorted_shares.extend(child.sort_branch_by_current_hs_distribution(hs_distribution))
+
+        return sorted_shares
+
+    def aggregate_hs_distribution(self, hs_distribution):
+        """
+        We have the current HS distribution values for the leaves, but want to propagate it updwards to the parents.
+        We will traverse the tree from top to bottom and bring up the aggregated values.
+        """
+        executing, queued, pledged = 0, 0, 0
+
+        # If the node has no children, it's a leave and should have an entry in the hs_distribution
+        if not self.children:
+            try:
+                executing = hs_distribution[self.name][EXECUTING]
+                queued = hs_distribution[self.name][QUEUED]
+                pledged = hs_distribution[self.name][PLEDGED]
+            except KeyError:
+                pass
+
+            return executing, queued, pledged
+
+        # If the node has children, sum up the values of the children
+        executing = 0
+        queued = 0
+        pledged = 0
+
+        for child in self.children:
+            (
+                executing_child,
+                queued_child,
+                pledged_child,
+            ) = child.aggregate_hs_distribution(hs_distribution)
+            executing += executing_child
+            queued += queued_child
+            pledged += pledged_child
+
+        # Add the aggregated value to the map
+        hs_distribution[self.name] = {
+            EXECUTING: executing,
+            QUEUED: queued,
+            PLEDGED: pledged,
+        }
+
+        # Return the aggregated values
+        return executing, queued, pledged
+
+    # return column names
+    def column_names(cls):
+        ret = ""
+        for attr in cls._attributes:
+            if ret != "":
+                ret += ","
+            ret += attr
+        return ret
+
+    column_names = classmethod(column_names)

pandaserver/taskbuffer/HarvesterMetricsSpec.py

@@ -0,0 +1,92 @@
+"""
+worker specification
+
+"""
+
+import datetime
+
+
+class HarvesterMetricsSpec(object):
+    # attributes
+    _attributes = ("harvester_ID", "creation_time", "harvester_host", "metrics")
+    # slots
+    __slots__ = _attributes + ("_changedAttrs",)
+    # attributes which have 0 by default
+    _zeroAttrs = ()
+
+    # constructor
+    def __init__(self):
+        # install attributes
+        for attr in self._attributes:
+            object.__setattr__(self, attr, None)
+        # map of changed attributes
+        object.__setattr__(self, "_changedAttrs", {})
+
+    # override __setattr__ to collecte the changed attributes
+    def __setattr__(self, name, value):
+        oldVal = getattr(self, name)
+        # convert string to datetime
+        if isinstance(value, str) and value.startswith("datetime/"):
+            value = datetime.datetime.strptime(value.split("/")[-1], "%Y-%m-%d %H:%M:%S.%f")
+        object.__setattr__(self, name, value)
+        # collect changed attributes
+        if oldVal != value:
+            self._changedAttrs[name] = value
+
+    # reset changed attribute list
+    def resetChangedList(self):
+        self._oldPandaID = self.PandaID
+        object.__setattr__(self, "_changedAttrs", {})
+
+    # return map of values
+    def valuesMap(self, onlyChanged=False):
+        ret = {}
+        for attr in self._attributes:
+            if onlyChanged and attr not in self._changedAttrs:
+                continue
+            val = getattr(self, attr)
+            if val is None:
+                if attr in self._zeroAttrs:
+                    val = 0
+            ret[f":{attr}"] = val
+        return ret
+
+    # pack tuple into FileSpec
+    def pack(self, values):
+        for i in range(len(self._attributes)):
+            attr = self._attributes[i]
+            val = values[i]
+            object.__setattr__(self, attr, val)
+
+    # return column names for INSERT
+    def columnNames(cls):
+        ret = ""
+        for attr in cls._attributes:
+            ret += f"{attr},"
+        ret = ret[:-1]
+        return ret
+
+    columnNames = classmethod(columnNames)
+
+    # return expression of bind variables for INSERT
+    def bindValuesExpression(cls):
+        from pandaserver.config import panda_config
+
+        ret = "VALUES("
+        for attr in cls._attributes:
+            ret += f":{attr},"
+        ret = ret[:-1]
+        ret += ")"
+        return ret
+
+    bindValuesExpression = classmethod(bindValuesExpression)
+
+    # return an expression of bind variables for UPDATE to update only changed attributes
+    def bindUpdateChangesExpression(self):
+        ret = ""
+        for attr in self._attributes:
+            if attr not in self._changedAttrs:
+                continue
+            ret += "{0}=:{0},".format(attr)
+        ret = ret[:-1]
+        return ret

pandaserver/taskbuffer/Initializer.py

@@ -0,0 +1,82 @@
+import sys
+from threading import Lock
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.config import panda_config
+
+_logger = PandaLogger().getLogger("Initializer")
+
+
+class Initializer:
+    """
+    Initialize a dummy database connection.
+
+    Returns:
+        bool: True if the initialization is successful, False otherwise.
+    """
+
+    def __init__(self):
+        self.lock = Lock()
+        self.first = True
+
+    def init(self):
+        _logger.debug(f"init new={self.first}")
+        # do nothing when nDBConnection is 0
+        if panda_config.nDBConnection == 0:
+            return True
+        # lock
+        self.lock.acquire()
+        if self.first:
+            self.first = False
+            try:
+                _logger.debug("connect")
+                # connect
+                if panda_config.backend == "oracle":
+                    import oracledb
+
+                    oracledb.init_oracle_client()
+                    conn = oracledb.connect(user=panda_config.dbuser, password=panda_config.dbpasswd, dsn=panda_config.dbhost)
+
+                elif panda_config.backend == "postgres":
+                    import psycopg2
+
+                    conn = psycopg2.connect(
+                        host=panda_config.dbhost,
+                        dbname=panda_config.dbname,
+                        port=panda_config.dbport,
+                        connect_timeout=panda_config.dbtimeout,
+                        user=panda_config.dbuser,
+                        password=panda_config.dbpasswd,
+                    )
+
+                else:
+                    import MySQLdb
+
+                    conn = MySQLdb.connect(
+                        host=panda_config.dbhost,
+                        db=panda_config.dbname,
+                        port=panda_config.dbport,
+                        connect_timeout=panda_config.dbtimeout,
+                        user=panda_config.dbuser,
+                        passwd=panda_config.dbpasswd,
+                    )
+
+                # close
+                conn.close()
+                _logger.debug("done")
+
+            except Exception:
+                self.lock.release()
+                exception_type, exception_value, traceback = sys.exc_info()
+                _logger.error(f"connect : {exception_type} {exception_value}")
+                return False
+
+        # release the lock
+        self.lock.release()
+        return True
+
+
+# singleton
+initializer = Initializer()
+del Initializer