panda-server-source 0.8.0__py3-none-any.whl

pandaserver/taskbuffer/ResourceSpec.py

@@ -0,0 +1,175 @@
+"""
+Resource type specification for JEDI
+
+"""
+
+from . import JobUtils
+
+HIMEM_THRESHOLD = 2000  # MB per core
+BASIC_RESOURCE_TYPE = "SCORE"
+
+
+class ResourceSpecMapper(object):
+    def __init__(self, resource_types):
+        """
+        :param resource_types: list of ResourceSpec objects
+        """
+        self.resource_types = resource_types
+
+    def is_single_core(self, resource_name):
+        for resource_type in self.resource_types:
+            if resource_type.resource_name == resource_name:
+                return resource_type.is_single_core()
+        return False
+
+    def is_multi_core(self, resource_name):
+        for resource_type in self.resource_types:
+            if resource_type.resource_name == resource_name:
+                return resource_type.is_multi_core()
+        return False
+
+    def is_high_memory(self, resource_name, memory_threshold=HIMEM_THRESHOLD):
+        for resource_type in self.resource_types:
+            if resource_type.resource_name == resource_name:
+                if resource_type.maxrampercore is None or resource_type.maxrampercore > memory_threshold:
+                    return True
+        return False
+
+    def translate_resourcetype_to_cores(self, resource_name, cores_queue):
+        # if the resource type is multi-core, return the number of cores in the queue
+        if self.is_multi_core(resource_name):
+            return cores_queue
+
+        return 1
+
+    def filter_out_high_memory_resourcetypes(self, memory_threshold=HIMEM_THRESHOLD):
+        resource_names = list(
+            map(
+                lambda resource_type: resource_type.resource_name,
+                filter(lambda resource_type: not self.is_high_memory(resource_type.resource_name, memory_threshold=memory_threshold), self.resource_types),
+            )
+        )
+        return resource_names
+
+
+class ResourceSpec(object):
+    # attributes
+    attributes = (
+        "resource_name",
+        "mincore",
+        "maxcore",
+        "minrampercore",
+        "maxrampercore",
+    )
+
+    def __init__(self, resource_name, mincore, maxcore, minrampercore, maxrampercore):
+        object.__setattr__(self, "resource_name", resource_name)
+        object.__setattr__(self, "mincore", mincore)
+        object.__setattr__(self, "maxcore", maxcore)
+        object.__setattr__(self, "minrampercore", minrampercore)
+        object.__setattr__(self, "maxrampercore", maxrampercore)
+
+    def match_task(self, task_spec):
+        return self.match_task_basic(
+            task_spec.coreCount,
+            task_spec.ramCount,
+            task_spec.baseRamCount,
+            task_spec.ramUnit,
+        )
+
+    def match_task_basic(self, corecount, ramcount, base_ramcount, ram_unit):
+        # Default parameters
+        if corecount is None:  # corecount None is also used for 1
+            corecount = 1
+        elif corecount == 0:  # corecount 0 means it can be anything. We will use 8 as a standard MCORE default
+            corecount = 8
+
+        if ramcount is None:
+            ramcount = 0
+
+        if base_ramcount is None:
+            base_ramcount = 0
+
+        # check min cores
+        if self.mincore is not None and corecount < self.mincore:
+            return False
+
+        # check max cores
+        if self.maxcore is not None and corecount > self.maxcore:
+            return False
+
+        if ram_unit in ("MBPerCore", "MBPerCoreFixed"):
+            ram_per_core = (ramcount * corecount + base_ramcount) / corecount
+        else:
+            ram_per_core = (ramcount + base_ramcount) / corecount
+        ram_per_core = JobUtils.compensate_ram_count(ram_per_core)
+
+        # check min ram
+        if self.minrampercore is not None and ram_per_core < self.minrampercore:
+            return False
+
+        # check max ram
+        if self.maxrampercore is not None and ram_per_core > self.maxrampercore:
+            return False
+
+        return True
+
+    def match_job(self, job_spec):
+        # Default parameters
+        if job_spec.coreCount in (None, "NULL"):  # corecount None is also used for 1
+            corecount = 1
+        elif job_spec.coreCount == 0:  # corecount 0 means it can be anything. We will use 8 as a standard MCORE default
+            corecount = 8
+        else:
+            corecount = job_spec.coreCount
+
+        if job_spec.minRamCount in (
+            None,
+            "NULL",
+        ):  # jobs come with ram already pre-calculated and in MB
+            ramcount = 0
+        else:
+            ramcount = job_spec.minRamCount
+
+        # check min cores
+        if self.mincore is not None and corecount < self.mincore:
+            return False
+
+        # check max cores
+        if self.maxcore is not None and corecount > self.maxcore:
+            return False
+
+        # We assume ram unit is always MB
+        ram_per_core = round(ramcount / corecount)
+        # check min ram
+        if self.minrampercore is not None and ram_per_core < self.minrampercore:
+            return False
+
+        # check max ram
+        if self.maxrampercore is not None and ram_per_core > self.maxrampercore:
+            return False
+
+        return True
+
+    def is_single_core(self):
+        if self.mincore is not None and self.mincore == 1 and self.maxcore is not None and self.maxcore == 1:
+            return True
+        return False
+
+    def is_multi_core(self):
+        if self.mincore is not None and self.mincore > 1:
+            return True
+
+    def column_names(cls, prefix=None):
+        """
+        return column names for DB interactions
+        """
+        ret = ""
+        for attr in cls.attributes:
+            if prefix is not None:
+                ret += f"{prefix}."
+            ret += f"{attr},"
+        ret = ret[:-1]
+        return ret
+
+    column_names = classmethod(column_names)

pandaserver/taskbuffer/SQLDumper.py

@@ -0,0 +1,37 @@
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+# logger
+_logger = PandaLogger().getLogger("SQLDumper")
+
+
+class SQLDumper(object):
+    def __init__(self, cur):
+        self.cursor = cur
+
+    def __iter__(self):
+        return self.cursor.__iter__()
+
+    def next(self):
+        return self.cursor.next()
+
+    def my_execute(self, sql, var=None):
+        if var is None:
+            var = {}
+        _logger.debug(f"SQL={sql} var={str(var)}")
+        return self.cursor.execute(sql, var)
+
+    def my_executemany(self, sql, vars=None):
+        if vars is None:
+            vars = []
+        _logger.debug(f"SQL_many={sql} var_many={str(vars)}")
+        return self.cursor.executemany(sql, vars)
+
+    def __getattribute__(self, name):
+        if name == "execute":
+            return object.__getattribute__(self, "my_execute")
+        elif name == "executemany":
+            return object.__getattribute__(self, "my_executemany")
+        elif name in ["cursor", "__iter__", "next"]:
+            return object.__getattribute__(self, name)
+        else:
+            return getattr(self.cursor, name)

pandaserver/taskbuffer/SQLManager.py

@@ -0,0 +1,60 @@
+class SQLQueryManager:
+    def __init__(self, sql_query):
+        self.sql_query = sql_query
+
+    def format_query(self):
+        """Format the SQL query by removing line breaks and redundant spaces."""
+        # Replace newline characters with a single space
+        formatted_query = self.sql_query.replace('\n', ' ')
+        # Remove redundant spaces (more than one space in a row)
+        formatted_query = ' '.join(formatted_query.split())
+        return formatted_query
+
+    def set_query(self, sql_query):
+        """Set a new SQL query."""
+        self.sql_query = sql_query
+
+    def get_query(self):
+        """Get the current SQL query."""
+        return self.sql_query
+
+    def log_query(self):
+        """Log the formatted SQL query."""
+        formatted_query = self.format_query()
+        print(f"SQL Query: {formatted_query}")
+
+    def append(self, sql_text):
+        """Append more text to the existing SQL query."""
+        self.sql_query += ' ' + sql_text
+
+    def __str__(self):
+        """Custom string representation of the SQLQueryManager instance."""
+        return self.format_query()
+
+
+# Example usage:
+if __name__ == "__main__":
+    # Initialize the SQLQueryManager with an SQL query
+    sql_manager = SQLQueryManager("""
+        SELECT *
+        FROM your_table
+        WHERE condition = 'something'
+        ORDER BY column_name
+    """)
+
+    # Log the SQL query
+    print(sql_manager)  # This will print the formatted SQL query
+
+    # Change the SQL query
+    new_query = """
+        SELECT id, name
+        FROM another_table
+        WHERE condition = 'another condition'
+    """
+    sql_manager.set_query(new_query)
+    sql_manager.append("""
+    AND condition2 = 'more conditions'
+    """)
+
+    # Log the updated SQL query
+    print(sql_manager)  # This will print the updated formatted SQL query

pandaserver/taskbuffer/SiteSpec.py

@@ -0,0 +1,304 @@
+"""
+site specification
+
+"""
+
+import re
+
+catchall_keys = {
+    k: k
+    for k in [
+        "useJumboJobs",
+        "gpu",
+        "grandly_unified",
+        "nSimEvents",
+        "minEventsForJumbo",
+        "maxDiskPerCore",
+        "use_only_local_data",
+        "disableReassign",
+        "jobChunkSize",
+        "bareNucleus",
+        "secondaryNucleus",
+        "allowed_processing",
+        "excluded_processing",
+        "per_core_attr",
+        "allow_no_pilot",
+    ]
+}
+
+
+class SiteSpec(object):
+    # attributes
+    _attributes = (
+        "sitename",
+        "nickname",
+        "dq2url",
+        "cloud",
+        "ddm",
+        "ddm_input",
+        "ddm_output",
+        "type",
+        "releases",
+        "memory",
+        "maxtime",
+        "status",
+        "setokens_input",
+        "setokens_output",
+        "defaulttoken",
+        "validatedreleases",
+        "maxinputsize",
+        "comment",
+        "statusmodtime",
+        "pledgedCPU",
+        "coreCount",
+        "reliabilityLevel",
+        "iscvmfs",
+        "transferringlimit",
+        "maxwdir",
+        "fairsharePolicy",
+        "mintime",
+        "allowfax",
+        "pandasite",
+        "corepower",
+        "wnconnectivity",
+        "catchall",
+        "role",
+        "pandasite_state",
+        "ddm_endpoints_input",
+        "ddm_endpoints_output",
+        "maxrss",
+        "minrss",
+        "direct_access_lan",
+        "direct_access_wan",
+        "tier",
+        "objectstores",
+        "is_unified",
+        "unified_name",
+        "jobseed",
+        "capability",
+        "num_slots_map",
+        "workflow",
+        "maxDiskio",
+    )
+
+    # constructor
+    def __init__(self):
+        # install attributes
+        for attr in self._attributes:
+            setattr(self, attr, None)
+
+    # serialize
+    def __str__(self):
+        str = ""
+        for attr in self._attributes:
+            str += f"{attr}:{getattr(self, attr)} "
+        return str
+
+    # check if direct IO is used when tasks allow it
+    def isDirectIO(self):
+        if self.direct_access_lan is True:
+            return True
+        return False
+
+    # check what type of jobs are allowed
+    def getJobSeed(self):
+        tmpVal = self.jobseed
+        if tmpVal is None:
+            return "std"
+        return tmpVal
+
+    # get value from catchall
+    def getValueFromCatchall(self, key):
+        if self.catchall is None:
+            return None
+        if key not in catchall_keys:
+            return None
+        key = catchall_keys[key]
+        for tmpItem in self.catchall.split(","):
+            tmpMatch = re.search(f"^{key}=(.+)", tmpItem)
+            if tmpMatch is not None:
+                return tmpMatch.group(1)
+        return None
+
+    # has value in catchall
+    def hasValueInCatchall(self, key):
+        if self.catchall is None:
+            return False
+        if key not in catchall_keys:
+            return False
+        key = catchall_keys[key]
+        for tmpItem in self.catchall.split(","):
+            tmpMatch = re.search(f"^{key}(=|)*", tmpItem)
+            if tmpMatch is not None:
+                return True
+        return False
+
+    # allow WAN input access
+    def allowWanInputAccess(self):
+        return self.direct_access_lan is True and self.direct_access_wan is True
+
+    # use jumbo jobs
+    def useJumboJobs(self):
+        return self.hasValueInCatchall("useJumboJobs")
+
+    # GPU
+    def isGPU(self):
+        return self.hasValueInCatchall("gpu")
+
+    def is_grandly_unified(self):
+        if self.hasValueInCatchall("grandly_unified") or self.type == "unified":
+            return True
+        return False
+
+    def runs_production(self):
+        if self.type == "production" or self.is_grandly_unified():
+            return True
+        return False
+
+    def runs_analysis(self):
+        if self.type == "analysis" or self.is_grandly_unified():
+            return True
+        return False
+
+    # get unified name
+    def get_unified_name(self):
+        if self.unified_name is None:
+            return self.sitename
+        return self.unified_name
+
+    # get number of simulated events for dynamic number of events
+    def get_n_sim_events(self):
+        tmpVal = self.getValueFromCatchall("nSimEvents")
+        if tmpVal is None:
+            return None
+        return int(tmpVal)
+
+    # get minimum of remaining events for jumbo jobs
+    def getMinEventsForJumbo(self):
+        tmpVal = self.getValueFromCatchall("minEventsForJumbo")
+        if tmpVal is None:
+            return None
+        return int(tmpVal)
+
+    # check if opportunistic
+    def is_opportunistic(self):
+        return self.pledgedCPU == -1
+
+    # get number of jobs for standby
+    def getNumStandby(self, sw_id, resource_type):
+        numMap = self.num_slots_map
+        # neither gshare or workqueue is definied
+        if sw_id not in numMap:
+            if None in numMap:
+                sw_id = None
+            else:
+                return None
+        # give the total if resource type is undefined
+        if resource_type is None:
+            return sum(numMap[sw_id].values())
+        # give the number for the resource type
+        if resource_type in numMap[sw_id]:
+            return numMap[sw_id][resource_type]
+        elif None in numMap[sw_id]:
+            return numMap[sw_id][None]
+        return None
+
+    # get max disk per core
+    def get_max_disk_per_core(self):
+        tmpVal = self.getValueFromCatchall("maxDiskPerCore")
+        try:
+            return int(tmpVal)
+        except Exception:
+            pass
+        return None
+
+    # use local data only
+    def use_only_local_data(self):
+        return self.hasValueInCatchall("use_only_local_data")
+
+    # check if use VP
+    def use_vp(self, scope):
+        # use default scope if missing
+        if scope not in self.ddm_endpoints_input:
+            scope = "default"
+        # check if VP_DISK is associated
+        if scope in self.ddm_endpoints_input and [i for i in self.ddm_endpoints_input[scope].getAllEndPoints() if i.endswith("_VP_DISK")]:
+            return True
+        return False
+
+    # check if always uses direct IO
+    def always_use_direct_io(self):
+        return self.maxinputsize == -1
+
+    # disable reassign
+    def disable_reassign(self):
+        if self.hasValueInCatchall("disableReassign"):
+            return True
+        self.status == "paused"
+
+    # get job chunk size
+    def get_job_chunk_size(self):
+        try:
+            return int(self.getValueFromCatchall("jobChunkSize"))
+        except Exception:
+            return None
+
+    # get WN connectivity
+    def get_wn_connectivity(self):
+        if self.wnconnectivity is None:
+            return None
+        items = self.wnconnectivity.split("#")
+        if not items or not items[0]:
+            return None
+        else:
+            return items[0]
+
+    # get IP stack
+    def get_ipstack(self):
+        if self.wnconnectivity is None:
+            return None
+        items = self.wnconnectivity.split("#")
+        if len(items) == 2 and items[-1]:
+            return items[-1]
+        else:
+            return None
+
+    # get bare nucleus mode
+    def bare_nucleus_mode(self):
+        mode = self.getValueFromCatchall("bareNucleus")
+        if mode in ["only", "allow"]:
+            return mode
+        return None
+
+    # get secondary nucleus
+    def secondary_nucleus(self):
+        n = self.getValueFromCatchall("secondaryNucleus")
+        if n:
+            return n
+        return None
+
+    # get allowed processing types
+    def get_allowed_processing_types(self):
+        """
+        Get allowed processing types for processing type-based job brokerage to access only tasks with specific processing types.
+        They are defined in the catchall field as a pipe-separated list with the key "allowed_processing".
+        """
+        n = self.getValueFromCatchall("allowed_processing")
+        if n:
+            return n.split("|")
+        return None
+
+    # get excluded process types
+    def get_excluded_processing_types(self):
+        """
+        Get excluded processing types for processing type-based job brokerage to exclude tasks with specific processing types.
+        They are defined in the catchall field as a pipe-separated list with the key "excluded_processing".
+        """
+        n = self.getValueFromCatchall("excluded_processing")
+        if n:
+            return n.split("|")
+        return None
+
+    # use per-core attributes
+    def use_per_core_attr(self):
+        return self.hasValueInCatchall("per_core_attr")

pandaserver/taskbuffer/SupErrors.py

@@ -0,0 +1,10 @@
+# Supplemental error codes
+# Error values must be less than 1000
+
+
+class SupErrors(object):
+    error_codes = {
+        "SUCCEEDE": 0,
+        "INVALID_BATCH_ID": 1,
+        "WORKER_ALREADY_DONE": 2,
+    }