panda-server-source 0.8.0__py3-none-any.whl

pandaserver/taskbuffer/JobUtils.py

@@ -0,0 +1,232 @@
+import json
+import re
+
+from pandaserver.srvcore.CoreUtils import NonJsonObjectEncoder, as_python_object
+from pandaserver.taskbuffer.JobSpec import JobSpec
+
+# list of prod source label for pilot tests
+list_ptest_prod_sources = ["ptest", "rc_test", "rc_test2", "rc_alrb"]
+
+# mapping with prodsourcelabels that belong to analysis and production
+analy_sources = ["user", "panda"]
+prod_sources = ["managed", "prod_test"]
+neutral_sources = ["install"] + list_ptest_prod_sources
+
+ANALY_PS = "user"
+PROD_PS = "managed"
+
+ANALY_TASKTYPE = "anal"
+PROD_TASKTYPE = "prod"
+
+MEMORY_COMPENSATION = 0.9
+
+job_labels = [ANALY_PS, PROD_PS]
+
+# priority of tasks to jumbo over others
+priorityTasksToJumpOver = 1500
+
+
+def translate_prodsourcelabel_to_jobtype(queue_type, prodsourcelabel):
+    if prodsourcelabel in analy_sources:
+        return ANALY_PS
+
+    if prodsourcelabel in prod_sources:
+        return PROD_PS
+
+    if prodsourcelabel in neutral_sources:
+        if queue_type == "unified" or queue_type == "production":
+            return PROD_PS
+        if queue_type == "analysis":
+            return ANALY_PS
+
+    # currently unmapped
+    return prodsourcelabel
+
+
+def translate_tasktype_to_jobtype(task_type):
+    # any unrecognized tasktype will be defaulted to production
+    if task_type == ANALY_TASKTYPE:
+        return ANALY_PS
+    else:
+        return PROD_PS
+
+
+# get core count
+def getCoreCount(actualCoreCount, defCoreCount, jobMetrics):
+    coreCount = 1
+    try:
+        if actualCoreCount is not None:
+            coreCount = actualCoreCount
+        else:
+            tmpMatch = None
+            if jobMetrics is not None:
+                # extract coreCount
+                tmpMatch = re.search("coreCount=(\d+)", jobMetrics)
+            if tmpMatch is not None:
+                coreCount = int(tmpMatch.group(1))
+            else:
+                # use jobdef
+                if defCoreCount not in [None, 0]:
+                    coreCount = defCoreCount
+    except Exception:
+        pass
+    return coreCount
+
+
+# get HS06sec
+def getHS06sec(startTime, endTime, corePower, coreCount, baseWalltime=0, cpuEfficiency=100):
+    try:
+        # no scaling
+        if cpuEfficiency == 0:
+            return 0
+        # get execution time
+        tmpTimeDelta = endTime - startTime
+        tmpVal = tmpTimeDelta.seconds + tmpTimeDelta.days * 24 * 3600
+        if tmpVal <= baseWalltime:
+            return 0
+        hs06sec = float(tmpVal - baseWalltime) * corePower * coreCount * float(cpuEfficiency) / 100.0
+        return hs06sec
+    except Exception:
+        return None
+
+
+def get_job_co2(start_time, end_time, core_count, energy_emissions, watts_per_core):
+    energy_emissions_by_ts = {}
+    for entry in energy_emissions:
+        aux_timestamp, region, value = entry
+        energy_emissions_by_ts[aux_timestamp] = {"value": value}
+
+    try:
+        timestamps = sorted([entry[0] for entry in energy_emissions])
+
+        started = False
+        ended = False
+        i = 0
+
+        g_co2_job = 0
+
+        for timestamp in timestamps:
+            try:
+                if start_time < timestamps[i + 1] and not started:
+                    started = True
+            except IndexError:
+                pass
+
+            if end_time < timestamp and not ended:
+                ended = True
+
+            if started and not ended or i == len(timestamps) - 1:
+                bottom = max(start_time, timestamp)
+                try:
+                    top = min(end_time, timestamps[i + 1])
+                except IndexError:
+                    top = end_time
+
+                g_co2_perkWh = energy_emissions_by_ts[timestamp]["value"]
+
+                duration = max((top - bottom).total_seconds(), 0)
+                g_co2_job = g_co2_job + (duration * g_co2_perkWh * core_count * watts_per_core / 3600 / 1000)
+
+            if ended:
+                break
+
+            i = i + 1
+
+        return g_co2_job
+
+    except Exception:
+        return None
+
+
+# parse string for number of standby jobs
+def parseNumStandby(catchall):
+    retMap = {}
+    if catchall is not None:
+        for tmpItem in catchall.split(","):
+            tmpMatch = re.search("^nStandby=(.+)", tmpItem)
+            if tmpMatch is None:
+                continue
+            for tmpSubStr in tmpMatch.group(1).split("|"):
+                if len(tmpSubStr.split(":")) != 3:
+                    continue
+                sw_id, resource_type, num = tmpSubStr.split(":")
+                try:
+                    sw_id = int(sw_id)
+                except Exception:
+                    pass
+                if sw_id not in retMap:
+                    retMap[sw_id] = {}
+                if num == "":
+                    num = 0
+                else:
+                    num = int(num)
+                retMap[sw_id][resource_type] = num
+            break
+    return retMap
+
+
+# compensate memory count to prevent jobs with ramCount close to the HIMEM border from going to HIMEM PQs
+def compensate_ram_count(ram_count):
+    if ram_count in ("NULL", None):
+        return None
+    ram_count = int(ram_count * MEMORY_COMPENSATION)
+    return ram_count
+
+
+# undo the memory count compensation
+def decompensate_ram_count(ram_count):
+    if ram_count in ("NULL", None):
+        return None
+    ram_count = int(ram_count / MEMORY_COMPENSATION)
+    return ram_count
+
+
+# dump jobs to serialized json
+def dump_jobs_json(jobs):
+    state_objects = []
+    for job_spec in jobs:
+        state_objects.append(job_spec.dump_to_json_serializable())
+    return json.dumps(state_objects, cls=NonJsonObjectEncoder)
+
+
+# load serialized json to jobs
+def load_jobs_json(state):
+    state_objects = json.loads(state, object_hook=as_python_object)
+    jobs = []
+    for job_state in state_objects:
+        job_spec = JobSpec()
+        job_spec.load_from_json_serializable(job_state)
+        jobs.append(job_spec)
+    return jobs
+
+
+# get resource type for a job
+def get_resource_type_job(resource_map: list, job_spec: JobSpec) -> str:
+    """
+    Get the resource type for a job based on the job's resource type and the list of resource types.
+    :param resource_map: The list of resource types.
+    :param job_spec: The job.
+    :return: The resource type.
+    """
+    for resource_spec in resource_map:
+        if resource_spec.match_job(job_spec):
+            return resource_spec.resource_name
+    return "Undefined"
+
+
+# get min ram count for job
+def getJobMinRamCount(taskSpec, inputChunk, siteSpec, coreCount):
+    minRamCount = inputChunk.getMaxRamCount()
+    if inputChunk.isMerging:
+        minRamUnit = "MB"
+    else:
+        minRamUnit = taskSpec.ramUnit
+        if minRamUnit in [None, "", "NULL"]:
+            minRamUnit = "MB"
+        if taskSpec.ramPerCore():
+            minRamCount *= coreCount
+            minRamCount += taskSpec.baseRamCount
+            minRamUnit = re.sub("PerCore.*$", "", minRamUnit)
+    # round up with chunks
+    minRamCount = compensate_ram_count(minRamCount)
+    return minRamCount, minRamUnit

pandaserver/taskbuffer/NucleusSpec.py

@@ -0,0 +1,116 @@
+"""
+nucleus specification
+
+"""
+
+import re
+
+
+class NucleusSpec(object):
+    # constructor
+    def __init__(self, name):
+        self.name = name
+        self.allPandaSites = []
+        self.allDdmEndPoints = {}
+        self.all_ddm_endpoints_in = {}
+        self.state = None
+        self.bareNucleus = None
+        self.secondaryNucleus = None
+        self.nucleus = True
+        self.default_ddm_endpoint_out = None
+
+    # add
+    def add(self, siteName, ddmSpecDict, ddmSpecDictForInput=None):
+        if siteName not in self.allPandaSites:
+            self.allPandaSites.append(siteName)
+            # add local endpoints
+            for scope in ddmSpecDict:
+                ddmSpec = ddmSpecDict[scope]
+                for localEndPoint in ddmSpec.getLocalEndPoints():
+                    if localEndPoint not in self.allDdmEndPoints:
+                        self.allDdmEndPoints[localEndPoint] = ddmSpec.getEndPoint(localEndPoint)
+            if ddmSpecDictForInput is not None:
+                # add endpoints
+                for scope in ddmSpecDictForInput:
+                    ddmSpec = ddmSpecDictForInput[scope]
+                    for localEndPoint in ddmSpec.getAllEndPoints():
+                        if localEndPoint not in self.all_ddm_endpoints_in:
+                            self.all_ddm_endpoints_in[localEndPoint] = ddmSpec.getEndPoint(localEndPoint)
+
+    # check if associated panda site
+    def isAssociatedPandaSite(self, siteName):
+        return siteName in self.allPandaSites
+
+    # check if associated DDM endpoint
+    def isAssociatedEndpoint(self, endPoint):
+        return endPoint in self.allDdmEndPoints
+
+    # check if associated endpoint for input
+    def is_associated_for_input(self, endpoint):
+        return endpoint in self.all_ddm_endpoints_in
+
+    # get associated DDM endpoint
+    def getEndpoint(self, endpoint):
+        try:
+            if endpoint in self.allDdmEndPoints:
+                return self.allDdmEndPoints[endpoint]
+            return self.all_ddm_endpoints_in[endpoint]
+        except Exception:
+            return None
+
+    def getAssociatedEndpoint(self, pattern):
+        pattern = pattern.split("/")[-1]
+        if pattern.startswith("dst:"):
+            pattern = pattern.split(":")[-1]
+
+        if pattern in self.allDdmEndPoints:
+            return self.allDdmEndPoints[pattern]
+
+        for endpoint_name in self.allDdmEndPoints:
+            if re.search(pattern, endpoint_name) is not None:
+                return self.allDdmEndPoints[endpoint_name]
+
+            pattern_without_vo = re.sub("ATLAS", "", pattern)
+
+            if self.allDdmEndPoints[endpoint_name]["type"] == pattern_without_vo:
+                return self.allDdmEndPoints[endpoint_name]
+
+        return None
+
+    # get one panda site
+    def getOnePandaSite(self):
+        if len(self.allPandaSites) > 0:
+            return self.allPandaSites[0]
+        return None
+
+    # set bare nucleus mode
+    def set_bare_nucleus_mode(self, mode):
+        self.bareNucleus = mode
+
+    # get bare nucleus mode
+    def get_bare_nucleus_mode(self):
+        return self.bareNucleus
+
+    # set secondary nucleus
+    def set_secondary_nucleus(self, nucleus):
+        self.secondaryNucleus = nucleus
+
+    # get secondary nucleus
+    def get_secondary_nucleus(self):
+        return self.secondaryNucleus
+
+    # set satellite
+    def set_satellite(self):
+        self.nucleus = False
+
+    # check if nucleus
+    def is_nucleus(self):
+        return self.nucleus
+
+    # set default endpoint for output
+    def set_default_endpoint_out(self, rse):
+        self.default_ddm_endpoint_out = rse
+
+    # get default endpoint for output
+    def get_default_endpoint_out(self):
+        return self.getEndpoint(self.default_ddm_endpoint_out)

pandaserver/taskbuffer/OraDBProxy.py

@@ -0,0 +1,174 @@
+"""
+proxy for database connection
+
+"""
+
+import atexit
+import logging
+import warnings
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.db_proxy_mods import (
+    data_carousel_module,
+    entity_module,
+    job_complex_module,
+    job_standalone_module,
+    metrics_module,
+    misc_standalone_module,
+    task_complex_module,
+    task_event_module,
+    task_standalone_module,
+    task_utils_module,
+    worker_module,
+    workflow_module,
+)
+from pandaserver.taskbuffer.WrappedCursor import WrappedCursor
+
+try:
+    import idds.common.constants
+    import idds.common.utils
+    from idds.client.client import Client as iDDS_Client
+except ImportError:
+    pass
+
+if panda_config.backend == "oracle":
+    import oracledb
+
+    from . import wrapped_oracle_conn
+
+    oracledb.init_oracle_client()
+
+elif panda_config.backend == "postgres":
+    import psycopg2 as psycopg
+
+    from . import WrappedPostgresConn
+
+else:
+    import MySQLdb
+
+warnings.filterwarnings("ignore")
+
+# logger
+_logger = PandaLogger().getLogger("DBProxy")
+_loggerFiltered = PandaLogger().getLogger("DBProxyFiltered")
+
+# add handlers
+for handler in _loggerFiltered.handlers:
+    handler.setLevel(logging.INFO)
+    _logger.addHandler(handler)
+    _loggerFiltered.removeHandler(handler)
+
+
+# proxy
+class DBProxy(
+    entity_module.EntityModule,
+    metrics_module.MetricsModule,
+    worker_module.WorkerModule,
+    data_carousel_module.DataCarouselModule,
+    task_event_module.TaskEventModule,
+    job_complex_module.JobComplexModule,
+    job_standalone_module.JobStandaloneModule,
+    misc_standalone_module.MiscStandaloneModule,
+    task_complex_module.TaskComplexModule,
+    task_standalone_module.TaskStandaloneModule,
+    task_utils_module.TaskUtilsModule,
+    workflow_module.WorkflowModule,
+):
+    # constructor
+    def __init__(self, useOtherError=False):
+        # init modules
+        super().__init__(_logger)
+        # connection object
+        self.conn = None
+        # cursor object
+        self.cur = None
+
+        # use special error codes for reconnection in querySQL
+        self.useOtherError = useOtherError
+
+        # set composite modules. use self to share a single database connection
+        self.add_composite_module("entity", self)
+        self.add_composite_module("metrics", self)
+        self.add_composite_module("worker", self)
+        self.add_composite_module("task_event", self)
+        self.add_composite_module("task_utils", self)
+        self.add_composite_module("job_complex", self)
+
+    # connect to DB
+    def connect(
+        self,
+        dbhost=panda_config.dbhost,
+        dbpasswd=panda_config.dbpasswd,
+        dbuser=panda_config.dbuser,
+        dbname=panda_config.dbname,
+        dbtimeout=panda_config.dbtimeout,
+        reconnect=False,
+        dbport=panda_config.dbport,
+    ):
+        _logger.debug(f"connect : re={reconnect}")
+        # keep parameters for reconnect
+        if not reconnect:
+            self.dbhost = dbhost
+            self.dbpasswd = dbpasswd
+            self.dbuser = dbuser
+            self.dbname = dbname
+            self.dbtimeout = dbtimeout
+            self.dbport = dbport
+        # close old connection
+        if reconnect:
+            _logger.debug("closing old connection")
+            try:
+                self.conn.close()
+            except Exception:
+                _logger.debug("failed to close old connection")
+        # connect
+        try:
+            if self.backend == "oracle":
+                conn = oracledb.connect(dsn=self.dbhost, user=self.dbuser, password=self.dbpasswd)
+
+                def OutputTypeHandler(cursor, name, defaultType, size, precision, scale):
+                    if defaultType == oracledb.CLOB:
+                        return cursor.var(oracledb.LONG_STRING, arraysize=cursor.arraysize)
+
+                conn.outputtypehandler = OutputTypeHandler
+                self.conn = wrapped_oracle_conn.WrappedOracleConn(conn)
+
+            elif self.backend == "postgres":
+                dsn = {"dbname": self.dbname, "user": self.dbuser, "keepalives_idle": 30, "keepalives_interval": 30, "keepalives": 1}
+                if self.dbpasswd:
+                    dsn["password"] = self.dbpasswd
+                if self.dbhost:
+                    dsn["host"] = self.dbhost
+                if self.dbport:
+                    dsn["port"] = self.dbport
+                conn = psycopg.connect(**dsn)
+                self.conn = WrappedPostgresConn.WrappedPostgresConn(conn)
+            else:
+                self.conn = MySQLdb.connect(
+                    host=self.dbhost,
+                    db=self.dbname,
+                    port=self.dbport,
+                    connect_timeout=self.dbtimeout,
+                    user=self.dbuser,
+                    passwd=self.dbpasswd,
+                    charset="utf8",
+                )
+            self.cur = WrappedCursor(self.conn)
+            try:
+                # use SQL dumper
+                if panda_config.dump_sql:
+                    from pandaserver.taskbuffer.SQLDumper import SQLDumper
+
+                    self.cur = SQLDumper(self.cur)
+            except Exception:
+                pass
+            self.hostname = self.cur.initialize()
+            if not reconnect:
+                atexit.register(self.close_connection)
+            _logger.debug(f"connect : re={reconnect} ready")
+            return True
+        except Exception as e:
+            _logger.error(f"connect : {str(e)}")
+            return False

pandaserver/taskbuffer/PanDAMsgProcessor.py

@@ -0,0 +1,8 @@
+from pandacommon.pandalogger import logger_utils
+from pandacommon.pandamsgbkr import msg_processor
+from pandaserver.config import panda_config
+
+
+# Main message processing agent
+class MsgProcAgent(msg_processor.MsgProcAgentBase):
+    pass

pandaserver/taskbuffer/PandaDBSchemaInfo.py

@@ -0,0 +1,18 @@
+"""
+Minimum schema version required for the panda-server to work properly.
+Please always keep this version number up to date.
+"""
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+# logger
+_logger = PandaLogger().getLogger("DBSchema")
+
+
+class PandaDBSchemaInfo:
+    schema_version = None
+
+    def method(self):
+        schema_version = "0.1.1"
+        _logger.debug(f"PanDA schema version required for Server is : {schema_version}")
+        return schema_version