panda-server-source 0.8.0__py3-none-any.whl

pandaserver/taskbuffer/ParseJobXML.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python
+
+import sys
+import xml.dom.minidom
+from urllib.parse import quote
+
+
+class dom_job:
+    """infiles[inds]=[file1,file2...]
+    outfiles = [file1,file2...]
+    command  - script that will be executed on the grid
+    prepend  - list of (option,value) prepended to output file name
+    forward  - list of (option,value) forwarded to the grid job
+    """
+
+    def __init__(s, domjob=None, primaryds=None, defaultcmd=None, defaultout=[]):
+        """Loads <job></job> from xml file.
+        If primaryds is set, makes sure it is present in job spec"""
+        s.infiles = {}
+        s.outfiles = []
+        s.command = defaultcmd
+        s.prepend = []
+        s.forward = []
+        if not domjob:
+            return
+        # script executed on the grid node for this job
+        if len(domjob.getElementsByTagName("command")) > 0:
+            s.command = dom_parser.text(domjob.getElementsByTagName("command")[0])
+        # input files
+        for inds in domjob.getElementsByTagName("inds"):
+            name = dom_parser.text(inds.getElementsByTagName("name")[0])
+            files = inds.getElementsByTagName("file")
+            if len(files) == 0:
+                continue
+            s.infiles[name] = []
+            for file in files:
+                s.infiles[name].append(dom_parser.text(file))
+        if primaryds and primaryds not in s.infiles.keys():
+            print(f"ERROR: primaryds={primaryds} must be present in each job")
+            sys.exit(0)
+        # output files (also, drop duplicates within this job)
+        outfiles = set(defaultout)
+        [outfiles.add(dom_parser.text(v)) for v in domjob.getElementsByTagName("output")]
+        s.outfiles = list(outfiles)
+        # gearing options
+        for o in domjob.getElementsByTagName("option"):
+            name = o.attributes["name"].value
+            value = dom_parser.text(o)
+            prepend = dom_parser.true(o.attributes["prepend"].value)
+            forward = dom_parser.true(o.attributes["forward"].value)
+            if prepend:
+                s.prepend.append((name, value))
+            if forward:
+                s.forward.append((name, value))
+
+    def to_dom(s):
+        """Converts this job to a dom tree branch"""
+        x = xml.dom.minidom.Document()
+        job = x.createElement("job")
+        for inds in s.infiles.keys():
+            job.appendChild(x.createElement("inds"))
+            job.childNodes[-1].appendChild(x.createElement("name"))
+            job.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(inds))
+            for file in s.infiles[inds]:
+                job.childNodes[-1].appendChild(x.createElement("file"))
+                job.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(file))
+        for outfile in s.outfiles:
+            job.appendChild(x.createElement("output"))
+            job.childNodes[-1].appendChild(x.createTextNode(outfile))
+        if s.command:
+            job.appendChild(x.createElement("command"))
+            job.childNodes[-1].appendChild(x.createTextNode(s.command))
+        for option in s.prepend + list(set(s.prepend + s.forward) - set(s.prepend)):
+            job.appendChild(x.createElement("option"))
+            job.childNodes[-1].setAttribute("name", str(option[0]))
+            if option in s.forward:
+                job.childNodes[-1].setAttribute("forward", "true")
+            else:
+                job.childNodes[-1].setAttribute("forward", "false")
+            if option in s.prepend:
+                job.childNodes[-1].setAttribute("prepend", "true")
+            else:
+                job.childNodes[-1].setAttribute("prepend", "false")
+            job.childNodes[-1].appendChild(x.createTextNode(str(option[1])))
+        return job
+
+    def files_in_DS(s, DS):
+        """Returns a list of files used in a given job in a given dataset"""
+        if DS in s.infiles:
+            return s.infiles[DS]
+        else:
+            return []
+
+    def forward_opts(s):
+        """passable string of forward options"""
+        return " ".join([f"{v[0]}={v[1]}" for v in s.forward])
+
+    def prepend_string(s):
+        """a tag string prepended to output files"""
+        return "_".join([f"{v[0]}{v[1]}" for v in s.prepend])
+
+    def exec_string(s):
+        """exec string for prun.
+        If user requested to run script run.sh (via <command>run.sh</command>), it will return
+        opt1=value1 opt2=value2 opt3=value3 run.sh
+        This way, all options will be set inside run.sh
+        """
+        return f"{s.forward_opts()} {s.command}"
+
+    def exec_string_enc(s):
+        """exec string for prun.
+        If user requested to run script run.sh (via <command>run.sh</command>), it will return
+        opt1=value1 opt2=value2 opt3=value3 run.sh
+        This way, all options will be set inside run.sh
+        """
+        comStr = f"{s.forward_opts()} {s.command}"
+        return quote(comStr)
+
+    def get_outmap_str(s, outMap):
+        """return mapping of original and new filenames"""
+        newMap = {}
+        for oldLFN, fileSpec in outMap.items():
+            newMap[oldLFN] = str(fileSpec.lfn)
+        return str(newMap)
+
+    def outputs_list(s, prepend=False):
+        """python list with finalized output file names"""
+        if prepend and s.prepend_string():
+            return [s.prepend_string() + "." + o for o in s.outfiles]
+        else:
+            return [o for o in s.outfiles]
+
+    def outputs(s, prepend=False):
+        """Comma-separated list of output files accepted by prun"""
+        return ",".join(s.outputs_list(prepend))
+
+
+class dom_parser:
+    def __init__(s, fname=None, xmlStr=None):
+        """creates a dom object out of a text file (if provided)"""
+        s.fname = fname
+        s.dom = None
+        s.title = None
+        s.tag = None
+        s.command = None
+        s.outds = None
+        s.inds = {}
+        s.global_outfiles = []
+        s.jobs = []
+        s.primaryds = None
+        if fname:
+            s.dom = xml.dom.minidom.parse(fname)
+            s.parse()
+            s.check()
+        if xmlStr is not None:
+            s.dom = xml.dom.minidom.parseString(xmlStr)
+            s.parse()
+            s.check()
+
+    @staticmethod
+    def true(v):
+        """define True"""
+        return v in ("1", "true", "True", "TRUE", "yes", "Yes", "YES")
+
+    @staticmethod
+    def text(pnode):
+        """extracts the value stored in the node"""
+        rc = []
+        for node in pnode.childNodes:
+            if node.nodeType == node.TEXT_NODE:
+                rc.append(str(node.data).strip())
+        return "".join(rc)
+
+    def parse(s):
+        """loads submission configuration from an xml file"""
+        try:
+            # general settings
+            if len(s.dom.getElementsByTagName("title")) > 0:
+                s.title = dom_parser.text(s.dom.getElementsByTagName("title")[0])
+            else:
+                s.title = "Default title"
+            if len(s.dom.getElementsByTagName("tag")) > 0:
+                s.tag = dom_parser.text(s.dom.getElementsByTagName("tag")[0])
+            else:
+                s.tag = "default_tag"
+            s.command = None  # can be overridden in subjobs
+            for elm in s.dom.getElementsByTagName("submission")[0].childNodes:
+                if elm.nodeName != "command":
+                    continue
+                s.command = dom_parser.text(elm)
+                break
+            s.global_outfiles = []  # subjobs can append *additional* outputs
+            for elm in s.dom.getElementsByTagName("submission")[0].childNodes:
+                if elm.nodeName != "output":
+                    continue
+                s.global_outfiles.append(dom_parser.text(elm))
+            s.outds = dom_parser.text(s.dom.getElementsByTagName("outds")[0])
+            # declaration of all input datasets
+            primarydss = []
+            for elm in s.dom.getElementsByTagName("submission")[0].childNodes:
+                if elm.nodeName != "inds":
+                    continue
+                if "primary" in elm.attributes.keys() and dom_parser.true(elm.attributes["primary"].value):
+                    primary = True
+                else:
+                    primary = False
+                stream = dom_parser.text(elm.getElementsByTagName("stream")[0])
+                name = dom_parser.text(elm.getElementsByTagName("name")[0])
+                s.inds[name] = stream
+                if primary:
+                    primarydss.append(name)
+            # see if one of the input datasets was explicitly labeled as inDS
+            if len(primarydss) == 1:
+                s.primaryds = primarydss[0]
+            else:
+                s.primaryds = None
+            for job in s.dom.getElementsByTagName("job"):
+                s.jobs.append(dom_job(job, primaryds=s.primaryds, defaultcmd=s.command, defaultout=s.global_outfiles))
+        except Exception:
+            print("ERROR: failed to parse" + " " + s.fname)
+            raise
+
+    def to_dom(s):
+        """Converts this submission to a dom tree branch"""
+        x = xml.dom.minidom.Document()
+        submission = x.createElement("submission")
+        if s.title:
+            submission.appendChild(x.createElement("title"))
+            submission.childNodes[-1].appendChild(x.createTextNode(s.title))
+        if s.tag:
+            submission.appendChild(x.createElement("tag"))
+            submission.childNodes[-1].appendChild(x.createTextNode(s.tag))
+        for name, stream in s.inds.items():
+            submission.appendChild(x.createElement("inds"))
+            if name == s.primaryds:
+                submission.childNodes[-1].setAttribute("primary", "true")
+            else:
+                submission.childNodes[-1].setAttribute("primary", "false")
+            submission.childNodes[-1].appendChild(x.createElement("stream"))
+            submission.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(stream))
+            submission.childNodes[-1].appendChild(x.createElement("name"))
+            submission.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(name))
+        if s.command:
+            submission.appendChild(x.createElement("command"))
+            submission.childNodes[-1].appendChild(x.createTextNode(s.command))
+        for outfile in s.global_outfiles:
+            submission.appendChild(x.createElement("output"))
+            submission.childNodes[-1].appendChild(x.createTextNode(outfile))
+        submission.appendChild(x.createElement("outds"))
+        submission.childNodes[-1].appendChild(x.createTextNode(s.outds))
+        for job in s.jobs:
+            submission.appendChild(job.to_dom())
+        return submission
+
+    def check(s):
+        """checks that all output files have unique qualifiers"""
+        quals = []
+        for j in s.jobs:
+            quals += j.outputs_list(True)
+        if len(list(set(quals))) != len(quals):
+            print("ERROR: found non-unique output file names across the jobs")
+            print("(you likely need to review xml options with prepend=true)")
+            sys.exit(0)
+
+    def input_datasets(s):
+        """returns a list of all used input datasets"""
+        DSs = set()
+        for j in s.jobs:
+            for ds in j.infiles.keys():
+                DSs.add(ds)
+        return list(DSs)
+
+    def inDS(s):
+        """chooses a dataset we'll call inDS; others will become secondaryDS"""
+        # user manually labeled one of datasets as primary, so make it inDS:
+        if s.primaryds:
+            return s.primaryds
+        # OR: choose inDS dataset randomly
+        else:
+            return s.input_datasets()[0]
+
+    def secondaryDSs(s):
+        """returns all secondaryDSs. This excludes inDS, unless inDS is managed by prun"""
+        return [d for d in s.input_datasets() if d != s.inDS()]
+
+    def writeInputToTxt(s):
+        """Prepares prun option --writeInputToTxt
+        comma-separated list of STREAM:STREAM.files.dat
+        """
+        out = []
+        DSs = s.secondaryDSs()
+        for i, DS in enumerate(DSs):
+            if DS in s.inds:
+                stream = s.inds[DS]
+            else:
+                stream = "IN%d" % (i + 1,)
+            out.append(f"{stream}:{stream}.files.dat")
+        out.append("IN:IN.files.dat")
+        return ",".join(out)
+
+    def files_in_DS(s, DS, regex=False):
+        """Returns a list of all files from a given dataset
+        that will be used in at least one job in this submission
+        If regex==True, the list is converted to a regex string
+        """
+        assert DS in s.input_datasets(), f"ERROR: dataset {DS} was not requested in the xml file"
+        files = []
+        for j in s.jobs:
+            if DS in j.infiles.keys():
+                files += j.infiles[DS]
+        if regex:
+            return "|".join(sorted(list(set(files))))
+        else:
+            return sorted(list(set(files)))
+
+    def nJobs(s):
+        return len(s.jobs)
+
+    def dump(s, verbose=True):
+        """prints a summary of this submission"""
+
+        def P(key, value=""):
+            if value == "":
+                print(key)
+            else:
+                print((key + ":").ljust(14) + " " + value)
+
+        P("XML FILE LOADED", s.fname)
+        P("Title", s.title)
+        P("Command", s.command)
+        P("InDS", s.inDS())
+        P("Output DS", s.outds)
+        P("njobs", s.nJobs())
+        if verbose:
+            for i, job in enumerate(s.jobs):
+                P("===============> JOB%d" % i)
+                P("command", job.exec_string())
+                P("outfiles", job.outputs())
+                P("INPUTS:")
+                j = 0
+                for dsname, files in job.infiles.items():
+                    P("  Dataset%d" % j, dsname)
+                    for k, fname in enumerate(files):
+                        P("     File%d" % k, fname)
+                    j += 1
+
+
+if __name__ == "__main__":
+    p = dom_parser("./job.xml")
+    p.dump()

pandaserver/taskbuffer/PickleFileSpec.py

@@ -0,0 +1,18 @@
+# Derived FileSpec to preserve changed attributes in pickle as it is impossible to change FileSpec.reserveChangedState
+# consistently with all clients at the same time
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+
+
+class PickleFileSpec(FileSpec):
+    def __init__(self):
+        FileSpec.__init__(self)
+        object.__setattr__(self, "_reserveChangedState", True)
+
+    def update(self, spec):
+        spec._reserveChangedState = True
+        self.__setstate__(spec.__getstate__())
+
+    def __setstate__(self, state):
+        object.__setattr__(self, "_reserveChangedState", True)
+        FileSpec.__setstate__(self, state)

pandaserver/taskbuffer/PickleJobSpec.py

@@ -0,0 +1,25 @@
+# Derived JobSpec to preserve changed attributes in pickle as it is impossible to change JobSpec.reserveChangedState
+# consistently with all clients at the same time
+
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.taskbuffer.PickleFileSpec import PickleFileSpec
+
+
+class PickleJobSpec(JobSpec):
+    def __init__(self):
+        JobSpec.__init__(self)
+        object.__setattr__(self, "_reserveChangedState", True)
+
+    def update(self, spec):
+        spec._reserveChangedState = True
+        self.__setstate__(spec.__getstate__())
+        p_file_list = []
+        for file in self.Files:
+            p_file = PickleFileSpec()
+            p_file.update(file)
+            p_file_list.append(p_file)
+        object.__setattr__(self, "Files", p_file_list)
+
+    def __setstate__(self, state):
+        object.__setattr__(self, "_reserveChangedState", True)
+        JobSpec.__setstate__(self, state)

pandaserver/taskbuffer/PrioUtil.py

@@ -0,0 +1,40 @@
+import json
+import sys
+
+# special permission
+PERMISSION_KEY = "k"
+PERMISSION_PROXY = "p"
+PERMISSION_TOKEN_KEY = "t"
+PERMISSION_SUPER_USER = "s"
+PERMISSION_SUPER_GROUP = "g"
+
+
+# convert UTF-8 to ASCII in json dumps
+# This is needed for Python 2, but not for Python 3
+def unicodeConvert(input):
+    if isinstance(input, dict):
+        retMap = {}
+        for tmpKey in input:
+            tmpVal = input[tmpKey]
+            retMap[unicodeConvert(tmpKey)] = unicodeConvert(tmpVal)
+        return retMap
+    elif isinstance(input, list):
+        retList = []
+        for tmpItem in input:
+            retList.append(unicodeConvert(tmpItem))
+        return retList
+    elif isinstance(input, str) and sys.version_info[0] == 2:
+        return input.encode("utf-8")
+    return input
+
+
+# decode
+# This is needed for Python 2, but not for Python 3
+def decodeJSON(inputStr):
+    return json.loads(inputStr, object_hook=unicodeConvert)
+
+
+# calculate priority for user jobs
+def calculatePriority(priorityOffset, serNum, weight):
+    priority = int(1000 + priorityOffset - (serNum / 5) - int(100 * weight))
+    return priority

pandaserver/taskbuffer/ProcessGroups.py

@@ -0,0 +1,111 @@
+from pandaserver.taskbuffer import JobUtils
+
+processGroups = [
+    ("others", []),
+    ("evgen", ["evgen"]),
+    ("simul", ["simul"]),
+    ("reprocessing", ["reprocessing"]),
+    ("test", ["prod_test", "validation"] + JobUtils.list_ptest_prod_sources),
+    ("mcore", ["mcore"]),
+    ("group", ["group"]),
+    ("deriv", ["deriv"]),
+    ("pile", ["pile"]),
+    ("merge", ["merge"]),
+]
+
+# ('evgensimul',   ['evgen','simul']),
+
+# maximum number of debug jobs per user
+maxDebugJobs = 3
+
+# maximum number of debug jobs for prod role
+maxDebugProdJobs = 30
+
+# maximum number of debug jobs for working group
+maxDebugWgJobs = 10
+
+# extension level for GP
+extensionLevel_1 = 1
+
+
+# get corresponding group
+def getProcessGroup(valGroup):
+    tmpGroup = None
+    for tmpKey, tmpList in processGroups:
+        # set default
+        if tmpGroup is None:
+            tmpGroup = tmpKey
+            continue
+        if valGroup in tmpList:
+            tmpGroup = tmpKey
+            break
+    # return
+    return tmpGroup
+
+
+# convert cloud and processingType for extended PG
+def converCPTforEPG(cloud, processingType, coreCount, workingGroup=None):
+    if coreCount in [0, 1, None]:
+        # use group queue for GP jobs
+        if workingGroup is not None and workingGroup.startswith("GP_"):
+            return cloud, "group"
+        return cloud, processingType
+    else:
+        # use MCORE queue for MPC jobs in all clouds
+        return cloud, "mcore"
+
+
+# count the number of jobs per group
+def countJobsPerGroup(valMap):
+    ret = {}
+    # loop over all clouds
+    for cloud in valMap:
+        cloudVal = valMap[cloud]
+        # add cloud
+        ret.setdefault(cloud, {})
+        # loop over all sites
+        for site in cloudVal:
+            siteVal = cloudVal[site]
+            # add site
+            ret[cloud].setdefault(site, {})
+            # loop over all types
+            for pType in siteVal:
+                typeVal = siteVal[pType]
+                # get process group
+                tmpGroup = getProcessGroup(pType)
+                # add group
+                ret[cloud][site].setdefault(tmpGroup, {})
+                # loop over all status
+                for jobStatus in typeVal:
+                    statVal = typeVal[jobStatus]
+                    ret[cloud][site][tmpGroup].setdefault(jobStatus, 0)
+                    # add
+                    ret[cloud][site][tmpGroup][jobStatus] += statVal
+    # return
+    return ret
+
+
+# count the number of jobs per group for analysis
+def countJobsPerGroupForAnal(valMap):
+    ret = {}
+    # loop over all sites
+    for site in valMap:
+        siteVal = valMap[site]
+        # add site
+        ret.setdefault(site, {})
+        # loop over all types
+        for pType in siteVal:
+            typeVal = siteVal[pType]
+            # get process group
+            tmpGroup = getProcessGroup(pType)
+            # add group
+            if tmpGroup not in ret[site]:
+                ret[site][tmpGroup] = {}
+            # loop over all status
+            for jobStatus in typeVal:
+                statVal = typeVal[jobStatus]
+                ret[site][tmpGroup].setdefault(jobStatus, 0)
+                # add
+                ret[site][tmpGroup][jobStatus] += statVal
+    # return
+    return ret