panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jediorder/JobSplitter.py

@@ -0,0 +1,308 @@
+import copy
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore import Interaction
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandaserver.srvcore import CoreUtils
+from pandaserver.taskbuffer.InputChunk import InputChunk
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# class to split job
+class JobSplitter:
+    # constructor
+    def __init__(self):
+        self.sizeGradientsPerInSizeForMerge = 1.2
+        self.interceptsMerginForMerge = 500 * 1024 * 1024
+
+    # split
+    def doSplit(self, taskSpec, inputChunk, siteMapper, allow_chunk_size_limit=False):
+        # return for failure
+        retFatal = self.SC_FATAL, []
+        retTmpError = self.SC_FAILED, []
+        # make logger
+        tmpLog = MsgWrapper(logger, f"< jediTaskID={taskSpec.jediTaskID} datasetID={inputChunk.masterIndexName} >")
+        tmpLog.debug(f"--- start chunk_size_limit={allow_chunk_size_limit}")
+        if not inputChunk.isMerging:
+            # set maxNumFiles using taskSpec if specified
+            maxNumFiles = taskSpec.getMaxNumFilesPerJob()
+            # set fsize gradients using taskSpec
+            sizeGradients = taskSpec.getOutDiskSize()
+            # set fsize intercepts using taskSpec
+            sizeIntercepts = taskSpec.getWorkDiskSize()
+            # walltime
+            if not taskSpec.useHS06():
+                walltimeGradient = taskSpec.walltime
+            else:
+                walltimeGradient = taskSpec.getCpuTime()
+            # number of events per job if defined
+            nEventsPerJob = taskSpec.getNumEventsPerJob()
+            # number of files per job if defined
+            nFilesPerJob = taskSpec.getNumFilesPerJob()
+            if nFilesPerJob is None and nEventsPerJob is None and inputChunk.useScout() and not taskSpec.useLoadXML() and not taskSpec.respectSplitRule():
+                nFilesPerJob = 1
+            # grouping with boundaryID
+            useBoundary = taskSpec.useGroupWithBoundaryID()
+            # fsize intercepts per input size
+            sizeGradientsPerInSize = None
+            # set max primay output size to avoid producing huge unmerged files
+            maxOutSize = 10 * 1024 * 1024 * 1024
+            # max size per job
+            maxSizePerJob = taskSpec.getMaxSizePerJob()
+            if maxSizePerJob is not None:
+                maxSizePerJob += InputChunk.defaultOutputSize
+            # multiplicity of jobs
+            if taskSpec.useJobCloning():
+                multiplicity = 1
+            else:
+                multiplicity = taskSpec.getNumEventServiceConsumer()
+            # split with fields
+            if taskSpec.getFieldNumToLFN() is not None and taskSpec.useFileAsSourceLFN():
+                splitByFields = taskSpec.getFieldNumToLFN()
+            else:
+                splitByFields = None
+        else:
+            # set parameters for merging
+            maxNumFiles = taskSpec.getMaxNumFilesPerMergeJob()
+            sizeGradients = 0
+            walltimeGradient = 0
+            nFilesPerJob = taskSpec.getNumFilesPerMergeJob()
+            nEventsPerJob = taskSpec.getNumEventsPerMergeJob()
+            maxSizePerJob = None
+            useBoundary = {"inSplit": 3}
+            multiplicity = None
+            # gradients per input size is 1 + margin
+            sizeGradientsPerInSize = self.sizeGradientsPerInSizeForMerge
+            # intercepts for libDS
+            sizeIntercepts = taskSpec.getWorkDiskSize()
+            # mergein of 500MB
+            interceptsMergin = self.interceptsMerginForMerge
+            if sizeIntercepts < interceptsMergin:
+                sizeIntercepts = interceptsMergin
+            maxOutSize = taskSpec.getMaxSizePerMergeJob()
+            if maxOutSize is None:
+                # max output size is 5GB for merging by default
+                maxOutSize = 5 * 1024 * 1024 * 1024
+            # split with fields
+            if taskSpec.getFieldNumToLFN() is not None and taskSpec.useFileAsSourceLFN():
+                splitByFields = list(range(4 + 1, 4 + 1 + len(taskSpec.getFieldNumToLFN())))
+            else:
+                splitByFields = None
+        # LB
+        respectLB = taskSpec.respectLumiblock()
+        # no split for on-site merging
+        if taskSpec.on_site_merging():
+            no_split = True
+        else:
+            no_split = False
+        # dump
+        tmpLog.debug(f"maxNumFiles={maxNumFiles} sizeGradients={sizeGradients} sizeIntercepts={sizeIntercepts} useBoundary={useBoundary}")
+        tmpLog.debug(f"walltimeGradient={walltimeGradient} nFilesPerJob={nFilesPerJob} nEventsPerJob={nEventsPerJob}")
+        tmpLog.debug(f"useScout={inputChunk.useScout()} isMerging={inputChunk.isMerging}")
+        tmpLog.debug(f"sizeGradientsPerInSize={sizeGradientsPerInSize} maxOutSize={maxOutSize} respectLB={respectLB}")
+        tmpLog.debug(
+            f"multiplicity={multiplicity} splitByFields={str(splitByFields)} "
+            f"nFiles={inputChunk.getNumFilesInMaster()} no_split={no_split} "
+            f"maxEventsPerJob={taskSpec.get_max_events_per_job()}"
+        )
+        tmpLog.debug("--- main loop")
+        # split
+        returnList = []
+        subChunks = []
+        iSubChunks = 0
+        if inputChunk.useScout() and not inputChunk.isMerging:
+            default_nSubChunks = 2
+        elif taskSpec.is_hpo_workflow():
+            default_nSubChunks = 2
+        else:
+            default_nSubChunks = 25
+        subChunk = None
+        nSubChunks = default_nSubChunks
+        strict_chunkSize = False
+        tmp_ng_list = []
+        change_site_for_dist_dataset = False
+        while True:
+            # change site
+            if iSubChunks % nSubChunks == 0 or subChunk == [] or change_site_for_dist_dataset:
+                change_site_for_dist_dataset = False
+                # append to return map
+                if subChunks != []:
+                    # get site names for parallel execution
+                    if taskSpec.getNumSitesPerJob() > 1 and not inputChunk.isMerging and inputChunk.useJumbo != "fake":
+                        siteName = inputChunk.getParallelSites(taskSpec.getNumSitesPerJob(), nSubChunks, [siteName])
+                    returnList.append(
+                        {
+                            "siteName": siteName,
+                            "subChunks": subChunks,
+                            "siteCandidate": siteCandidate,
+                        }
+                    )
+                    try:
+                        gshare = taskSpec.gshare.replace(" ", "_")
+                    except Exception:
+                        gshare = None
+                    tmpLog.info(f"split to nJobs={len(subChunks)} at site={siteName} gshare={gshare}")
+                    # checkpoint
+                    inputChunk.checkpoint_file_usage()
+                    # reset
+                    subChunks = []
+                # skip PQs with chunk size limit
+                ngList = copy.copy(tmp_ng_list)
+                if not allow_chunk_size_limit:
+                    for siteName in inputChunk.get_candidate_names():
+                        siteSpec = siteMapper.getSite(siteName)
+                        if siteSpec.get_job_chunk_size() is not None:
+                            ngList.append(siteName)
+                # new candidate
+                siteCandidate, getCandidateMsg = inputChunk.getOneSiteCandidate(nSubChunks, ngSites=ngList, get_msg=True)
+                if siteCandidate is None:
+                    tmpLog.debug(f"no candidate: {getCandidateMsg}")
+                    break
+                tmp_ng_list = []
+                siteName = siteCandidate.siteName
+                siteSpec = siteMapper.getSite(siteName)
+                # set chunk size
+                nSubChunks = siteSpec.get_job_chunk_size()
+                if nSubChunks is None:
+                    nSubChunks = default_nSubChunks
+                    strict_chunkSize = False
+                else:
+                    strict_chunkSize = True
+                # directIO
+                if not CoreUtils.use_direct_io_for_job(taskSpec, siteSpec, inputChunk):
+                    useDirectIO = False
+                else:
+                    useDirectIO = True
+                # get maxSize if it is set in taskSpec
+                maxSize = maxSizePerJob
+                if maxSize is None:
+                    # use maxwdir as the default maxSize
+                    if not useDirectIO:
+                        maxSize = siteCandidate.get_overridden_attribute("maxwdir")
+                        if maxSize is None:
+                            maxSize = siteSpec.maxwdir
+                        if maxSize:
+                            maxSize *= 1024 * 1024
+                    elif nEventsPerJob is not None or nFilesPerJob is not None:
+                        maxSize = None
+                    else:
+                        maxSize = siteCandidate.get_overridden_attribute("maxwdir")
+                        if maxSize is None:
+                            maxSize = siteSpec.maxwdir
+                        if inputChunk.useScout():
+                            maxSize = max(inputChunk.maxInputSizeScouts, maxSize) * 1024 * 1024
+                        else:
+                            maxSize = max(inputChunk.maxInputSizeAvalanche, maxSize) * 1024 * 1024
+                else:
+                    # add offset
+                    maxSize += sizeIntercepts
+                # max disk size
+                maxDiskSize = siteCandidate.get_overridden_attribute("maxwdir")
+                if maxDiskSize is None:
+                    maxDiskSize = siteSpec.maxwdir
+                if maxDiskSize:
+                    maxDiskSize *= 1024 * 1024
+                # max walltime
+                maxWalltime = None
+                if not inputChunk.isMerging:
+                    maxWalltime = taskSpec.getMaxWalltime()
+                if maxWalltime is None:
+                    maxWalltime = siteSpec.maxtime
+                # core count
+                if siteSpec.coreCount:
+                    coreCount = siteSpec.coreCount
+                else:
+                    coreCount = 1
+                # core power
+                corePower = siteSpec.corepower
+                if taskSpec.dynamicNumEvents() and siteSpec.mintime:
+                    dynNumEvents = True
+                else:
+                    dynNumEvents = False
+                tmpLog.debug(f"chosen {siteName} : {getCandidateMsg} : nQueue={siteCandidate.nQueuedJobs} nRunCap={siteCandidate.nRunningJobsCap}")
+                tmpLog.debug(f"new weight {siteCandidate.weight}")
+                tmpLog.debug(
+                    f"maxSize={maxSize} maxWalltime={maxWalltime} coreCount={coreCount} corePower={corePower} maxDisk={maxDiskSize} dynNumEvents={dynNumEvents}"
+                )
+                tmpLog.debug(f"useDirectIO={useDirectIO} label={taskSpec.prodSourceLabel}")
+            # get sub chunk
+            subChunk, _ = inputChunk.getSubChunk(
+                siteName,
+                maxSize=maxSize,
+                maxNumFiles=maxNumFiles,
+                sizeGradients=sizeGradients,
+                sizeIntercepts=sizeIntercepts,
+                nFilesPerJob=nFilesPerJob,
+                walltimeGradient=walltimeGradient,
+                maxWalltime=maxWalltime,
+                nEventsPerJob=nEventsPerJob,
+                useBoundary=useBoundary,
+                sizeGradientsPerInSize=sizeGradientsPerInSize,
+                maxOutSize=maxOutSize,
+                coreCount=coreCount,
+                respectLB=respectLB,
+                corePower=corePower,
+                dynNumEvents=dynNumEvents,
+                multiplicity=multiplicity,
+                splitByFields=splitByFields,
+                tmpLog=tmpLog,
+                useDirectIO=useDirectIO,
+                maxDiskSize=maxDiskSize,
+                enableLog=True,
+                no_split=no_split,
+                min_walltime=siteSpec.mintime,
+                max_events=taskSpec.get_max_events_per_job(),
+            )
+            if subChunk is None:
+                break
+            if subChunk != []:
+                # append
+                subChunks.append(subChunk)
+                # intermediate checkpoint
+                inputChunk.checkpoint_file_usage(intermediate=True)
+                # check if the last file in master dist dataset is locally available
+                for tmp_dataset, tmp_files in subChunk:
+                    if tmp_dataset.isMaster():
+                        if tmp_dataset.isDistributed() and not siteCandidate.isAvailableFile(tmp_files[-1]) and siteCandidate.isAvailableFile(tmp_files[0]):
+                            change_site_for_dist_dataset = True
+                            tmpLog.debug(
+                                f"change site since the last file in distributed sub-dataset was unavailable at {siteName} while the first file was available"
+                            )
+                        break
+            else:
+                tmp_ng_list.append(siteName)
+                inputChunk.rollback_file_usage(intermediate=True)
+                tmpLog.debug("rollback to intermediate file usage")
+            iSubChunks += 1
+        # append to return map if remain
+        isSkipped = False
+        if subChunks != []:
+            # skip if chunk size is not enough
+            if allow_chunk_size_limit and strict_chunkSize and len(subChunks) < nSubChunks:
+                tmpLog.debug(f"skip splitting since chunk size {len(subChunks)} is less than chunk size limit {nSubChunks} at {siteName}")
+                inputChunk.rollback_file_usage()
+                isSkipped = True
+            else:
+                # get site names for parallel execution
+                if taskSpec.getNumSitesPerJob() > 1 and not inputChunk.isMerging:
+                    siteName = inputChunk.getParallelSites(taskSpec.getNumSitesPerJob(), nSubChunks, [siteName])
+                returnList.append(
+                    {
+                        "siteName": siteName,
+                        "subChunks": subChunks,
+                        "siteCandidate": siteCandidate,
+                    }
+                )
+                try:
+                    gshare = taskSpec.gshare.replace(" ", "_")
+                except Exception:
+                    gshare = None
+                tmpLog.info(f"split to nJobs={len(subChunks)} at site={siteName} gshare={gshare}")
+        # return
+        tmpLog.debug("--- done")
+        return self.SC_SUCCEEDED, returnList, isSkipped
+
+
+Interaction.installSC(JobSplitter)

pandajedi/jediorder/JobThrottler.py

@@ -0,0 +1,28 @@
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore.FactoryBase import FactoryBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# factory class for throttling
+class JobThrottler(FactoryBase):
+    # constructor
+    def __init__(self, vo, sourceLabel):
+        FactoryBase.__init__(self, vo, sourceLabel, logger, jedi_config.jobthrottle.modConfig)
+
+    # main
+    def toBeThrottled(self, vo, sourceLabel, cloudName, workQueue, resourceType):
+        impl = self.getImpl(vo, sourceLabel)
+        retVal = impl.toBeThrottled(vo, sourceLabel, cloudName, workQueue, resourceType)
+        # retrieve min priority and max number of jobs from concrete class
+        self.minPriority = impl.minPriority
+        self.maxNumJobs = impl.maxNumJobs
+        self.lackOfJobs = impl.underNqLimit
+        return retVal
+
+    # check throttle level
+    def mergeThrottled(self, vo, sourceLabel, thrLevel):
+        impl = self.getImpl(vo, sourceLabel)
+        return impl.mergeThrottled(thrLevel)

pandajedi/jediorder/PostProcessor.py

@@ -0,0 +1,166 @@
+import datetime
+import os
+import socket
+import sys
+import time
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore import Interaction
+from pandajedi.jedicore.FactoryBase import FactoryBase
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool, WorkerThread
+
+from .JediKnight import JediKnight
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# worker class to do post-processing
+class PostProcessor(JediKnight, FactoryBase):
+    # constructor
+    def __init__(self, commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels):
+        self.vos = self.parseInit(vos)
+        self.prodSourceLabels = self.parseInit(prodSourceLabels)
+        self.pid = f"{socket.getfqdn().split('.')[0]}-{os.getpid()}-post"
+        JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
+        FactoryBase.__init__(self, self.vos, self.prodSourceLabels, logger, jedi_config.postprocessor.modConfig)
+
+    # main
+    def start(self):
+        # start base classes
+        JediKnight.start(self)
+        FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
+        # go into main loop
+        while True:
+            startTime = naive_utcnow()
+            try:
+                # get logger
+                tmpLog = MsgWrapper(logger)
+                tmpLog.info("start")
+                # loop over all vos
+                for vo in self.vos:
+                    # loop over all sourceLabels
+                    for prodSourceLabel in self.prodSourceLabels:
+                        # prepare tasks to be finished
+                        tmpLog.info(f"preparing tasks to be finished for vo={vo} label={prodSourceLabel}")
+                        tmp_ret_list = self.taskBufferIF.prepareTasksToBeFinished_JEDI(vo, prodSourceLabel, jedi_config.postprocessor.nTasks, pid=self.pid)
+                        if tmp_ret_list is None:
+                            # failed
+                            tmpLog.error("failed to prepare tasks")
+                        # get tasks to be finished
+                        tmpLog.info("getting tasks to be finished")
+                        tmpList = self.taskBufferIF.getTasksToBeFinished_JEDI(
+                            vo, prodSourceLabel, self.pid, jedi_config.postprocessor.nTasks, target_tasks=tmp_ret_list
+                        )
+                        if tmpList is None:
+                            # failed
+                            tmpLog.error("failed to get tasks to be finished")
+                        else:
+                            tmpLog.info(f"got {len(tmpList)} tasks")
+                            # put to a locked list
+                            taskList = ListWithLock(tmpList)
+                            # make thread pool
+                            threadPool = ThreadPool()
+                            # make workers
+                            nWorker = jedi_config.postprocessor.nWorkers
+                            for iWorker in range(nWorker):
+                                thr = PostProcessorThread(taskList, threadPool, self.taskBufferIF, self.ddmIF, self)
+                                thr.start()
+                            # join
+                            threadPool.join()
+                tmpLog.info("done")
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                tmpLog.error(f"failed in {self.__class__.__name__}.start() with {errtype.__name__} {errvalue}")
+            # sleep if needed
+            loopCycle = 60
+            timeDelta = naive_utcnow() - startTime
+            sleepPeriod = loopCycle - timeDelta.seconds
+            if sleepPeriod > 0:
+                time.sleep(sleepPeriod)
+
+
+# thread for real worker
+class PostProcessorThread(WorkerThread):
+    # constructor
+    def __init__(self, taskList, threadPool, taskbufferIF, ddmIF, implFactory):
+        # initialize woker with no semaphore
+        WorkerThread.__init__(self, None, threadPool, logger)
+        # attributres
+        self.taskList = taskList
+        self.taskBufferIF = taskbufferIF
+        self.ddmIF = ddmIF
+        self.implFactory = implFactory
+
+    # post process tasks
+    def post_process_tasks(self, task_list):
+        for taskSpec in task_list:
+            # make logger
+            tmpLog = MsgWrapper(self.logger, f"<jediTaskID={taskSpec.jediTaskID}>")
+            tmpLog.info("start")
+            tmpStat = Interaction.SC_SUCCEEDED
+            # get impl
+            impl = self.implFactory.instantiateImpl(taskSpec.vo, taskSpec.prodSourceLabel, None, self.taskBufferIF, self.ddmIF)
+            if impl is None:
+                # post processor is undefined
+                tmpLog.error(f"post-processor is undefined for vo={taskSpec.vo} sourceLabel={taskSpec.prodSourceLabel}")
+                tmpStat = Interaction.SC_FATAL
+            # execute
+            if tmpStat == Interaction.SC_SUCCEEDED:
+                tmpLog.info(f"post-process with {impl.__class__.__name__}")
+                try:
+                    tmpStat = impl.doPostProcess(taskSpec, tmpLog)
+                except Exception as e:
+                    tmpLog.error(f"post-process failed with {str(e)}")
+                    tmpStat = Interaction.SC_FATAL
+            # done
+            if tmpStat == Interaction.SC_FATAL or (tmpStat == Interaction.SC_FAILED and taskSpec.status in ["toabort", "tobroken"]):
+                # task is broken
+                tmpErrStr = "post-process permanently failed"
+                tmpLog.error(tmpErrStr)
+                taskSpec.status = "broken"
+                taskSpec.setErrDiag(tmpErrStr)
+                taskSpec.lockedBy = None
+                self.taskBufferIF.updateTask_JEDI(taskSpec, {"jediTaskID": taskSpec.jediTaskID})
+            elif tmpStat == Interaction.SC_FAILED:
+                tmpErrStr = "post-processing temporarily failed"
+                taskSpec.setErrDiag(tmpErrStr, True)
+                self.taskBufferIF.updateTask_JEDI(taskSpec, {"jediTaskID": taskSpec.jediTaskID})
+                tmpLog.info(f"set task_status={taskSpec.status} since {taskSpec.errorDialog}")
+                tmpLog.info("done")
+                continue
+            # final procedure
+            try:
+                impl.doFinalProcedure(taskSpec, tmpLog)
+            except Exception as e:
+                tmpLog.error(f"final procedure failed with {str(e)}")
+            # done
+            tmpLog.info("done")
+
+    # main
+    def runImpl(self):
+        while True:
+            try:
+                # get a part of list
+                nTasks = 10
+                taskList = self.taskList.get(nTasks)
+                # no more datasets
+                if len(taskList) == 0:
+                    self.logger.debug(f"{self.__class__.__name__} terminating since no more items")
+                    return
+                # post process tasks
+                self.post_process_tasks(taskList)
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                logger.error(f"{self.__class__.__name__} failed in runImpl() with {errtype.__name__}:{errvalue}")
+
+
+# launch
+
+
+def launcher(commuChannel, taskBufferIF, ddmIF, vos=None, prodSourceLabels=None):
+    p = PostProcessor(commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels)
+    p.start()