panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedipprocess/AtlasProdPostProcessor.py

@@ -0,0 +1,178 @@
+import sys
+
+from pandaserver.dataservice import DataServiceUtils
+from pandaserver.taskbuffer import EventServiceUtils
+
+from . import AtlasPostProcessorUtils
+from .PostProcessorBase import PostProcessorBase
+
+
+# post processor for ATLAS production
+class AtlasProdPostProcessor(PostProcessorBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        PostProcessorBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main
+    def doPostProcess(self, taskSpec, tmpLog):
+        # pre-check
+        try:
+            tmpStat = self.doPreCheck(taskSpec, tmpLog)
+            if tmpStat:
+                return self.SC_SUCCEEDED
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            tmpLog.error(f"doPreCheck failed with {errtype.__name__}:{errvalue}")
+            return self.SC_FATAL
+        # get DDM I/F
+        ddmIF = self.ddmIF.getInterface(taskSpec.vo)
+        # loop over all datasets
+        for datasetSpec in taskSpec.datasetSpecList:
+            # skip pseudo output datasets
+            if datasetSpec.type in ["output"] and datasetSpec.isPseudo():
+                continue
+            try:
+                # remove wrong files
+                if datasetSpec.type in ["output"]:
+                    # get successful files
+                    okFiles = self.taskBufferIF.getSuccessfulFiles_JEDI(datasetSpec.jediTaskID, datasetSpec.datasetID)
+                    if okFiles is None:
+                        tmpLog.warning(f"failed to get successful files for {datasetSpec.datasetName}")
+                        return self.SC_FAILED
+                    # get files in dataset
+                    ddmFiles = ddmIF.getFilesInDataset(datasetSpec.datasetName, skipDuplicate=False, ignoreUnknown=True)
+                    tmpLog.debug(
+                        f"datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName} has {len(okFiles)} files in DB, {len(ddmFiles)} files in DDM"
+                    )
+                    # check all files
+                    toDelete = []
+                    for tmpGUID, attMap in ddmFiles.items():
+                        if attMap["lfn"] not in okFiles:
+                            did = {"scope": attMap["scope"], "name": attMap["lfn"]}
+                            toDelete.append(did)
+                            tmpLog.debug(f"delete {attMap['lfn']} from {datasetSpec.datasetName}")
+                    # delete
+                    if toDelete != []:
+                        ddmIF.deleteFilesFromDataset(datasetSpec.datasetName, toDelete)
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                tmpLog.warning(f"failed to remove wrong files with {errtype.__name__}:{errvalue}")
+                return self.SC_FAILED
+            try:
+                # freeze output and log datasets
+                if datasetSpec.type in ["output", "log", "trn_log"]:
+                    tmpLog.info(f"freezing datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName}")
+                    ddmIF.freezeDataset(datasetSpec.datasetName, ignoreUnknown=True)
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                tmpLog.warning(f"failed to freeze datasets with {errtype.__name__}:{errvalue}")
+                return self.SC_FAILED
+            try:
+                # delete transient datasets
+                if datasetSpec.type in ["trn_output"]:
+                    tmpLog.debug(f"deleting datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName}")
+                    retStr = ddmIF.deleteDataset(datasetSpec.datasetName, False, ignoreUnknown=True)
+                    tmpLog.info(retStr)
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                tmpLog.warning(f"failed to delete datasets with {errtype.__name__}:{errvalue}")
+        # check duplication
+        if self.getFinalTaskStatus(taskSpec) in ["finished", "done"] and taskSpec.gshare != "Test":
+            nDup = self.taskBufferIF.checkDuplication_JEDI(taskSpec.jediTaskID)
+            tmpLog.debug(f"checked duplication with {nDup}")
+            if nDup is not None and nDup > 0:
+                errStr = f"paused since {nDup} duplication found"
+                taskSpec.oldStatus = self.getFinalTaskStatus(taskSpec)
+                taskSpec.status = "paused"
+                taskSpec.setErrDiag(errStr)
+                tmpLog.debug(errStr)
+        # delete ES datasets
+        if taskSpec.registerEsFiles():
+            try:
+                targetName = EventServiceUtils.getEsDatasetName(taskSpec.jediTaskID)
+                tmpLog.debug(f"deleting ES dataset name={targetName}")
+                retStr = ddmIF.deleteDataset(targetName, False, ignoreUnknown=True)
+                tmpLog.debug(retStr)
+            except Exception:
+                errtype, errvalue = sys.exc_info()[:2]
+                tmpLog.warning(f"failed to delete ES dataset with {errtype.__name__}:{errvalue}")
+        try:
+            AtlasPostProcessorUtils.send_notification(self.taskBufferIF, ddmIF, taskSpec, tmpLog)
+        except Exception as e:
+            tmpLog.error(f"failed to talk to external system with {str(e)}")
+            return self.SC_FAILED
+        try:
+            self.doBasicPostProcess(taskSpec, tmpLog)
+        except Exception:
+            errtype, errvalue = sys.exc_info()[:2]
+            tmpLog.error(f"doBasicPostProcess failed with {errtype.__name__}:{errvalue}")
+            return self.SC_FATAL
+        return self.SC_SUCCEEDED
+
+    # final procedure
+    def doFinalProcedure(self, taskSpec, tmpLog):
+        tmpLog.info(f"final procedure for status={taskSpec.status} processingType={taskSpec.processingType}")
+        if taskSpec.status in ["done", "finished"] or (taskSpec.status == "paused" and taskSpec.oldStatus in ["done", "finished"]):
+            trnLifeTime = 14 * 24 * 60 * 60
+            trnLifeTimeMerge = 40 * 24 * 60 * 60
+            ddmIF = self.ddmIF.getInterface(taskSpec.vo)
+            # set lifetime to transient datasets
+            metaData = {"lifetime": trnLifeTime}
+            datasetTypeListI = set()
+            datasetTypeListO = set()
+            for datasetSpec in taskSpec.datasetSpecList:
+                if datasetSpec.type in ["log", "output"]:
+                    if datasetSpec.getTransient() is True:
+                        tmpLog.debug(f"set metadata={str(metaData)} to datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName}")
+                        for metadataName, metadaValue in metaData.items():
+                            ddmIF.setDatasetMetadata(datasetSpec.datasetName, metadataName, metadaValue)
+                # collect dataset types
+                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
+                if datasetType not in ["", None]:
+                    if datasetSpec.type == "input":
+                        datasetTypeListI.add(datasetType)
+                    elif datasetSpec.type == "output":
+                        datasetTypeListO.add(datasetType)
+            # set lifetime to parent transient datasets
+            if taskSpec.processingType in ["merge"] and (
+                taskSpec.status == "done"
+                or (taskSpec.status == "finished" and self.getFinalTaskStatus(taskSpec, checkParent=False) == "done")
+                or (
+                    taskSpec.status == "paused"
+                    and (taskSpec.oldStatus == "done" or (taskSpec.oldStatus == "finished" and self.getFinalTaskStatus(taskSpec, checkParent=False) == "done"))
+                )
+            ):
+                # get parent task
+                if taskSpec.parent_tid not in [None, taskSpec.jediTaskID]:
+                    # get parent
+                    tmpStat, parentTaskSpec = self.taskBufferIF.getTaskDatasetsWithID_JEDI(taskSpec.parent_tid, None, False)
+                    if tmpStat and parentTaskSpec is not None:
+                        # set lifetime to parent datasets if they are transient
+                        for datasetSpec in parentTaskSpec.datasetSpecList:
+                            if datasetSpec.type in ["output"]:
+                                # check dataset type
+                                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
+                                if datasetType not in datasetTypeListI or datasetType not in datasetTypeListO:
+                                    continue
+                                metaData = {"lifetime": trnLifeTimeMerge}
+                                tmpMetadata = ddmIF.getDatasetMetaData(datasetSpec.datasetName)
+                                if tmpMetadata["transient"] is True:
+                                    tmpLog.debug(
+                                        "set metadata={0} to parent jediTaskID={1}:datasetID={2}:Name={3}".format(
+                                            str(metaData), taskSpec.parent_tid, datasetSpec.datasetID, datasetSpec.datasetName
+                                        )
+                                    )
+                                    for metadataName, metadaValue in metaData.items():
+                                        ddmIF.setDatasetMetadata(datasetSpec.datasetName, metadataName, metadaValue)
+        # set lifetime to failed datasets
+        if taskSpec.status in ["failed", "broken", "aborted"]:
+            trnLifeTime = 30 * 24 * 60 * 60
+            ddmIF = self.ddmIF.getInterface(taskSpec.vo)
+            # only log datasets
+            metaData = {"lifetime": trnLifeTime}
+            for datasetSpec in taskSpec.datasetSpecList:
+                if datasetSpec.type in ["log"]:
+                    tmpLog.debug(f"set metadata={str(metaData)} to failed datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName}")
+                    for metadataName, metadaValue in metaData.items():
+                        ddmIF.setDatasetMetadata(datasetSpec.datasetName, metadataName, metadaValue)
+        return self.SC_SUCCEEDED

pandajedi/jedipprocess/GenPostProcessor.py

@@ -0,0 +1,35 @@
+from .PostProcessorBase import PostProcessorBase
+
+
+# post processor for general purpose
+class GenPostProcessor(PostProcessorBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        PostProcessorBase.__init__(self, taskBufferIF, ddmIF)
+        self.failOnZeroOkFile = True
+
+    # main
+    def doPostProcess(self, taskSpec, tmpLog):
+        try:
+            # get DDM I/F
+            ddmIF = self.ddmIF.getInterface(taskSpec.vo, taskSpec.cloud)
+            # skip if DDM I/F is inactive
+            if not ddmIF:
+                tmpLog.info("skip due to inactive DDM I/F")
+            else:
+                # loop over all datasets
+                for datasetSpec in taskSpec.datasetSpecList:
+                    # only output and log datasets
+                    if datasetSpec.type not in ["log", "output"]:
+                        continue
+                    tmpLog.info(f"freezing datasetID={datasetSpec.datasetID}:Name={datasetSpec.datasetName}")
+                    ddmIF.freezeDataset(datasetSpec.datasetName, ignoreUnknown=True)
+        except Exception as e:
+            tmpLog.warning(f"failed to freeze datasets with {str(e)}")
+            return self.SC_FAILED
+        try:
+            self.doBasicPostProcess(taskSpec, tmpLog)
+        except Exception as e:
+            tmpLog.error(f"doBasicPostProcess failed with {str(e)}")
+            return self.SC_FATAL
+        return self.SC_SUCCEEDED

pandajedi/jedipprocess/MailTemplates.py

@@ -0,0 +1,185 @@
+jedi_task_plain = """
+Subject: JEDI notification for TaskID:{jedi_task_id} ({n_succeeded}/{n_total} {msg_succeeded})
+
+Summary of TaskID:{jedi_task_id}
+
+Created : {creation_time} (UTC)
+Ended   : {end_time} (UTC)
+
+Final Status : {task_status}
+
+Total Number of {input_str}   : {n_total}
+             Succeeded   : {n_succeeded}
+             Failed      : {n_failed}
+             {cancelled_str} : {n_cancelled}
+
+
+Error Dialog : {error_dialog}
+
+{dataset_summary}
+
+Parameters : {command}
+
+
+PandaMonURL : https://bigpanda.cern.ch/task/{jedi_task_id}/
+
+Please use the smiley interface on this link to provide feedback on the task execution. 
+
+Estimated carbon footprint for the task
+     Succeeded   : {carbon_succeeded}
+     Failed      : {carbon_failed}
+     Cancelled   : {carbon_cancelled}
+     _________________
+     Total       : {carbon_total}
+(More details on estimation: https://panda-wms.readthedocs.io/en/latest/advanced/carbon_footprint.html)
+
+
+Report Panda problems of any sort to
+
+  the eGroup for help request
+    hn-atlas-dist-analysis-help@cern.ch
+    
+  the Discourse forum for distributed computing help  
+    https://atlas-talk.web.cern.ch/c/distributed-computing-help
+
+  the JIRA portal for software bug
+    https://its.cern.ch/jira/browse/ATLASPANDA
+"""
+
+html_head = """
+<!DOCTYPE html>
+<html>
+<head>
+    <style>
+        body {{
+            font-family: Arial, sans-serif;
+        }}
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 10px;
+        }}
+        th, td {{
+            border: 1px solid #dddddd;
+            text-align: left;
+            padding: 8px;
+        }}
+        th {{
+            background-color: #f2f2f2;
+        }}
+        .logo {{
+            width: 100px; /* Adjust the size as needed */
+        }}
+    </style>
+    <title> {title} </title>
+</head>
+"""
+
+jedi_task_html_body = """
+<body>
+    <img class="logo" src="https://panda-wms.readthedocs.io/en/latest/_static/PanDA-rev-logo-small-200px.jpg" alt="PanDA Logo">
+    <h2>Summary of TaskID: <a href="https://bigpanda.cern.ch/task/{jedi_task_id}">{jedi_task_id}</a> </h2>
+    <p> 💬 Please use the smiley interface on this link to provide feedback on the task execution.</p>
+    <table>
+        <tr>
+            <th>Detail</th>
+            <th>Value</th>
+        </tr>
+        <tr>
+            <td>Created</td>
+            <td>{creation_time}</td>
+        </tr>
+        <tr>
+            <td>Ended</td>
+            <td>{end_time}</td>
+        </tr>
+        <tr>
+            <td>Final Status</td>
+            <td>{task_status}</td>
+        </tr>
+    </table>
+
+    <h3>Total Number of {input_str}</h3>
+    <table>
+        <tr>
+            <th>Category</th>
+            <th>Count</th>
+        </tr>
+        <tr>
+            <td>Succeeded</td>
+            <td>{n_succeeded}</td>
+        </tr>
+        <tr>
+            <td>Failed</td>
+            <td>{n_failed}</td>
+        </tr>
+        <tr>
+            <td>{cancelled_str}</td>
+            <td>{n_cancelled}</td>
+        </tr>
+        <tr>
+            <td>Total</td>
+            <td>{n_total}</td>
+        </tr>
+    </table>
+
+    <p><strong>Error Dialog:</strong> {error_dialog}</p>
+
+    <h3>Datasets</h3>
+    <table>
+        <tr>
+            <td><strong>In</strong></td>
+            <td>{datasets_in}</td>
+        </tr>
+        <tr>
+            <td><strong>Out</strong></td>
+            <td>{datasets_out}</td>
+        </tr>
+        <tr>
+            <td><strong>Log</strong></td>
+            <td>{datasets_log}</td>
+        </tr>
+    </table>
+
+    <h3>Parameters</h3>
+    <table>
+        <tr>
+            <td><strong>Command</strong></td>
+            <td>{command}</td>
+        </tr>
+    </table>
+
+    <h3>Estimated Carbon Footprint for the Task</h3>
+    <table>
+        <tr>
+            <th>Category</th>
+            <th>gCO2</th>
+        </tr>
+        <tr>
+            <td>Succeeded</td>
+            <td>{carbon_succeeded}</td>
+        </tr>
+        <tr>
+            <td>Failed</td>
+            <td>{carbon_failed}</td>
+        </tr>
+        <tr>
+            <td>Cancelled</td>
+            <td>{carbon_cancelled}</td>
+        </tr>
+        <tr>
+            <td><strong>Total</strong></td>
+            <td><strong>{carbon_total}</strong></td>
+        </tr>
+    </table>
+    <p>More details on estimation: <a href="https://panda-wms.readthedocs.io/en/latest/advanced/carbon_footprint.html">https://panda-wms.readthedocs.io/en/latest/advanced/carbon_footprint.html</a></p>
+
+    <h3>Panda Problem Reporting</h3>
+    <ul>
+        <li>The eGroup for help request: <a href="mailto:hn-atlas-dist-analysis-help@cern.ch">hn-atlas-dist-analysis-help@cern.ch</a></li>
+        <li>The Discourse forum for distributed computing help: <a href="https://atlas-talk.web.cern.ch/c/distributed-computing-help">https://atlas-talk.web.cern.ch/c/distributed-computing-help</a></li>
+        <li>The JIRA portal for software bug: <a href="https://its.cern.ch/jira/browse/ATLASPANDA">https://its.cern.ch/jira/browse/ATLASPANDA</a></li>
+    </ul>
+</body>
+</html>
+"""

pandajedi/jedipprocess/PostProcessorBase.py

@@ -0,0 +1,269 @@
+import datetime
+import smtplib
+import time
+import uuid
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandajedi.jedicore import Interaction
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer import EventServiceUtils
+
+# port for SMTP server
+smtpPortList = [25, 587]
+
+
+# wrapper to patch smtplib.stderr to send debug info to logger
+class StderrLogger(object):
+    def __init__(self, tmpLog):
+        self.tmpLog = tmpLog
+
+    def write(self, message):
+        message = message.strip()
+        if message != "":
+            self.tmpLog.debug(message)
+
+
+# wrapper of SMTP to redirect messages
+class MySMTP(smtplib.SMTP):
+    def set_log(self, tmp_log):
+        self.tmpLog = tmp_log
+        try:
+            self.org_stderr = getattr(smtplib, "stderr")
+            setattr(smtplib, "stderr", tmp_log)
+        except Exception:
+            self.org_stderr = None
+
+    def _print_debug(self, *args):
+        self.tmpLog.write(" ".join(map(str, args)))
+
+    def reset_log(self):
+        if self.org_stderr is not None:
+            setattr(smtplib, "stderr", self.org_stderr)
+
+
+# base class for post process
+class PostProcessorBase(object):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        self.ddmIF = ddmIF
+        self.taskBufferIF = taskBufferIF
+        self.msgType = "postprocessor"
+        self.failOnZeroOkFile = False
+        self.refresh()
+
+    # refresh
+    def refresh(self):
+        self.siteMapper = self.taskBufferIF.get_site_mapper()
+
+    # basic post procedure
+    def doBasicPostProcess(self, taskSpec, tmpLog):
+        # update task status
+        taskSpec.lockedBy = None
+        taskSpec.status = self.getFinalTaskStatus(taskSpec)
+        if taskSpec.status == "failed":
+            # set dialog for preprocess
+            if taskSpec.usePrePro() and not taskSpec.checkPreProcessed():
+                taskSpec.setErrDiag("Preprocessing step failed", True)
+        tmpMsg = f"set task_status={taskSpec.status}"
+        tmpLog.info(tmpMsg)
+        tmpLog.sendMsg(f"set task_status={taskSpec.status}", self.msgType)
+        # update dataset
+        for datasetSpec in taskSpec.datasetSpecList:
+            if taskSpec.status in ["failed", "broken", "aborted"]:
+                datasetSpec.status = "failed"
+            else:
+                # set dataset status
+                if datasetSpec.type in ["output", "log", "lib"]:
+                    # normal output datasets
+                    if datasetSpec.nFiles and datasetSpec.nFilesFinished and datasetSpec.nFiles > datasetSpec.nFilesFinished:
+                        datasetSpec.status = "finished"
+                    else:
+                        datasetSpec.status = "done"
+                elif datasetSpec.type.startswith("trn_") or datasetSpec.type.startswith("tmpl_"):
+                    # set done for template or transient datasets
+                    datasetSpec.status = "done"
+                else:
+                    # not for input
+                    continue
+            # set nFiles
+            if datasetSpec.type in ["output", "log", "lib"]:
+                datasetSpec.nFiles = datasetSpec.nFilesFinished
+            self.taskBufferIF.updateDataset_JEDI(datasetSpec, {"datasetID": datasetSpec.datasetID, "jediTaskID": datasetSpec.jediTaskID})
+        # trigger internal dataset cleanup
+        self.taskBufferIF.trigger_cleanup_internal_datasets(taskSpec.jediTaskID)
+        # end time
+        taskSpec.endTime = naive_utcnow()
+        # update task
+        self.taskBufferIF.updateTask_JEDI(taskSpec, {"jediTaskID": taskSpec.jediTaskID}, updateDEFT=True)
+        # kill or kick child tasks
+        if taskSpec.status in ["failed", "broken", "aborted"]:
+            self.taskBufferIF.killChildTasks_JEDI(taskSpec.jediTaskID, taskSpec.status)
+        else:
+            self.taskBufferIF.kickChildTasks_JEDI(taskSpec.jediTaskID)
+        tmpLog.debug(f"doBasicPostProcess done with taskStatus={taskSpec.status}")
+        return
+
+    # final procedure
+    def doFinalProcedure(self, taskSpec, tmpLog):
+        return self.SC_SUCCEEDED
+
+    # send mail
+    def sendMail(self, jediTaskID, fromAdd, toAdd, msgBody, nTry, fileBackUp, tmpLog):
+        tmpLog.debug(f"sending notification to {toAdd}\n{msgBody}")
+        for iTry in range(nTry):
+            try:
+                stderrLog = StderrLogger(tmpLog)
+                smtpPort = smtpPortList[iTry % len(smtpPortList)]
+                server = MySMTP(panda_config.emailSMTPsrv, smtpPort)
+                server.set_debuglevel(1)
+                server.set_log(stderrLog)
+                server.ehlo()
+                server.starttls()
+                out = server.sendmail(fromAdd, toAdd, msgBody)
+                tmpLog.debug(str(out))
+                server.quit()
+                break
+            except Exception as e:
+                if iTry + 1 < nTry:
+                    # sleep for retry
+                    tmpLog.debug(f"sleep {iTry} due to {str(e)}")
+                    time.sleep(30)
+                else:
+                    tmpLog.error(f"failed to send notification with {str(e)}")
+                    if fileBackUp:
+                        # write to file which is processed in add.py
+                        mailFile = "{0}/jmail_{1}_{2}" % (panda_config.logdir, jediTaskID, uuid.uuid4())
+                        oMail = open(mailFile, "w")
+                        oMail.write(str(jediTaskID) + "\n" + toAdd + "\n" + msgBody)
+                        oMail.close()
+                break
+        try:
+            server.reset_log()
+        except Exception:
+            pass
+
+    # return email sender
+    def senderAddress(self):
+        return panda_config.emailSender
+
+    # get task completeness
+    def getTaskCompleteness(self, taskSpec):
+        nFiles = 0
+        nFilesFinished = 0
+        totalInputEvents = 0
+        totalOkEvents = 0
+        for datasetSpec in taskSpec.datasetSpecList:
+            if datasetSpec.isMasterInput():
+                if datasetSpec.status == "removed":
+                    continue
+                nFiles += datasetSpec.nFiles
+                nFilesFinished += datasetSpec.nFilesFinished
+                try:
+                    totalInputEvents += datasetSpec.nEvents
+                except Exception:
+                    pass
+                try:
+                    totalOkEvents += datasetSpec.nEventsUsed
+                except Exception:
+                    pass
+        # completeness
+        if totalInputEvents != 0:
+            taskCompleteness = float(totalOkEvents) / float(totalInputEvents) * 1000.0
+        elif nFiles != 0:
+            taskCompleteness = float(nFilesFinished) / float(nFiles) * 1000.0
+        else:
+            taskCompleteness = 0
+        return nFiles, nFilesFinished, totalInputEvents, totalOkEvents, taskCompleteness
+
+    # get final task status
+    def getFinalTaskStatus(self, taskSpec, checkParent=True, checkGoal=False):
+        # count nFiles and nEvents
+        nFiles, nFilesFinished, totalInputEvents, totalOkEvents, taskCompleteness = self.getTaskCompleteness(taskSpec)
+        # set new task status
+        if taskSpec.status == "tobroken":
+            status = "broken"
+        elif taskSpec.status == "toabort":
+            status = "aborted"
+        elif taskSpec.status == "paused":
+            status = "paused"
+        elif self.failOnZeroOkFile and nFiles == nFilesFinished == 0:
+            status = "failed"
+        elif nFiles == nFilesFinished:
+            # check parent status
+            if checkParent and taskSpec.parent_tid not in [None, taskSpec.jediTaskID]:
+                parent_status = self.taskBufferIF.getTaskStatus_JEDI(taskSpec.parent_tid)
+                if parent_status in ["failed", "broken", "aborted"]:
+                    status = "failed"
+                elif parent_status != "done":
+                    status = "finished"
+                else:
+                    # check if input is mutable
+                    inputMutable = False
+                    for datasetSpec in taskSpec.datasetSpecList:
+                        if datasetSpec.isMasterInput() and datasetSpec.state == "mutable":
+                            inputMutable = True
+                            break
+                    if inputMutable:
+                        status = "finished"
+                    else:
+                        status = "done"
+            else:
+                status = "done"
+        elif nFilesFinished == 0:
+            status = "failed"
+        else:
+            status = "finished"
+        # task goal
+        if taskSpec.goal is None:
+            taskGoal = 1000
+        else:
+            taskGoal = taskSpec.goal
+        # fail if goal is not reached
+        if (
+            taskSpec.failGoalUnreached()
+            and status == "finished"
+            and (not taskSpec.useExhausted() or (taskSpec.useExhausted() and taskSpec.status in ["passed"]))
+        ):
+            if taskCompleteness < taskGoal:
+                status = "failed"
+        # HPO tasks always go to finished
+        if taskSpec.is_hpo_workflow():
+            event_stat = self.taskBufferIF.get_event_statistics(taskSpec.jediTaskID)
+            if event_stat is not None and event_stat.get(EventServiceUtils.ST_finished):
+                status = "finished"
+        # check goal only
+        if checkGoal:
+            # no goal
+            if taskSpec.goal is not None and taskCompleteness >= taskGoal:
+                return True
+            return False
+        # return status
+        return status
+
+    # pre-check
+    def doPreCheck(self, taskSpec, tmpLog):
+        # send task to exhausted
+        if (
+            taskSpec.useExhausted()
+            and taskSpec.status not in ["passed"]
+            and self.getFinalTaskStatus(taskSpec) in ["finished"]
+            and not self.getFinalTaskStatus(taskSpec, checkGoal=True)
+        ):
+            taskSpec.status = "exhausted"
+            if self.getFinalTaskStatus(taskSpec, checkParent=False) == "done":
+                taskSpec.errorDialog = "exhausted since the parent task was incomplete and didn't reach the goal"
+            else:
+                taskSpec.errorDialog = "exhausted since the task was incomplete and didn't reach the goal"
+            taskSpec.lockedBy = None
+            taskSpec.lockedTime = None
+            # update task
+            tmpLog.info(f"set task_status={taskSpec.status}")
+            self.taskBufferIF.updateTask_JEDI(taskSpec, {"jediTaskID": taskSpec.jediTaskID}, updateDEFT=True)
+            # kick child tasks
+            self.taskBufferIF.kickChildTasks_JEDI(taskSpec.jediTaskID)
+            return True
+        return False
+
+
+Interaction.installSC(PostProcessorBase)