panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedisetup/AtlasTaskSetupper.py

@@ -0,0 +1,309 @@
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+from pandaserver.dataservice import DataServiceUtils
+from pandaserver.taskbuffer import EventServiceUtils, JobUtils
+
+from .TaskSetupperBase import TaskSetupperBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# task setup for ATLAS
+class AtlasTaskSetupper(TaskSetupperBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskSetupperBase.__init__(self, taskBufferIF, ddmIF)
+        self.user_container_lifetime = taskBufferIF.getConfigValue("user_output", "OUTPUT_CONTAINER_LIFETIME", "jedi")
+        if not self.user_container_lifetime:
+            self.user_container_lifetime = 14
+
+    # main to setup task
+    def doSetup(self, taskSpec, datasetToRegister, pandaJobs):
+        # make logger
+        tmpLog = MsgWrapper(logger, f"< jediTaskID={taskSpec.jediTaskID} >")
+        tmpLog.info(f"start label={taskSpec.prodSourceLabel} taskType={taskSpec.taskType}")
+
+        # returns
+        retFatal = self.SC_FATAL
+        retOK = self.SC_SUCCEEDED
+
+        try:
+            # get DDM I/F
+            ddmIF = self.ddmIF.getInterface(taskSpec.vo)
+            # register datasets
+            if datasetToRegister != [] or taskSpec.prodSourceLabel in ["user"]:
+                # prod vs anal
+                userSetup = False
+                if taskSpec.prodSourceLabel in ["user"]:
+                    userSetup = True
+                    # collect datasetID to register datasets/containers just in case
+                    for tmpPandaJob in pandaJobs:
+                        if not tmpPandaJob.produceUnMerge():
+                            for tmpFileSpec in tmpPandaJob.Files:
+                                if tmpFileSpec.type in ["output", "log"]:
+                                    if tmpFileSpec.datasetID not in datasetToRegister:
+                                        datasetToRegister.append(tmpFileSpec.datasetID)
+                tmpLog.info(f"datasetToRegister={str(datasetToRegister)}")
+                # get site mapper
+                siteMapper = self.taskBufferIF.get_site_mapper()
+
+                # loop over all datasets
+                avDatasetList = []
+                cnDatasetMap = {}
+                for datasetID in datasetToRegister:
+                    # get output and log datasets
+                    tmpLog.info(f"getting datasetSpec with datasetID={datasetID}")
+                    tmpStat, datasetSpec = self.taskBufferIF.getDatasetWithID_JEDI(taskSpec.jediTaskID, datasetID)
+                    if not tmpStat:
+                        tmpLog.error("failed to get output and log datasets")
+                        return retFatal
+                    if datasetSpec.isPseudo():
+                        tmpLog.info("skip pseudo dataset")
+                        continue
+                    # DDM backend
+                    ddmBackEnd = taskSpec.getDdmBackEnd()
+                    tmpLog.info(f"checking {datasetSpec.datasetName}")
+                    # secondary nucleus
+                    nucleusSpec = siteMapper.getNucleus(taskSpec.nucleus)
+                    secNucleusSpecBase = None
+                    if nucleusSpec and not taskSpec.is_intermediate_task():
+                        secondaryNucleus = nucleusSpec.get_secondary_nucleus()
+                        if secondaryNucleus:
+                            secNucleusSpecBase = siteMapper.getNucleus(secondaryNucleus)
+                    # check if dataset and container are available in DDM
+                    for targetName in [datasetSpec.datasetName, datasetSpec.containerName]:
+                        if targetName is None:
+                            continue
+                        if targetName not in avDatasetList:
+                            # set lifetime
+                            if targetName.startswith("panda"):
+                                if datasetSpec.type == "trn_log" and taskSpec.prodSourceLabel == "managed":
+                                    lifetime = 365
+                                else:
+                                    lifetime = 14
+                            else:
+                                lifetime = None
+                            # check dataset/container in DDM
+                            tmpList = ddmIF.listDatasets(targetName)
+                            if tmpList == []:
+                                # get location
+                                location = None
+                                locForRule = None
+                                if targetName == datasetSpec.datasetName:
+                                    # dataset
+                                    if datasetSpec.site in ["", None]:
+                                        if DataServiceUtils.getDistributedDestination(datasetSpec.storageToken) is not None:
+                                            locForRule = datasetSpec.destination
+                                        elif DataServiceUtils.getDestinationSE(datasetSpec.storageToken) is not None:
+                                            location = DataServiceUtils.getDestinationSE(datasetSpec.storageToken)
+                                    else:
+                                        tmpLog.info(f"site={datasetSpec.site} token={datasetSpec.storageToken}")
+                                        location = siteMapper.getDdmEndpoint(
+                                            datasetSpec.site,
+                                            datasetSpec.storageToken,
+                                            taskSpec.prodSourceLabel,
+                                            JobUtils.translate_tasktype_to_jobtype(taskSpec.taskType),
+                                        )
+                                if locForRule is None:
+                                    locForRule = location
+                                # set metadata
+                                if taskSpec.prodSourceLabel in ["managed", "test"] and targetName == datasetSpec.datasetName:
+                                    metaData = {}
+                                    metaData["task_id"] = taskSpec.jediTaskID
+                                    if taskSpec.campaign not in [None, ""]:
+                                        metaData["campaign"] = taskSpec.campaign
+                                    if datasetSpec.getTransient() is not None:
+                                        metaData["transient"] = datasetSpec.getTransient()
+                                else:
+                                    metaData = None
+                                # use secondary nucleus only for production output
+                                if (
+                                    secNucleusSpecBase
+                                    and taskSpec.prodSourceLabel in ["managed", "test"]
+                                    and targetName == datasetSpec.datasetName
+                                    and datasetSpec.type == "output"
+                                ):
+                                    secNucleusSpec = secNucleusSpecBase
+                                else:
+                                    secNucleusSpec = None
+                                # register dataset/container
+                                tmpLog.info(f"registering {targetName} with location={location} backend={ddmBackEnd} lifetime={lifetime} meta={str(metaData)}")
+                                tmpStat = ddmIF.registerNewDataset(targetName, backEnd=ddmBackEnd, location=location, lifetime=lifetime, metaData=metaData)
+                                if not tmpStat:
+                                    tmpLog.error(f"failed to register {targetName}")
+                                    return retFatal
+                                # procedures for user
+                                if userSetup or DataServiceUtils.getDistributedDestination(datasetSpec.storageToken) is not None or secNucleusSpec:
+                                    # register location
+                                    tmpToRegister = False
+                                    if userSetup and targetName == datasetSpec.datasetName and datasetSpec.site not in ["", None]:
+                                        if taskSpec.workingGroup:
+                                            userName = taskSpec.workingGroup
+                                        else:
+                                            userName = taskSpec.userName
+                                        grouping = None
+                                        tmpToRegister = True
+                                    elif DataServiceUtils.getDistributedDestination(datasetSpec.storageToken) is not None:
+                                        userName = None
+                                        grouping = "NONE"
+                                        tmpToRegister = True
+                                    elif secNucleusSpec:
+                                        userName = None
+                                        grouping = None
+                                        tmpToRegister = True
+                                        locForRule = siteMapper.getDdmEndpoint(
+                                            secNucleusSpec.getOnePandaSite(),
+                                            datasetSpec.storageToken,
+                                            taskSpec.prodSourceLabel,
+                                            JobUtils.translate_tasktype_to_jobtype(taskSpec.taskType),
+                                        )
+
+                                    if tmpToRegister:
+                                        activity = DataServiceUtils.getActivityForOut(taskSpec.prodSourceLabel)
+                                        tmpLog.info(
+                                            "registering location={} lifetime={} days activity={} grouping={} "
+                                            "owner={}".format(locForRule, lifetime, activity, grouping, userName)
+                                        )
+                                        tmpStat = ddmIF.registerDatasetLocation(
+                                            targetName, locForRule, owner=userName, lifetime=lifetime, backEnd=ddmBackEnd, activity=activity, grouping=grouping
+                                        )
+                                        if not tmpStat:
+                                            tmpLog.error(f"failed to register location {locForRule} for {targetName}")
+                                            return retFatal
+                                        # double copy
+                                        if userSetup and datasetSpec.type == "output":
+                                            if datasetSpec.destination != datasetSpec.site:
+                                                tmpLog.info(f"skip making double copy as destination={datasetSpec.destination} is not site={datasetSpec.site}")
+                                            else:
+                                                second_copy = True
+                                                try:
+                                                    if taskSpec.site:
+                                                        panda_site = siteMapper.getSite(taskSpec.site)
+                                                        if panda_site.catchall and "skip_2nd_copy" in panda_site.catchall:
+                                                            tmpLog.info(f"skip making double copy as specified in {panda_site} catchall")
+                                                            second_copy = False
+                                                except Exception:
+                                                    second_copy = True
+
+                                                if second_copy:
+                                                    locForDouble = f"(type=SCRATCHDISK)\\notforextracopy=True\\{locForRule}"
+                                                    tmpMsg = "registering double copy "
+                                                    tmpMsg += f'location="{locForDouble}" lifetime={lifetime}days activity={activity} for dataset={targetName}'
+                                                    tmpLog.info(tmpMsg)
+                                                    tmpStat = ddmIF.registerDatasetLocation(
+                                                        targetName,
+                                                        locForDouble,
+                                                        copies=1,
+                                                        owner=userName,
+                                                        lifetime=lifetime,
+                                                        activity=activity,
+                                                        grouping="NONE",
+                                                        weight="freespace",
+                                                        ignore_availability=False,
+                                                    )
+                                                    if not tmpStat:
+                                                        tmpLog.error(f"failed to register double copy location {locForDouble} for {targetName}")
+                                                        return retFatal
+                                    # container-level rules for user
+                                    if userSetup and datasetSpec.type == "output" and targetName == datasetSpec.containerName:
+                                        # rule to keep the outputs where they are produced
+                                        container_location = "type=SCRATCHDISK"
+                                        tmpLog.info(f"registering container-level rule for {targetName}")
+                                        tmpStat = ddmIF.registerDatasetLocation(
+                                            targetName,
+                                            container_location,
+                                            owner=userName,
+                                            lifetime=self.user_container_lifetime,
+                                            backEnd=ddmBackEnd,
+                                            activity=activity,
+                                            grouping="DATASET",
+                                        )
+                                        if not tmpStat:
+                                            tmpLog.error(f"failed to register location {container_location} for container {targetName}")
+                                            return retFatal
+                                        # rule with 2 copies and no grouping
+                                        container_location = f"(type=SCRATCHDISK)\\notforextracopy=True"
+                                        tmpLog.info(f"registering container-level 2nd copy rule for {targetName}")
+                                        tmpStat = ddmIF.registerDatasetLocation(
+                                            targetName,
+                                            container_location,
+                                            copies=1,
+                                            owner=userName,
+                                            lifetime=self.user_container_lifetime,
+                                            activity=activity,
+                                            grouping="NONE",
+                                            weight="freespace",
+                                            ignore_availability=False,
+                                        )
+                                        if not tmpStat:
+                                            tmpLog.error(f"failed to register 2nd copy location {container_location} for container {targetName}")
+                                            return retFatal
+                                avDatasetList.append(targetName)
+                            else:
+                                tmpLog.info(f"{targetName} already registered")
+                    # check if dataset is in the container
+                    if datasetSpec.containerName is not None and datasetSpec.containerName != datasetSpec.datasetName:
+                        # get list of constituent datasets in the container
+                        if datasetSpec.containerName not in cnDatasetMap:
+                            cnDatasetMap[datasetSpec.containerName] = ddmIF.listDatasetsInContainer(datasetSpec.containerName)
+                        # add dataset
+                        if datasetSpec.datasetName not in cnDatasetMap[datasetSpec.containerName]:
+                            tmpLog.info(f"adding {datasetSpec.datasetName} to {datasetSpec.containerName}")
+                            tmpStat = ddmIF.addDatasetsToContainer(datasetSpec.containerName, [datasetSpec.datasetName], backEnd=ddmBackEnd)
+                            if not tmpStat:
+                                tmpLog.error(f"failed to add {datasetSpec.datasetName} to {datasetSpec.containerName}")
+                                return retFatal
+                            cnDatasetMap[datasetSpec.containerName].append(datasetSpec.datasetName)
+                        else:
+                            tmpLog.info(f"{datasetSpec.datasetName} already in {datasetSpec.containerName}")
+                    # update dataset
+                    datasetSpec.status = "registered"
+                    self.taskBufferIF.updateDataset_JEDI(datasetSpec, {"jediTaskID": taskSpec.jediTaskID, "datasetID": datasetID})
+            # register ES datasets
+            if taskSpec.registerEsFiles():
+                targetName = EventServiceUtils.getEsDatasetName(taskSpec.jediTaskID)
+                location = None
+                metaData = {}
+                metaData["task_id"] = taskSpec.jediTaskID
+                metaData["hidden"] = True
+                tmpLog.info(f"registering ES dataset {targetName} with location={location} meta={str(metaData)}")
+                tmpStat = ddmIF.registerNewDataset(targetName, location=location, metaData=metaData, resurrect=True)
+                if not tmpStat:
+                    tmpLog.error(f"failed to register ES dataset {targetName}")
+                    return retFatal
+                # register rule
+                location = "type=DATADISK"
+                activity = DataServiceUtils.getActivityForOut(taskSpec.prodSourceLabel)
+                grouping = "NONE"
+                tmpLog.info(f"registering location={location} activity={activity} grouping={grouping}")
+                tmpStat = ddmIF.registerDatasetLocation(targetName, location, activity=activity, grouping=grouping)
+                if not tmpStat:
+                    tmpLog.error(f"failed to register location {location} with {activity} for {targetName}")
+                    return retFatal
+            # open datasets
+            if taskSpec.prodSourceLabel in ["managed", "test"]:
+                # get the list of output/log datasets
+                outDatasetList = []
+                for tmpPandaJob in pandaJobs:
+                    for tmpFileSpec in tmpPandaJob.Files:
+                        if tmpFileSpec.type in ["output", "log"]:
+                            if tmpFileSpec.destinationDBlock not in outDatasetList:
+                                outDatasetList.append(tmpFileSpec.destinationDBlock)
+                # open datasets
+                for outDataset in outDatasetList:
+                    tmpLog.info(f"open {outDataset}")
+                    ddmIF.openDataset(outDataset)
+                    # unset lifetime
+                    ddmIF.setDatasetMetadata(outDataset, "lifetime", None)
+            # return
+            tmpLog.info("done")
+            return retOK
+        except Exception as e:
+            tmpLog.error(f"doSetup failed with {str(e)}")
+            tb = traceback.format_exc()
+            taskSpec.setErrDiag(tmpLog.uploadLog(taskSpec.jediTaskID))
+            tmpLog.error(tb)
+            return retFatal

pandajedi/jedisetup/GenTaskSetupper.py

@@ -0,0 +1,16 @@
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from .TaskSetupperBase import TaskSetupperBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# task setup for general purpose
+class GenTaskSetupper(TaskSetupperBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskSetupperBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main to setup task
+    def doSetup(self, taskSpec, datasetToRegister, pandaJobs):
+        return self.SC_SUCCEEDED

pandajedi/jedisetup/SimpleTaskSetupper.py

@@ -0,0 +1,152 @@
+import traceback
+
+from pandacommon.pandalogger.PandaLogger import PandaLogger
+
+from pandajedi.jedicore.MsgWrapper import MsgWrapper
+
+from .TaskSetupperBase import TaskSetupperBase
+
+logger = PandaLogger().getLogger(__name__.split(".")[-1])
+
+
+# task setup for general purpose
+class SimpleTaskSetupper(TaskSetupperBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskSetupperBase.__init__(self, taskBufferIF, ddmIF)
+
+    # main to setup task
+    def doSetup(self, taskSpec, datasetToRegister, pandaJobs):
+        # make logger
+        tmpLog = MsgWrapper(logger, f"< jediTaskID={taskSpec.jediTaskID} >")
+        tmpLog.info(f"start label={taskSpec.prodSourceLabel} taskType={taskSpec.taskType}")
+        # returns
+        retFatal = self.SC_FATAL
+        retOK = self.SC_SUCCEEDED
+        try:
+            # get DDM I/F
+            ddmIF = self.ddmIF.getInterface(taskSpec.vo, taskSpec.cloud)
+            # skip if DDM I/F is inactive
+            if not ddmIF:
+                tmpLog.info("skip due to inactive DDM I/F")
+                return retOK
+            # collect datasetID to register datasets/containers just in case
+            for tmpPandaJob in pandaJobs:
+                if not tmpPandaJob.produceUnMerge():
+                    for tmpFileSpec in tmpPandaJob.Files:
+                        if tmpFileSpec.type in ["output", "log"]:
+                            if tmpFileSpec.datasetID not in datasetToRegister:
+                                datasetToRegister.append(tmpFileSpec.datasetID)
+            # register datasets
+            if datasetToRegister:
+                tmpLog.info(f"datasetToRegister={str(datasetToRegister)}")
+                # get site mapper
+                siteMapper = self.taskBufferIF.get_site_mapper()
+
+                # loop over all datasets
+                avDatasetList = []
+                cnDatasetMap = {}
+                ddmBackEnd = "rucio"
+                for datasetID in datasetToRegister:
+                    # get output and log datasets
+                    tmpLog.info(f"getting datasetSpec with datasetID={datasetID}")
+                    tmpStat, datasetSpec = self.taskBufferIF.getDatasetWithID_JEDI(taskSpec.jediTaskID, datasetID)
+                    if not tmpStat:
+                        tmpLog.error("failed to get output and log datasets")
+                        return retFatal
+                    if datasetSpec.isPseudo():
+                        tmpLog.info("skip pseudo dataset")
+                        continue
+
+                    # set lifetime
+                    key_name = f"DATASET_LIFETIME_{datasetSpec.type}"
+                    lifetime = self.taskBufferIF.getConfigValue("task_setup", key_name, "jedi", taskSpec.cloud)
+                    if not lifetime:
+                        lifetime = self.taskBufferIF.getConfigValue("task_setup", key_name, "jedi", taskSpec.vo)
+
+                    tmpLog.info(f"checking {datasetSpec.datasetName}")
+                    # check if dataset and container are available in DDM
+                    for targetName in [datasetSpec.datasetName, datasetSpec.containerName]:
+                        if not targetName:
+                            continue
+                        if targetName in avDatasetList:
+                            tmpLog.info(f"{targetName} already registered")
+                            continue
+                        # check dataset/container in DDM
+                        tmpList = ddmIF.listDatasets(targetName)
+                        if not tmpList:
+                            # get location
+                            location = None
+                            locForRule = None
+                            if targetName == datasetSpec.datasetName:
+                                # dataset
+                                tmpLog.info(f"dest={datasetSpec.destination}")
+                                if datasetSpec.destination:
+                                    if siteMapper.checkSite(datasetSpec.destination):
+                                        location = siteMapper.getSite(datasetSpec.destination).ddm_output["default"]
+                                    else:
+                                        location = datasetSpec.destination
+                            if locForRule is None:
+                                locForRule = location
+                            # set metadata
+                            if targetName == datasetSpec.datasetName:
+                                metaData = {}
+                                metaData["task_id"] = taskSpec.jediTaskID
+                                if taskSpec.campaign:
+                                    metaData["campaign"] = taskSpec.campaign
+                            else:
+                                metaData = None
+                            # register dataset/container
+                            tmpLog.info(f"registering {targetName} with location={location} backend={ddmBackEnd} lifetime={lifetime} meta={str(metaData)}")
+                            tmpStat = ddmIF.registerNewDataset(targetName, backEnd=ddmBackEnd, location=location, lifetime=lifetime, metaData=metaData)
+                            if not tmpStat:
+                                tmpLog.error(f"failed to register {targetName}")
+                                return retFatal
+                            # register location
+                            if locForRule:
+                                """
+                                if taskSpec.workingGroup:
+                                    userName = taskSpec.workingGroup
+                                else:
+                                    userName = taskSpec.userName
+                                """
+                                userName = None
+                                activity = None
+                                grouping = None
+                                tmpLog.info(
+                                    f"registering location={locForRule} lifetime={lifetime} days activity={activity} grouping={grouping} owner={userName}"
+                                )
+                                tmpStat = ddmIF.registerDatasetLocation(
+                                    targetName, locForRule, owner=userName, lifetime=lifetime, backEnd=ddmBackEnd, activity=activity, grouping=grouping
+                                )
+                                if not tmpStat:
+                                    tmpLog.error(f"failed to register location {locForRule} for {targetName}")
+                                    return retFatal
+                            avDatasetList.append(targetName)
+
+                    # check if dataset is in the container
+                    if datasetSpec.containerName and datasetSpec.containerName != datasetSpec.datasetName:
+                        # get list of constituent datasets in the container
+                        if datasetSpec.containerName not in cnDatasetMap:
+                            cnDatasetMap[datasetSpec.containerName] = ddmIF.listDatasetsInContainer(datasetSpec.containerName)
+                        # add dataset
+                        if datasetSpec.datasetName not in cnDatasetMap[datasetSpec.containerName]:
+                            tmpLog.info(f"adding {datasetSpec.datasetName} to {datasetSpec.containerName}")
+                            tmpStat = ddmIF.addDatasetsToContainer(datasetSpec.containerName, [datasetSpec.datasetName], backEnd=ddmBackEnd)
+                            if not tmpStat:
+                                tmpLog.error(f"failed to add {datasetSpec.datasetName} to {datasetSpec.containerName}")
+                                return retFatal
+                            cnDatasetMap[datasetSpec.containerName].append(datasetSpec.datasetName)
+                        else:
+                            tmpLog.info(f"{datasetSpec.datasetName} already in {datasetSpec.containerName}")
+                    # update dataset
+                    datasetSpec.status = "registered"
+                    self.taskBufferIF.updateDataset_JEDI(datasetSpec, {"jediTaskID": taskSpec.jediTaskID, "datasetID": datasetID})
+            # return
+            tmpLog.info("done")
+            return retOK
+        except Exception as e:
+            errStr = f"doSetup failed with {str(e)}"
+            tmpLog.error(errStr + traceback.format_exc())
+            taskSpec.setErrDiag(errStr)
+            return retFatal

pandajedi/jedisetup/TaskSetupperBase.py

@@ -0,0 +1,15 @@
+from pandajedi.jedicore import Interaction
+
+
+# base class for task setup
+class TaskSetupperBase(object):
+    def __init__(self, taskBufferIF, ddmIF):
+        self.ddmIF = ddmIF
+        self.taskBufferIF = taskBufferIF
+        self.refresh()
+
+    def refresh(self):
+        self.siteMapper = self.taskBufferIF.get_site_mapper()
+
+
+Interaction.installSC(TaskSetupperBase)

pandajedi/jeditest/SchemaChecker.py

@@ -0,0 +1,41 @@
+"""
+Checking DB schema version for PanDA JEDI.
+If there is an issue and the pandaEmailNotification var is
+defined in panda_config, it will send an email notification.
+"""
+
+from packaging import version
+
+from pandajedi.jediconfig import jedi_config
+from pandajedi.jedicore import JediDBSchemaInfo
+from pandajedi.jedicore.JediDBProxy import DBProxy
+from pandaserver.config import panda_config
+from pandaserver.srvcore.MailUtils import MailUtils
+
+proxyS = DBProxy()
+proxyS.connect(jedi_config.db.dbhost, jedi_config.db.dbpasswd, jedi_config.db.dbuser, jedi_config.db.dbname)
+
+sql = "select major || '.' || minor || '.' || patch from ATLAS_PANDA.pandadb_version where component = 'PanDA'"
+
+res = proxyS.querySQL(sql)
+dbVersion = res[0][0]
+
+serverDBVersion = JediDBSchemaInfo.JediDBSchemaInfo().method()
+
+if version.parse(dbVersion) >= version.parse(serverDBVersion):
+    print("DB schema check: OK")
+else:
+    msgBody = (
+        "There is an issue with "
+        + panda_config.pserveralias
+        + ". PanDA DB schema installed is "
+        + dbVersion
+        + " while PanDA JEDI requires version "
+        + serverDBVersion
+        + " to be installed. Please check the official docs for instructions on how to upgrade the schema."
+    )
+    print(msgBody)
+    if "pandaEmailNotification" in panda_config.__dict__:
+        MailUtils().send(
+            panda_config.pandaEmailNotification, "PanDA DB Version installed is not correct for JEDI running on " + panda_config.pserveralias, msgBody
+        )

pandajedi/jeditest/addContWoSoTask.py

@@ -0,0 +1,54 @@
+import sys
+import uuid
+
+from userinterface import Client
+
+inFileList = ["file1", "file2", "file3"]
+
+logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
+
+taskParamMap = {}
+
+taskParamMap["nFilesPerJob"] = 1
+taskParamMap["nFiles"] = len(inFileList)
+# taskParamMap['nEventsPerInputFile']  = 10000
+# taskParamMap['nEventsPerJob'] = 10000
+# taskParamMap['nEvents'] = 25000
+taskParamMap["noInput"] = True
+taskParamMap["pfnList"] = inFileList
+# taskParamMap['mergeOutput'] = True
+taskParamMap["taskName"] = str(uuid.uuid4())
+taskParamMap["userName"] = "someone"
+taskParamMap["vo"] = "wlcg"
+taskParamMap["taskPriority"] = 900
+# taskParamMap['reqID'] = reqIdx
+taskParamMap["architecture"] = ""
+taskParamMap["transUses"] = ""
+taskParamMap["transHome"] = None
+taskParamMap["transPath"] = "https://atlpan.web.cern.ch/atlpan/bash-c"
+taskParamMap["processingType"] = "step1"
+taskParamMap["prodSourceLabel"] = "test"
+taskParamMap["taskType"] = "test"
+taskParamMap["workingGroup"] = "lsst"
+# taskParamMap['coreCount'] = 1
+# taskParamMap['walltime'] = 1
+taskParamMap["cloud"] = "US"
+taskParamMap["site"] = "BNL_OSG_1"
+
+"""
+taskParamMap['log'] = {'dataset': logDatasetName,
+                       'type':'template',
+                       'param_type':'log',
+                       'token':'local',
+                       'destination':'local',
+                       'value':'{0}.${{SN}}.log.tgz'.format(logDatasetName)}
+"""
+
+taskParamMap["jobParameters"] = [
+    {
+        "type": "constant",
+        "value": 'singularity exec docker://busybox echo "hello world with ${IN/L}"',
+    },
+]
+
+print(Client.insertTaskParams(taskParamMap))

pandajedi/jeditest/addDataCarouselRequest.py

@@ -0,0 +1,47 @@
+import sys
+
+from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
+from pandajedi.jediddm.DDMInterface import DDMInterface
+from pandajedi.jedirefine import RefinerUtils
+from pandaserver.taskbuffer.DataCarousel import DataCarouselInterface
+
+vo = "atlas"
+jediTaskID = int(sys.argv[1])
+
+print("set tbIF")
+tbIF = JediTaskBufferInterface()
+tbIF.setupInterface(max_size=1)
+
+print("set ddmIF")
+ddmIF = DDMInterface()
+ddmIF.setupInterface()
+
+print("set DCIF")
+data_carousel_interface = DataCarouselInterface(tbIF, ddmIF.getInterface(vo))
+if data_carousel_interface is None:
+    # data carousel interface is undefined
+    errStr = f"data carousel interface is undefined for vo={vo}"
+    print(errStr)
+    sys.exit(1)
+
+print(f"get task params of {jediTaskID}")
+taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID)
+taskParamMap = RefinerUtils.decodeJSON(taskParam)
+
+print(f"get_input_datasets_to_prestage")
+prestaging_list, ret_map = data_carousel_interface.get_input_datasets_to_prestage(jediTaskID, taskParamMap)
+
+if not prestaging_list:
+    # no dataset needs pre-staging; unset inputPreStaging
+    print("no need to prestage")
+    # resume the task from staging
+    print(f"resume task {jediTaskID}")
+    tbIF.sendCommandTaskPanda(jediTaskID, "Test addDataCarouselRequest. No need to prestage. Resumed from staging", True, "resume")
+else:
+    # submit data carousel requests for dataset to pre-stage
+    print("to prestage, submitting data carousel requests")
+    tmp_ret = data_carousel_interface.submit_data_carousel_requests(jediTaskID, prestaging_list)
+    if tmp_ret:
+        print(f"submitted data carousel requests for {jediTaskID}: {tmp_ret}")
+    else:
+        print(f"failed to submit data carousel requests")