panda-server-source 0.8.0__py3-none-any.whl

pandajedi/jedirefine/AtlasAnalTaskRefiner.py

@@ -0,0 +1,259 @@
+import json
+import random
+import re
+import traceback
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice import DataServiceUtils
+from pandaserver.taskbuffer import JobUtils
+from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
+
+from .TaskRefinerBase import TaskRefinerBase
+
+
+# brokerage for ATLAS analysis
+class AtlasAnalTaskRefiner(TaskRefinerBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
+
+    # extract common parameters
+    def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule):
+        processingTypes = taskParamMap["processingType"].split("-")
+        # set ddmBackEnd
+        if "ddmBackEnd" not in taskParamMap:
+            taskParamMap["ddmBackEnd"] = "rucio"
+        # set sourceURL
+        try:
+            if "sourceURL" in taskParamMap:
+                for tmpItem in taskParamMap["jobParameters"]:
+                    if "value" in tmpItem:
+                        tmpItem["value"] = re.sub("\$\{SURL\}", taskParamMap["sourceURL"], tmpItem["value"])
+        except Exception:
+            pass
+        if hasattr(panda_config, "wn_script_base_url"):
+            base_url = panda_config.wn_script_base_url
+        else:
+            protocol = "https" if panda_config.pserverportcache == 443 else "http"
+            base_url = f"{protocol}://{panda_config.pserveralias}:{panda_config.pserverportcache}"
+        # set transPath
+        if "transPath" not in taskParamMap:
+            if "athena" in processingTypes:
+                # athena
+                taskParamMap["transPath"] = f"{base_url}/trf/user/runAthena-00-00-12"
+            elif "cont" in processingTypes:
+                # container
+                taskParamMap["transPath"] = f"{base_url}/trf/user/runcontainer"
+            else:
+                # general executable
+                taskParamMap["transPath"] = f"{base_url}/trf/user/runGen-00-00-02"
+                # shorter base walltime
+                if "baseWalltime" not in taskParamMap:
+                    taskParamMap["baseWalltime"] = 60
+        # set transPath for build
+        if "buildSpec" in taskParamMap and "transPath" not in taskParamMap["buildSpec"]:
+            if "athena" in processingTypes:
+                # athena
+                taskParamMap["buildSpec"]["transPath"] = f"{base_url}/trf/user/buildJob-00-00-03"
+            else:
+                # general executable
+                taskParamMap["buildSpec"]["transPath"] = f"{base_url}/trf/user/buildGen-00-00-01"
+        # set transPath for preprocessing
+        if "preproSpec" in taskParamMap and "transPath" not in taskParamMap["preproSpec"]:
+            if "evp" in processingTypes:
+                # event picking
+                taskParamMap["preproSpec"]["transPath"] = f"{base_url}/trf/user/preEvtPick-00-00-01"
+            elif "grl" in processingTypes:
+                # good run list
+                taskParamMap["preproSpec"]["transPath"] = f"{base_url}/trf/user/preGoodRunList-00-00-01"
+        # set transPath for merge
+        if "mergeSpec" in taskParamMap and "transPath" not in taskParamMap["mergeSpec"]:
+            taskParamMap["mergeSpec"]["transPath"] = f"{base_url}/trf/user/runMerge-00-00-02"
+        # min ram count
+        if "ramCount" not in taskParamMap:
+            taskParamMap["ramCount"] = 2000
+            taskParamMap["ramUnit"] = "MBPerCore"
+        # disk count
+        if "outDiskCount" not in taskParamMap:
+            out_disk_count_default = self.taskBufferIF.getConfigValue("taskrefiner", "OUTDISKCOUNT_ANALY_KB", "jedi", "atlas")
+            if out_disk_count_default is None or out_disk_count_default < 0:
+                out_disk_count_default = 500
+            taskParamMap["outDiskCount"] = out_disk_count_default
+            taskParamMap["outDiskUnit"] = "kB"
+        # set cpu time unit to use HS06
+        if "cpuTimeUnit" not in taskParamMap:
+            taskParamMap["cpuTimeUnit"] = "HS06sPerEvent"
+        # use local IO for ancient releases since inputfilepeeker+xrootd is problematic
+        if "transUses" in taskParamMap and taskParamMap["transUses"]:
+            try:
+                ver = taskParamMap["transUses"].split("-")[1]
+                m = re.search("^(\d{2})\.(\d{2})\.", ver)
+                if m is not None:
+                    major = int(m.group(1))
+                    minor = int(m.group(2))
+                    if major < 20 or (major == 20 and minor <= 20):
+                        taskParamMap["useLocalIO"] = 1
+            except Exception:
+                pass
+        # scout success rate
+        if "scoutSuccessRate" not in taskParamMap:
+            taskParamMap["scoutSuccessRate"] = 5
+        # directIO
+        if "useLocalIO" not in taskParamMap and "allowInputLAN" not in taskParamMap:
+            taskParamMap["allowInputLAN"] = "use"
+        # current priority
+        if "currentPriority" in taskParamMap and (taskParamMap["currentPriority"] < 900 or taskParamMap["currentPriority"] > 1100):
+            taskParamMap["currentPriority"] = 1000
+        isSU, isSG = self.taskBufferIF.isSuperUser(taskParamMap["userName"])
+        if isSU or (isSG and "workingGroup" in taskParamMap):
+            # super high priority to jump over others
+            if "currentPriority" not in taskParamMap or taskParamMap["currentPriority"] < JobUtils.priorityTasksToJumpOver:
+                taskParamMap["currentPriority"] = JobUtils.priorityTasksToJumpOver
+        # max attempts
+        if "maxAttempt" not in taskParamMap:
+            taskParamMap["maxAttempt"] = 10
+        if "maxFailure" not in taskParamMap:
+            taskParamMap["maxFailure"] = 3
+        # target walltime
+        if "maxWalltime" not in taskParamMap:
+            tgtWalltime = self.taskBufferIF.getConfigValue("taskrefiner", "USER_JOB_TARGET_WALLTIME", "jedi", "atlas")
+            if tgtWalltime:
+                taskParamMap["maxWalltime"] = tgtWalltime
+        # choose N % of tasks to enable input data motion
+        fracTaskWithDataMotion = self.taskBufferIF.getConfigValue("taskrefiner", "USER_TASKS_MOVE_INPUT", "jedi", "atlas")
+        if fracTaskWithDataMotion is not None and fracTaskWithDataMotion > 0:
+            if random.randint(1, 100) <= fracTaskWithDataMotion:
+                if "currentPriority" not in taskParamMap:
+                    taskParamMap["currentPriority"] = taskParamMap["taskPriority"]
+                taskParamMap["taskPriority"] = 1001
+        # image name
+        if "container_name" not in taskParamMap:
+            try:
+                for tmpItem in taskParamMap["jobParameters"]:
+                    if "value" in tmpItem:
+                        tmpM = re.search("--containerImage\s+([^\s]+)", tmpItem["value"])
+                        if tmpM is not None:
+                            taskParamMap["container_name"] = tmpM.group(1)
+                            break
+            except Exception:
+                pass
+        # extract container name from architecture
+        if "container_name" not in taskParamMap and "architecture" in taskParamMap and isinstance(taskParamMap["architecture"], str):
+            try:
+                # json-serialized architecture
+                check_str = json.loads(taskParamMap["architecture"]).get("encoded_platform", "")
+            except Exception:
+                # encoded architecture
+                check_str = taskParamMap["architecture"]
+            m = re.search(r"\+([^\s+@#&]+)", check_str)
+            if m:
+                # use the architecture as container name
+                tmp_container_name = m.group(1)
+                taskParamMap["container_name"] = tmp_container_name
+                # remove the container name from architecture
+                tmp_architecture = re.sub(rf"\+{tmp_container_name}", "", taskParamMap["architecture"])
+                taskParamMap["architecture"] = tmp_architecture
+                self.tmpLog.debug(f"tweaked architecture: container_name={tmp_container_name} new_architecture={tmp_architecture}")
+
+        # message driven, choose N % of tasks to enable
+        if "messageDriven" not in taskParamMap:
+            analy_md_percent = self.taskBufferIF.getConfigValue("taskrefiner", "USER_TASKS_MESSAGE_DRIVEN_PERCENT", "jedi", "atlas")
+            if analy_md_percent and random.uniform(0, 100) <= analy_md_percent:
+                taskParamMap["messageDriven"] = True
+        # push status changes, choose N % of tasks to enable
+        if "pushStatusChanges" not in taskParamMap:
+            analy_pc_percent = self.taskBufferIF.getConfigValue("taskrefiner", "USER_TASKS_PUSH_STATUS_CHANGES_PERCENT", "jedi", "atlas")
+            if analy_pc_percent and random.uniform(0, 100) <= analy_pc_percent:
+                taskParamMap["pushStatusChanges"] = True
+        # disable skipScout fot tasks without for group production role
+        if "skipScout" in taskParamMap and ("official" not in taskParamMap or not taskParamMap["official"]):
+            del taskParamMap["skipScout"]
+        # max core count
+        if "maxCoreCount" not in taskParamMap:
+            max_core_count = self.taskBufferIF.getConfigValue("taskrefiner", "USER_TASKS_MAX_CORE_COUNT", "jedi", "atlas")
+            if max_core_count is None:
+                max_core_count = 16
+            taskParamMap["maxCoreCount"] = max_core_count
+        # set forceStaged when input dataset is RAW
+        if "forceStaged" not in taskParamMap:
+            for item in taskParamMap.get("jobParameters", []):
+                if item["type"] == "template" and "dataset" in item and item["param_type"] == "input":
+                    dataset_name = item["dataset"]
+                    if DataServiceUtils.getDatasetType(dataset_name) == "RAW":
+                        taskParamMap["forceStaged"] = True
+                        break
+        # update task parameters
+        self.updatedTaskParams = taskParamMap
+        # call base method
+        TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
+
+    # main
+    def doRefine(self, jediTaskID, taskParamMap):
+        # make logger
+        tmpLog = self.tmpLog
+        tmpLog.debug(f"start taskType={self.taskSpec.taskType}")
+        try:
+            # preprocessing
+            tmpStat, taskParamMap = self.doPreProRefine(taskParamMap)
+            if tmpStat is True:
+                tmpLog.debug("done for preprocessing")
+                return self.SC_SUCCEEDED
+            if tmpStat is False:
+                # failed
+                tmpLog.error("doPreProRefine failed")
+                return self.SC_FAILED
+            # normal refine
+            self.doBasicRefine(taskParamMap)
+            # set nosplit+repeat for DBR
+            for datasetSpec in self.inSecDatasetSpecList:
+                # get the latest version of DBR
+                if datasetSpec.datasetName == "DBR_LATEST":
+                    tmpLog.debug(f"resolving real name for {datasetSpec.datasetName}")
+                    datasetSpec.datasetName = self.ddmIF.getInterface(self.taskSpec.vo).getLatestDBRelease(useResultCache=3600)
+                    datasetSpec.containerName = datasetSpec.datasetName
+                # set attributes to DBR
+                if DataServiceUtils.isDBR(datasetSpec.datasetName):
+                    datasetSpec.attributes = "repeat,nosplit"
+            # check invalid characters
+            for datasetSpec in self.outDatasetSpecList:
+                if not DataServiceUtils.checkInvalidCharacters(datasetSpec.datasetName):
+                    errStr = f"invalid characters in {datasetSpec.datasetName}"
+                    tmpLog.error(errStr)
+                    self.taskSpec.setErrDiag(errStr, None)
+                    return self.SC_FATAL
+            # destination
+            if "destination" in taskParamMap:
+                for datasetSpec in self.outDatasetSpecList:
+                    datasetSpec.destination = taskParamMap["destination"]
+            # use build
+            if "buildSpec" in taskParamMap:
+                self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken["useBuild"])
+            # use template dataset
+            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken["instantiateTmpl"])
+            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken["instantiateTmplSite"])
+            for datasetSpec in self.outDatasetSpecList:
+                datasetSpec.type = f"tmpl_{datasetSpec.type}"
+            # get jobsetID
+            tmpStat, tmpJobID = self.taskBufferIF.getUserJobsetID_JEDI(self.taskSpec.userName)
+            if not tmpStat:
+                tmpLog.error("failed to get jobsetID failed")
+                return self.SC_FAILED
+            self.taskSpec.reqID = tmpJobID
+            # site limitation
+            if "excludedSite" in taskParamMap and "includedSite" in taskParamMap:
+                self.taskSpec.setLimitedSites("incexc")
+            elif "excludedSite" in taskParamMap:
+                self.taskSpec.setLimitedSites("exc")
+            elif "includedSite" in taskParamMap:
+                self.taskSpec.setLimitedSites("inc")
+            # input prestaging
+            if self.taskSpec.inputPreStaging():
+                # set first contents feed flag
+                self.taskSpec.set_first_contents_feed(True)
+        except Exception as e:
+            errStr = f"doRefine failed with {str(e)}"
+            tmpLog.error(f"{errStr} {traceback.format_exc()}")
+            self.taskSpec.setErrDiag(errStr, None)
+            raise e
+        tmpLog.debug("done")
+        return self.SC_SUCCEEDED

pandajedi/jedirefine/AtlasProdTaskRefiner.py

@@ -0,0 +1,162 @@
+import datetime
+import json
+import random
+import re
+import traceback
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+from pandajedi.jedicore import JediException
+from pandaserver.dataservice import DataServiceUtils
+
+from .TaskRefinerBase import TaskRefinerBase
+
+
+# brokerage for ATLAS production
+class AtlasProdTaskRefiner(TaskRefinerBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
+
+    # extract common parameters
+    def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule):
+        tmpLog = self.tmpLog
+        # set ddmBackEnd
+        if "ddmBackEnd" not in taskParamMap:
+            taskParamMap["ddmBackEnd"] = "rucio"
+        # get number of unprocessed events for event service
+        autoEsConversion = False
+        if "esConvertible" in taskParamMap and taskParamMap["esConvertible"] is True:
+            maxPrio = self.taskBufferIF.getConfigValue("taskrefiner", "AES_MAXTASKPRIORITY", "jedi", "atlas")
+            minPrio = self.taskBufferIF.getConfigValue("taskrefiner", "AES_MINTASKPRIORITY", "jedi", "atlas")
+            if maxPrio is not None and maxPrio < taskParamMap["taskPriority"]:
+                pass
+            elif minPrio is not None and minPrio > taskParamMap["taskPriority"]:
+                pass
+            else:
+                # get threshold
+                minNumEvents = self.taskBufferIF.getConfigValue("taskrefiner", "AES_EVENTPOOLSIZE", "jedi", "atlas")
+                maxPending = self.taskBufferIF.getConfigValue("taskrefiner", "AES_MAXPENDING", "jedi", "atlas")
+                nEvents, lastTaskTime, nPendingTasks = self.taskBufferIF.getNumUnprocessedEvents_JEDI(
+                    taskParamMap["vo"], taskParamMap["prodSourceLabel"], {"eventService": 1}, {"gshare": "Validation"}
+                )
+                tmpStr = "check for ES "
+                tmpStr += f"tot_num_unprocessed_events_AES={nEvents} target_num_events_AES={minNumEvents} last_AES_task_time={lastTaskTime} "
+                tmpStr += f"num_pending_tasks_AES={nPendingTasks} max_pending_tasks_AES={maxPending} "
+                tmpLog.info(tmpStr)
+                # not chane many tasks at once
+                if lastTaskTime is None or (lastTaskTime < naive_utcnow() - datetime.timedelta(minutes=5)):
+                    if minNumEvents is not None and nEvents < minNumEvents and maxPending is not None and (maxPending is None or maxPending > nPendingTasks):
+                        autoEsConversion = True
+                        tmpLog.info("will be converted to AES unless it goes to pending")
+        # add ES paramsters
+        if ("esFraction" in taskParamMap and taskParamMap["esFraction"] > 0) or ("esConvertible" in taskParamMap and taskParamMap["esConvertible"] is True):
+            tmpStr = "<PANDA_ES_ONLY>--eventService=True</PANDA_ES_ONLY>"
+            taskParamMap["jobParameters"].append({"type": "constant", "value": tmpStr})
+        if ("esFraction" in taskParamMap and taskParamMap["esFraction"] > 0) or autoEsConversion:
+            if "nEventsPerWorker" not in taskParamMap and (("esFraction" in taskParamMap and taskParamMap["esFraction"] > random.random()) or autoEsConversion):
+                taskParamMap["nEventsPerWorker"] = 1
+                taskParamMap["registerEsFiles"] = True
+                if "nEsConsumers" not in taskParamMap:
+                    tmpVal = self.taskBufferIF.getConfigValue("taskrefiner", "AES_NESCONSUMERS", "jedi", "atlas")
+                    if tmpVal is None:
+                        tmpVal = 1
+                    taskParamMap["nEsConsumers"] = tmpVal
+                if "nSitesPerJob" not in taskParamMap:
+                    tmpVal = self.taskBufferIF.getConfigValue("taskrefiner", "AES_NSITESPERJOB", "jedi", "atlas")
+                    if tmpVal is not None:
+                        taskParamMap["nSitesPerJob"] = tmpVal
+                if "mergeEsOnOS" not in taskParamMap:
+                    taskParamMap["mergeEsOnOS"] = True
+                if "maxAttemptES" not in taskParamMap:
+                    taskParamMap["maxAttemptES"] = 1
+                if "maxAttemptEsJob" not in taskParamMap:
+                    taskParamMap["maxAttemptEsJob"] = 0
+                if "notDiscardEvents" not in taskParamMap:
+                    taskParamMap["notDiscardEvents"] = True
+                if "decAttOnFailedES" not in taskParamMap:
+                    taskParamMap["decAttOnFailedES"] = True
+                taskParamMap["coreCount"] = 0
+                taskParamMap["resurrectConsumers"] = True
+        # push status changes, choose N % of tasks to enable
+        if "pushStatusChanges" not in taskParamMap:
+            prod_pc_percent = self.taskBufferIF.getConfigValue("taskrefiner", "PROD_TASKS_PUSH_STATUS_CHANGES_PERCENT", "jedi", "atlas")
+            if prod_pc_percent and random.uniform(0, 100) <= prod_pc_percent:
+                taskParamMap["pushStatusChanges"] = True
+        # serialize architecture
+        if "architecture" in taskParamMap and isinstance(taskParamMap["architecture"], dict):
+            # convert to string
+            taskParamMap["architecture"] = json.dumps(taskParamMap["architecture"])
+        # call base method
+        TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
+
+    # main
+    def doRefine(self, jediTaskID, taskParamMap):
+        # make logger
+        tmpLog = self.tmpLog
+        tmpLog.debug(f"start taskType={self.taskSpec.taskType}")
+        try:
+            # basic refine
+            self.doBasicRefine(taskParamMap)
+            # set nosplit+repeat for DBR
+            for datasetSpec in self.inSecDatasetSpecList:
+                if DataServiceUtils.isDBR(datasetSpec.datasetName):
+                    datasetSpec.attributes = "repeat,nosplit"
+            # enable consistency check
+            if self.taskSpec.parent_tid not in [None, self.taskSpec.jediTaskID]:
+                for datasetSpec in self.inMasterDatasetSpec:
+                    if datasetSpec.isMaster() and datasetSpec.type == "input":
+                        datasetSpec.enableCheckConsistency()
+            # append attempt number
+            for tmpKey, tmpOutTemplateMapList in self.outputTemplateMap.items():
+                for tmpOutTemplateMap in tmpOutTemplateMapList:
+                    outFileTemplate = tmpOutTemplateMap["filenameTemplate"]
+                    if re.search("\.\d+$", outFileTemplate) is None and not outFileTemplate.endswith(".panda.um"):
+                        tmpOutTemplateMap["filenameTemplate"] = outFileTemplate + ".1"
+            # extract input datatype
+            datasetTypeListIn = []
+            for datasetSpec in self.inMasterDatasetSpec + self.inSecDatasetSpecList:
+                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
+                if datasetType not in ["", None]:
+                    datasetTypeListIn.append(datasetType)
+            # extract datatype and set destination if necessary
+            datasetTypeList = []
+            for datasetSpec in self.outDatasetSpecList:
+                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
+                if datasetType not in ["", None]:
+                    datasetTypeList.append(datasetType)
+            # set numThrottled to use the task throttling mechanism
+            if "noThrottle" not in taskParamMap:
+                self.taskSpec.numThrottled = 0
+            # set to register datasets
+            self.taskSpec.setToRegisterDatasets()
+            # set transient to parent datasets
+            if self.taskSpec.processingType in ["merge"] and self.taskSpec.parent_tid not in [None, self.taskSpec.jediTaskID]:
+                # get parent
+                tmpStat, parentTaskSpec = self.taskBufferIF.getTaskDatasetsWithID_JEDI(self.taskSpec.parent_tid, None, False)
+                if tmpStat and parentTaskSpec is not None:
+                    # set transient to parent datasets
+                    metaData = {"transient": True}
+                    for datasetSpec in parentTaskSpec.datasetSpecList:
+                        if datasetSpec.type in ["log", "output"]:
+                            datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
+                            if datasetType not in ["", None] and datasetType in datasetTypeList and datasetType in datasetTypeListIn:
+                                tmpLog.info(
+                                    "set metadata={0} to parent jediTaskID={1}:datasetID={2}:Name={3}".format(
+                                        str(metaData), self.taskSpec.parent_tid, datasetSpec.datasetID, datasetSpec.datasetName
+                                    )
+                                )
+                                for metadataName, metadaValue in metaData.items():
+                                    self.ddmIF.getInterface(self.taskSpec.vo).setDatasetMetadata(datasetSpec.datasetName, metadataName, metadaValue)
+            # input prestaging
+            if self.taskSpec.inputPreStaging():
+                # set first contents feed flag
+                self.taskSpec.set_first_contents_feed(True)
+        except JediException.UnknownDatasetError as e:
+            tmpLog.debug(f"in doRefine. {str(e)}")
+            raise e
+        except Exception as e:
+            tmpLog.error(f"doRefine failed with {str(e)} {traceback.format_exc()}")
+            raise e
+        tmpLog.debug("done")
+        return self.SC_SUCCEEDED

pandajedi/jedirefine/GenTaskRefiner.py

@@ -0,0 +1,61 @@
+import re
+
+from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
+
+from .TaskRefinerBase import TaskRefinerBase
+
+
+# refiner for general purpose
+class GenTaskRefiner(TaskRefinerBase):
+    # constructor
+    def __init__(self, taskBufferIF, ddmIF):
+        TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
+
+    # extract common parameters
+    def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule):
+        if "cloud" not in taskParamMap and "workingGroup" in taskParamMap:
+            taskParamMap["cloud"] = taskParamMap["workingGroup"]
+        if "transPath" not in taskParamMap:
+            taskParamMap["transPath"] = "https://pandaserver-doma.cern.ch/trf/user/runGen-00-00-02"
+        # set sourceURL
+        try:
+            if "sourceURL" in taskParamMap:
+                for tmpItem in taskParamMap["jobParameters"]:
+                    if "value" in tmpItem:
+                        tmpItem["value"] = re.sub("\$\{SURL\}", taskParamMap["sourceURL"], tmpItem["value"])
+        except Exception:
+            pass
+        # min ram count
+        if "ramCount" not in taskParamMap:
+            taskParamMap["ramCount"] = 2000
+            taskParamMap["ramUnit"] = "MBPerCore"
+        # push status changes
+        if "pushStatusChanges" not in taskParamMap:
+            taskParamMap["pushStatusChanges"] = True
+        # fine-grained
+        if "fineGrainedProc" in taskParamMap:
+            taskParamMap["notDiscardEvents"] = True
+            taskParamMap["maxAttemptEsJob"] = 0
+            taskParamMap["maxAttemptES"] = 1
+        # message driven
+        if "messageDriven" not in taskParamMap:
+            taskParamMap["messageDriven"] = True
+        # use cloud as VO
+        taskParamMap["cloudAsVO"] = True
+        # update task parameters
+        self.updatedTaskParams = taskParamMap
+        # call base method
+        TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
+
+    # main
+    def doRefine(self, jediTaskID, taskParamMap):
+        # normal refine
+        self.doBasicRefine(taskParamMap)
+        # get DDM I/F to check
+        if self.ddmIF.getInterface(self.taskSpec.vo, self.taskSpec.cloud):
+            # use template dataset
+            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken["instantiateTmpl"])
+            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken["instantiateTmplSite"])
+            for datasetSpec in self.outDatasetSpecList:
+                datasetSpec.type = f"tmpl_{datasetSpec.type}"
+        return self.SC_SUCCEEDED

pandajedi/jedirefine/RefinerUtils.py

@@ -0,0 +1,154 @@
+import json
+import re
+
+
+# convert UTF-8 to ASCII in json dumps
+def unicodeConvert(input):
+    if isinstance(input, dict):
+        retMap = {}
+        for tmpKey, tmpVal in input.items():
+            retMap[unicodeConvert(tmpKey)] = unicodeConvert(tmpVal)
+        return retMap
+    elif isinstance(input, list):
+        retList = []
+        for tmpItem in input:
+            retList.append(unicodeConvert(tmpItem))
+        return retList
+
+    return input
+
+
+# decode
+def decodeJSON(inputStr):
+    return json.loads(inputStr, object_hook=unicodeConvert)
+
+
+# encode
+def encodeJSON(inputMap):
+    return json.dumps(inputMap)
+
+
+# extract stream name
+def extractStreamName(valStr):
+    tmpMatch = re.search("\$\{([^\}]+)\}", valStr)
+    if tmpMatch is None:
+        return None
+    # remove decorators
+    streamName = tmpMatch.group(1)
+    streamName = streamName.split("/")[0]
+    return streamName
+
+
+# extract output filename template and replace the value field
+def extractReplaceOutFileTemplate(valStr, streamName):
+    outFileTempl = valStr.split("=")[-1]
+    outFileTempl = outFileTempl.replace("'", "")
+    valStr = valStr.replace(outFileTempl, f"${{{streamName}}}")
+    return outFileTempl, valStr
+
+
+# extract file list
+def extractFileList(taskParamMap, datasetName):
+    baseDatasetName = datasetName.split(":")[-1]
+    if "log" in taskParamMap:
+        itemList = taskParamMap["jobParameters"] + [taskParamMap["log"]]
+    else:
+        itemList = taskParamMap["jobParameters"]
+    fileList = []
+    includePatt = []
+    excludePatt = []
+    for tmpItem in itemList:
+        if (
+            tmpItem["type"] == "template"
+            and "dataset" in tmpItem
+            and (
+                (tmpItem["dataset"] == datasetName or tmpItem["dataset"] == baseDatasetName)
+                or ("expandedList" in tmpItem and (datasetName in tmpItem["expandedList"] or baseDatasetName in tmpItem["expandedList"]))
+            )
+        ):
+            if "files" in tmpItem:
+                fileList = tmpItem["files"]
+            if "include" in tmpItem:
+                includePatt = tmpItem["include"].split(",")
+            if "exclude" in tmpItem:
+                excludePatt = tmpItem["exclude"].split(",")
+    return fileList, includePatt, excludePatt
+
+
+# append dataset
+def appendDataset(taskParamMap, datasetSpec, fileList):
+    # make item for dataset
+    tmpItem = {}
+    tmpItem["type"] = "template"
+    tmpItem["value"] = ""
+    tmpItem["dataset"] = datasetSpec.datasetName
+    tmpItem["param_type"] = datasetSpec.type
+    if fileList != []:
+        tmpItem["files"] = fileList
+    # append
+    if "jobParameters" not in taskParamMap:
+        taskParamMap["jobParameters"] = []
+    taskParamMap["jobParameters"].append(tmpItem)
+    return taskParamMap
+
+
+# check if use random seed
+def useRandomSeed(taskParamMap):
+    for tmpItem in taskParamMap["jobParameters"]:
+        if "value" in tmpItem:
+            # get offset for random seed
+            if tmpItem["type"] == "template" and tmpItem["param_type"] == "number":
+                if "${RNDMSEED}" in tmpItem["value"]:
+                    return True
+    return False
+
+
+# get initial global share
+def get_initial_global_share(task_buffer, task_id, task_spec=None, task_param_map=None):
+    """
+    Get the initial global share for a task
+    :param task_buffer: task buffer interface
+    :param task_id: task ID
+    :param task_spec: task specification object. read through task_buffer if None
+    :param task_param_map: task parameter map. read through task_buffer if None
+    :return:
+    """
+    if task_param_map is None:
+        # get task parameters from DB
+        tmp_str = task_buffer.getTaskParamsWithID_JEDI(task_id)
+        task_param_map = decodeJSON(tmp_str)
+    if "gshare" in task_param_map and task_buffer.is_valid_share(task_param_map["gshare"]):
+        # global share was already specified in ProdSys
+        gshare = task_param_map["gshare"]
+    else:
+        if task_spec is None:
+            # get task specification from DB
+            _, task_spec = task_buffer.getTaskWithID_JEDI(task_id)
+        # get share based on definition
+        gshare = task_buffer.get_share_for_task(task_spec)
+        if gshare == "Undefined":
+            error_message = "task definition does not match any global share"
+            raise RuntimeError(error_message)
+    return gshare
+
+
+# get sandbox name
+def get_sandbox_name(task_param_map: dict) -> str | None:
+    """
+    Get the sandbox name from the task parameters
+    :param task_param_map: dictionary of task parameters
+    :return: sandbox name or None
+    """
+    sandbox_name = None
+    if "fixedSandbox" in task_param_map:
+        sandbox_name = task_param_map["fixedSandbox"]
+    elif "buildSpec" in task_param_map:
+        sandbox_name = task_param_map["buildSpec"]["archiveName"]
+    else:
+        for tmpParam in task_param_map["jobParameters"]:
+            if tmpParam["type"] == "constant":
+                m = re.search("^-a ([^ ]+)$", tmpParam["value"])
+                if m is not None:
+                    sandbox_name = m.group(1)
+                    break
+    return sandbox_name