panda-server-source 0.8.0__py3-none-any.whl

pandaserver/test/reassignTask.py

@@ -0,0 +1,117 @@
+import argparse
+import datetime
+import time
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+# password
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.OraDBProxy import DBProxy
+from pandaserver.userinterface import Client
+
+proxyS = DBProxy()
+proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
+
+
+option_parser = argparse.ArgumentParser(conflict_handler="resolve", description="Reassign jobs in a task")
+option_parser.add_argument("taskid", action="store", metavar="TASKID", help="taskID of the task")
+option_parser.add_argument(
+    "-m",
+    dest="limit",
+    type=int,
+    action="store",
+    default=60,
+    metavar="MIMUTES",
+    help="time limit in minute",
+)
+option_parser.add_argument(
+    "-9",
+    action="store_const",
+    const=True,
+    dest="forceKill",
+    default=False,
+    help="kill jobs before next heartbeat is coming",
+)
+option_parser.add_argument(
+    "--keepUnmerged",
+    action="store_const",
+    const=True,
+    dest="keepUnmerged",
+    default=False,
+    help="generate new jobs after kiliing old jobs, to keep unmerged events",
+)
+options = option_parser.parse_args()
+
+taskid = options.taskid
+
+print("")
+print(f"trying to reassign jobs with modificationTime < CURRENT-{options.limit}min. Change the limit using -m if necessary")
+
+codeV = 51
+if options.forceKill:
+    codeV = 9
+
+jobs = []
+jediJobs = []
+
+timeLimit = naive_utcnow() - datetime.timedelta(minutes=options.limit)
+varMap = {}
+varMap[":modificationTime"] = timeLimit
+varMap[":taskID"] = taskid
+sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsDefined4 WHERE taskID=:taskID AND modificationTime<:modificationTime "
+status, res = proxyS.querySQLS(sql, varMap)
+if res is not None:
+    for id, lockedby in res:
+        if lockedby == "jedi":
+            jediJobs.append(id)
+        else:
+            jobs.append(id)
+
+
+varMap = {}
+varMap[":js1"] = "activated"
+varMap[":js2"] = "starting"
+varMap[":modificationTime"] = timeLimit
+varMap[":taskID"] = taskid
+sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsActive4 WHERE jobStatus IN (:js1,:js2) AND taskID=:taskID AND modificationTime<:modificationTime "
+status, res = proxyS.querySQLS(sql, varMap)
+if res is not None:
+    for id, lockedby in res:
+        if lockedby == "jedi":
+            jediJobs.append(id)
+        else:
+            jobs.append(id)
+
+varMap = {}
+varMap[":jobStatus"] = "waiting"
+varMap[":modificationTime"] = timeLimit
+varMap[":taskID"] = taskid
+sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsDefined4 WHERE jobStatus=:jobStatus AND taskID=:taskID AND modificationTime<:modificationTime "
+status, res = proxyS.querySQLS(sql, varMap)
+if res is not None:
+    for id, lockedby in res:
+        if lockedby == "jedi":
+            jediJobs.append(id)
+        else:
+            jobs.append(id)
+
+# reassign
+jobs.sort()
+if len(jobs):
+    nJob = 100
+    iJob = 0
+    while iJob < len(jobs):
+        print(f"reassign  {str(jobs[iJob:iJob + nJob])}")
+        Client.reassign_jobs(jobs[iJob : iJob + nJob])
+        iJob += nJob
+        time.sleep(10)
+
+if len(jediJobs) != 0:
+    nJob = 100
+    iJob = 0
+    while iJob < len(jediJobs):
+        print(f"kill JEDI jobs {str(jediJobs[iJob:iJob + nJob])}")
+        Client.kill_jobs(jediJobs[iJob : iJob + nJob], codeV, keep_unmerged=options.keepUnmerged)
+        iJob += nJob
+
+print(f"\nreassigned {len(jobs + jediJobs)} jobs")

pandaserver/test/reassignWaiting.py

@@ -0,0 +1,39 @@
+import datetime
+import time
+
+from pandacommon.pandautils.PandaUtils import naive_utcnow
+
+# password
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.OraDBProxy import DBProxy
+from pandaserver.userinterface import Client
+
+# time limit
+timeLimit = naive_utcnow() - datetime.timedelta(hours=1)
+
+# instantiate DB proxies
+proxyS = DBProxy()
+proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
+
+while True:
+    # get PandaIDs
+    varMap = {}
+    varMap[":modificationTime"] = timeLimit
+    sql = "SELECT PandaID FROM ATLAS_PANDA.jobsDefined4 WHERE modificationTime<:modificationTime ORDER BY PandaID"
+    status, res = proxyS.querySQLS(sql, varMap)
+
+    # escape
+    if len(res) == 0:
+        break
+    # convert to list
+    jobs = []
+    for (id,) in res:
+        jobs.append(id)
+    # reassign
+    nJob = 300
+    iJob = 0
+    while iJob < len(jobs):
+        print(f"reassignJobs({jobs[iJob:iJob + nJob]})")
+        Client.reassign_jobs(jobs[iJob : iJob + nJob])
+        iJob += nJob
+        time.sleep(60)

pandaserver/test/recoverLostFiles.py

@@ -0,0 +1,7 @@
+from pandaserver.dataservice import RecoverLostFilesCore
+
+s, o = RecoverLostFilesCore.main(exec_options={"userName": "CLI", "isProductionManager": True})
+if s:
+    print("OK")
+else:
+    print("ERROR:", o)

pandaserver/test/reloadInputDS.py

@@ -0,0 +1,51 @@
+import argparse
+
+from rucio.client import Client as RucioClient
+from rucio.common.exception import DataIdentifierNotFound
+
+from pandaserver.config import panda_config
+from pandaserver.dataservice.ddm import rucioAPI
+from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+from pandaserver.userinterface import Client
+
+taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1)
+
+# parse option
+parser = argparse.ArgumentParser()
+parser.add_argument("--tid", action="store", dest="tid", default=None, required=True, help="task ID")
+parser.add_argument(
+    "--resurrectDS",
+    action="store_const",
+    const=True,
+    dest="resurrectDS",
+    default=False,
+    help="resurrect output and log datasets if they were already deleted",
+)
+
+options = parser.parse_args()
+
+task_id = int(options.tid)
+
+if True:
+    if options.resurrectDS:
+        sd, so = taskBuffer.querySQLS(
+            "SELECT datasetName FROM ATLAS_PANDA.JEDI_Datasets WHERE jediTaskID=:id AND type IN (:t1,:t2)",
+            {":id": task_id, ":t1": "output", ":t2": "log"},
+        )
+        rucio_client = RucioClient()
+        for (dataset_name,) in so:
+            for i in range(3):
+                try:
+                    scope, name = rucioAPI.extract_scope(dataset_name)
+                    rucio_client.get_did(scope, name)
+                    break
+                except DataIdentifierNotFound:
+                    print(f"resurrect {dataset_name}")
+                    rucio_client.resurrect([{"scope": scope, "name": name}])
+                    try:
+                        rucio_client.set_metadata(scope, name, "lifetime", None)
+                    except Exception:
+                        pass
+
+    print(Client.reload_input(task_id))
+    print(f"done for task_id={task_id}")

pandaserver/test/sendCommandToJob.py

@@ -0,0 +1,26 @@
+import argparse
+
+from pandaserver.userinterface import Client
+
+# parse option
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--panda_id",
+    action="store",
+    dest="panda_id",
+    type=int,
+    required=True,
+    help="PandaID of the job",
+)
+parser.add_argument(
+    "--com_str",
+    action="store",
+    dest="com",
+    required=True,
+    help="The command string passed to the pilot. max 250 chars",
+)
+
+options = parser.parse_args()
+
+status, output = Client.send_command_to_job(options.panda_id, options.com)
+print(f"Status: {status}. Output: {output}")

pandaserver/test/setDebugMode.py

@@ -0,0 +1,38 @@
+import optparse
+import sys
+
+from pandaserver.userinterface import Client
+
+option_parser = optparse.OptionParser(conflict_handler="resolve", usage="%prog [options] <PandaID>")
+option_parser.add_option(
+    "--on",
+    action="store_const",
+    const=True,
+    dest="modeOn",
+    default=False,
+    help="turn the debug mode on",
+)
+option_parser.add_option(
+    "--off",
+    action="store_const",
+    const=True,
+    dest="modeOff",
+    default=False,
+    help="turn the debug mode off",
+)
+options, args = option_parser.parse_args()
+
+
+if (options.modeOn and options.modeOff) or (not options.modeOn and not options.modeOff):
+    print("ERROR: please set --on or --off")
+    sys.exit(1)
+
+job_id = int(args[0])
+
+if options.modeOn:
+    status, output = Client.set_debug_mode(job_id, True)
+else:
+    status, output = Client.set_debug_mode(job_id, False)
+
+print(f"status: {status}, output: {output}")
+sys.exit(0)

pandaserver/test/setPriority.py

@@ -0,0 +1,28 @@
+import optparse
+import sys
+
+# password
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.OraDBProxy import DBProxy
+
+usage = """%prog <taskID> <priority>
+
+  Set a priority to jobs in a task"""
+
+option_parser = optparse.OptionParser(usage=usage, conflict_handler="resolve")
+options, args = option_parser.parse_args()
+
+
+proxyS = DBProxy()
+proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
+
+varMap = {}
+varMap[":prodSourceLabel"] = "managed"
+varMap[":taskID"] = sys.argv[1]
+varMap[":prio"] = sys.argv[2]
+sql = "UPDATE %s SET currentPriority=:prio WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID"
+for table in [
+    "ATLAS_PANDA.jobsActive4",
+    "ATLAS_PANDA.jobsDefined4",
+]:
+    status, res = proxyS.querySQLS(sql % table, varMap)

pandaserver/test/testEvgen.py

@@ -0,0 +1,60 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+if len(sys.argv) > 1:
+    site = sys.argv[1]
+else:
+    site = None
+
+dataset_name = f"panda.destDB.{str(uuid.uuid4())}"
+destination_se = None
+
+job_list = []
+
+n_jobs = 1
+
+for i in range(n_jobs):
+    job = JobSpec()
+    job.jobDefinitionID = int(time.time()) % 10000
+    job.jobName = "%s_%d" % (str(uuid.uuid4()), i)
+    job.AtlasRelease = "Atlas-14.1.0"
+    job.homepackage = "AtlasProduction/14.1.0.3"
+    job.transformation = "csc_evgen_trf.py"
+    job.destinationDBlock = dataset_name
+    job.destinationSE = destination_se
+    job.currentPriority = 100
+    job.prodSourceLabel = "test"
+    job.computingSite = site
+    job.cloud = "US"
+    job.cmtConfig = "i686-slc4-gcc34-opt"
+
+    file = FileSpec()
+    file.lfn = f"{job.jobName}.evgen.pool.root"
+    file.destinationDBlock = job.destinationDBlock
+    file.destinationSE = job.destinationSE
+    file.dataset = job.destinationDBlock
+    file.destinationDBlockToken = "ATLASDATADISK"
+    file.type = "output"
+    job.addFile(file)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{job.jobName}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+
+    job.jobParameters = f"5144 1 5000 1 CSC.005144.PythiaZee.py {file.lfn} NONE NONE NONE"
+    job_list.append(job)
+
+for i in range(n_jobs):
+    status, output = Client.submit_jobs(job_list)
+    print("---------------------")
+    print(f"Status: {status}. Output: {output}")

pandaserver/test/testEvgen17.py

@@ -0,0 +1,58 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+site = sys.argv[1]
+cloud = sys.argv[2]
+
+dataset_name = f"panda.destDB.{str(uuid.uuid4())}"
+destination_se = None
+
+n_jobs = 1
+job_list = []
+
+for i in range(n_jobs):
+    job = JobSpec()
+    job.jobDefinitionID = int(time.time()) % 10000
+    job.jobName = "%s_%d" % (str(uuid.uuid4()), i)
+    job.AtlasRelease = "Atlas-17.0.5"
+    job.homepackage = "AtlasProduction/17.0.5.6"
+    job.transformation = "Evgen_trf.py"
+    job.destinationDBlock = dataset_name
+    job.destinationSE = destination_se
+    job.currentPriority = 10000
+    job.prodSourceLabel = "test"
+    job.computingSite = site
+    job.cloud = cloud
+    job.cmtConfig = "i686-slc5-gcc43-opt"
+
+    file = FileSpec()
+    file.lfn = f"{job.jobName}.evgen.pool.root"
+    file.destinationDBlock = job.destinationDBlock
+    file.destinationSE = job.destinationSE
+    file.dataset = job.destinationDBlock
+    file.destinationDBlockToken = "ATLASDATADISK"
+    file.type = "output"
+    job.addFile(file)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{job.jobName}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+
+    job.jobParameters = f"7000 108316 1 5000 1 MC11.108316.Pythia8_minbias_ND.py {file.lfn}"
+
+    job_list.append(job)
+
+for i in range(n_jobs):
+    status, output = Client.submit_jobs(job_list)
+    print("---------------------")
+    print(f"Status: {status}. Output: {output}")

pandaserver/test/testG4sim.py

@@ -0,0 +1,81 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+if len(sys.argv) > 1:
+    site = sys.argv[1]
+else:
+    site = None
+
+dataset_name = f"panda.destDB.{str(uuid.uuid4())}"
+destination_se = "BNL_ATLAS_2"
+# destination_se    = 'BU_ATLAS_Tier2'
+
+files = {
+    "mc11.007204.singlepart_mu4.evgen.EVNT.v11000302._00037.pool.root.1": None,
+    "mc11.007204.singlepart_mu4.evgen.EVNT.v11000302._00038.pool.root.1": None,
+}
+
+job_list = []
+
+for lfn in files:
+    job = JobSpec()
+    job.jobDefinitionID = int(time.time()) % 10000
+    job.jobName = str(uuid.uuid4())
+    job.AtlasRelease = "Atlas-11.0.3"
+    job.homepackage = "JobTransforms-11-00-03-02"
+    job.transformation = "share/csc.simul.trf"
+    job.destinationDBlock = dataset_name
+    job.destinationSE = destination_se
+    job.computingSite = site
+    job.prodDBlock = "mc11.007204.singlepart_mu4.evgen.EVNT.v11000302"
+    job.cmtConfig = "i686-slc4-gcc34-opt"
+
+    job.prodSourceLabel = "test"
+    job.currentPriority = 1000
+
+    fileI = FileSpec()
+    fileI.dataset = job.prodDBlock
+    fileI.prodDBlock = job.prodDBlock
+    fileI.lfn = lfn
+    fileI.type = "input"
+    job.addFile(fileI)
+
+    fileOE = FileSpec()
+    fileOE.lfn = f"{str(uuid.uuid4())}.HITS.pool.root"
+    fileOE.destinationDBlock = job.destinationDBlock
+    fileOE.destinationSE = job.destinationSE
+    fileOE.dataset = job.destinationDBlock
+    fileOE.destinationDBlockToken = "ATLASDATADISK"
+    fileOE.type = "output"
+    job.addFile(fileOE)
+
+    fileOA = FileSpec()
+    fileOA.lfn = f"{str(uuid.uuid4())}.RDO.pool.root"
+    fileOA.destinationDBlock = job.destinationDBlock
+    fileOA.destinationSE = job.destinationSE
+    fileOA.dataset = job.destinationDBlock
+    fileOA.destinationDBlockToken = "ATLASDATADISK"
+    fileOA.type = "output"
+    job.addFile(fileOA)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{str(uuid.uuid4())}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+
+    job.jobParameters = f"{fileI.lfn} {fileOE.lfn} {fileOA.lfn}  100 700 2158"
+
+    job_list.append(job)
+
+status, output = Client.submit_jobs(job_list)
+print("---------------------")
+print(f"Status: {status}. Output: {output}")

pandaserver/test/testG4sim17.py

@@ -0,0 +1,90 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+site = sys.argv[1]
+cloud = sys.argv[2]
+
+prod_destination_block = "mc10_7TeV.105001.pythia_minbias.evgen.EVNT.e574_tid153937_00"
+input_file = "EVNT.153937._000184.pool.root.1"
+
+if len(sys.argv) == 5:
+    site = sys.argv[1]
+    cloud = sys.argv[2]
+    prod_destination_block = sys.argv[3]
+    input_file = sys.argv[4]
+
+datasetName = f"panda.destDB.{str(uuid.uuid4())}"
+
+files = {
+    input_file: None,
+}
+
+job_list = []
+
+index = 0
+for lfn in files:
+    index += 1
+    job = JobSpec()
+    job.jobDefinitionID = (time.time()) % 10000
+    job.jobName = "%s_%d" % (str(uuid.uuid4()), index)
+    job.AtlasRelease = "Atlas-17.0.5"
+    job.homepackage = "AtlasProduction/17.0.5.6"
+    job.transformation = "AtlasG4_trf.py"
+    job.destinationDBlock = datasetName
+    job.computingSite = site
+    job.prodDBlock = prod_destination_block
+
+    job.prodSourceLabel = "test"
+    job.processingType = "test"
+    job.currentPriority = 10000
+    job.cloud = cloud
+    job.cmtConfig = "i686-slc5-gcc43-opt"
+
+    fileI = FileSpec()
+    fileI.dataset = job.prodDBlock
+    fileI.prodDBlock = job.prodDBlock
+    fileI.lfn = lfn
+    fileI.type = "input"
+    job.addFile(fileI)
+
+    fileD = FileSpec()
+    fileD.dataset = "ddo.000001.Atlas.Ideal.DBRelease.v170602"
+    fileD.prodDBlock = fileD.dataset
+    fileD.lfn = "DBRelease-17.6.2.tar.gz"
+    fileD.type = "input"
+    job.addFile(fileD)
+
+    fileOA = FileSpec()
+    fileOA.lfn = f"{job.jobName}.HITS.pool.root"
+    fileOA.destinationDBlock = job.destinationDBlock
+    fileOA.destinationSE = job.destinationSE
+    fileOA.dataset = job.destinationDBlock
+    fileOA.destinationDBlockToken = "ATLASDATADISK"
+    fileOA.type = "output"
+    job.addFile(fileOA)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{job.jobName}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+    job.jobParameters = (
+        f"inputEvgenFile={fileI.lfn} outputHitsFile={fileOA.lfn} maxEvents=3 skipEvents=0 "
+        f"DBRelease={fileD.lfn} geometryVersion=ATLAS-GEO-18-01-03_VALIDATION "
+        f"conditionsTag=OFLCOND-SDR-BS7T-05-14 randomSeed=1 physicsList=QGSP_BERT "
+        f"RunNumber=116870 firstEvent=1"
+    )
+
+    job_list.append(job)
+
+status, output = Client.submit_jobs(job_list)
+print("---------------------")
+print(f"Status: {status}. Output: {output}")