panda-server-source 0.8.0__py3-none-any.whl

pandaserver/test/testGlobalShares.py

@@ -0,0 +1,131 @@
+import sys
+
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.OraDBProxy import DBProxy
+
+if __name__ == "__main__":
+    """
+    Functional testing of the shares tree
+    """
+    proxyS = DBProxy()
+    proxyS.connect(
+        panda_config.dbhost,
+        panda_config.dbpasswd,
+        panda_config.dbuser,
+        panda_config.dbname,
+    )
+
+    # print the shares in order of under usage
+    print("--------------LEAVE SHARES SORTED BY UNDER-PLEDGING---------------")
+    print(proxyS.get_sorted_leaves())
+
+    # print the global share structure
+    print("--------------GLOBAL SHARES TREE---------------")
+    print(proxyS.tree)
+
+    # print the normalized leaves, which will be the actual applied shares
+    print("--------------LEAVE SHARES---------------")
+    print(proxyS.leave_shares)
+
+    # print the current grid status
+    print("--------------CURRENT GRID STATUS---------------")
+    print(proxyS.tree.pretty_print_hs_distribution(proxyS._DBProxy__hs_distribution))
+
+    # check a couple of shares if they are valid leave names
+    share_name = "wrong_share"
+    print(f"Share {share_name} is valid: {proxyS.is_valid_share(share_name)}")
+    share_name = "MC16Pile"
+    print(f"Share {share_name} is valid: {proxyS.is_valid_share(share_name)}")
+
+    try:
+        from pandajedi.jedicore.JediTaskSpec import JediTaskSpec
+    except ImportError:
+        print("Skipped task tests since JEDI module depency not satisfied")
+        sys.exit(0)
+
+    # create a fake tasks with relevant fields and retrieve its share
+    task_spec = JediTaskSpec()
+
+    # Analysis task
+    task_spec.prodSourceLabel = "user"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "dummy_wg"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Analysis')")
+
+    # Production task without any matching leave
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "dummy_wg"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Undefined')")
+
+    # Test task
+    task_spec.prodSourceLabel = "test123"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "dummy_wg"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Test')")
+
+    # Derivations task without any matching leave
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "GP_PHYS"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Undefined')")
+
+    # Reprocessing task without any matching leave
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "AP_REPR"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Undefined')")
+
+    # Group production task
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "GP_LOL"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Group production')")
+
+    # Upgrade task
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "AP_UPG"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Upgrade')")
+
+    # HLT Reprocessing
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "AP_THLT"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'HLT Reprocessing')")
+
+    # Validation
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "AP_VALI"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Validation')")
+
+    # Event Index
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "dummy_campaign"
+    task_spec.workingGroup = "proj-evind"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Event Index')")
+
+    # MC Derivations
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "mc.*"
+    task_spec.workingGroup = "GP_PHYS"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'MC Derivations')")
+
+    # Data Derivations
+    task_spec.prodSourceLabel = "managed"
+    task_spec.campaign = "data.*"
+    task_spec.workingGroup = "GP_PHYS"
+    task_spec.processingType = "dummy_type"
+    print(f"Share for task is {proxyS.get_share_for_task(task_spec)}(should be 'Data Derivations')")

pandaserver/test/testReco.py

@@ -0,0 +1,103 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+if len(sys.argv) > 1:
+    site = sys.argv[1]
+else:
+    site = None
+
+dataset_name = f"panda.destDB.{str(uuid.uuid4())}"
+destination_se = None
+
+files = {
+    "misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610._11615.pool.root.1": None,
+    # 'misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610._11639.pool.root.1':None,
+    # 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03634.pool.root.1':None,
+    # 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03248.pool.root.1':None,
+    # 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03634.pool.root.1':None,
+}
+
+job_list = []
+
+index = 0
+for lfn in files:
+    index += 1
+    job = JobSpec()
+    job.jobDefinitionID = int(time.time()) % 10000
+    job.jobName = "%s_%d" % (str(uuid.uuid4()), index)
+    job.AtlasRelease = "Atlas-12.0.6"
+    job.homepackage = "AtlasProduction/12.0.6.4"
+    job.transformation = "csc_reco_trf.py"
+    job.destinationDBlock = dataset_name
+    job.destinationSE = destination_se
+    job.computingSite = site
+    # job.prodDBlock        = 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554'
+    job.prodDBlock = "misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610"
+    job.cloud = "US"
+
+    job.prodSourceLabel = "test"
+    job.currentPriority = 10000
+    job.cmtConfig = "i686-slc4-gcc34-opt"
+
+    fileI = FileSpec()
+    fileI.dataset = job.prodDBlock
+    fileI.prodDBlock = job.prodDBlock
+    fileI.lfn = lfn
+    fileI.type = "input"
+    job.addFile(fileI)
+
+    fileD = FileSpec()
+    fileD.dataset = "ddo.000001.Atlas.Ideal.DBRelease.v030101"
+    fileD.prodDBlock = "ddo.000001.Atlas.Ideal.DBRelease.v030101"
+    fileD.lfn = "DBRelease-3.1.1.tar.gz"
+    fileD.type = "input"
+    job.addFile(fileD)
+
+    fileOE = FileSpec()
+    fileOE.lfn = f"{job.jobName}.ESD.pool.root"
+    fileOE.destinationDBlock = job.destinationDBlock
+    fileOE.destinationSE = job.destinationSE
+    fileOE.dataset = job.destinationDBlock
+    fileOE.destinationDBlockToken = "ATLASDATADISK"
+    fileOE.type = "output"
+    job.addFile(fileOE)
+
+    fileOA = FileSpec()
+    fileOA.lfn = f"{job.jobName}.AOD.pool.root"
+    fileOA.destinationDBlock = job.destinationDBlock
+    fileOA.destinationSE = job.destinationSE
+    fileOA.dataset = job.destinationDBlock
+    fileOA.destinationDBlockToken = "ATLASDATADISK"
+    fileOA.type = "output"
+    job.addFile(fileOA)
+
+    fileOC = FileSpec()
+    fileOC.lfn = f"{job.jobName}.NTUP.root"
+    fileOC.destinationDBlock = job.destinationDBlock
+    fileOC.destinationSE = job.destinationSE
+    fileOC.dataset = job.destinationDBlock
+    fileOC.destinationDBlockToken = "ATLASDATADISK"
+    fileOC.type = "output"
+    job.addFile(fileOC)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{job.jobName}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+
+    job.jobParameters = f"{fileI.lfn} {fileOE.lfn} {fileOA.lfn} {fileOC.lfn} 250 0 ATLAS-CSC-01-02-00 CSC-06 NoRestrictedESDRecConfig.py {fileD.lfn}"
+
+    job_list.append(job)
+
+status, output = Client.submit_jobs(job_list)
+print("---------------------")
+print(f"Status: {status}. Output: {output}")

pandaserver/test/testSimulReco14.py

@@ -0,0 +1,98 @@
+import sys
+import time
+import uuid
+
+from pandaserver.taskbuffer.FileSpec import FileSpec
+from pandaserver.taskbuffer.JobSpec import JobSpec
+from pandaserver.userinterface import Client
+
+if len(sys.argv) > 1:
+    site = sys.argv[1]
+    cloud = None
+else:
+    site = None
+    cloud = "US"
+
+cloud = "US"
+
+dataset_name = f"panda.destDB.{str(uuid.uuid4())}"
+destination_se = "BNL_ATLAS_2"
+
+files = {
+    "EVNT.023986._00001.pool.root.1": None,
+    # 'EVNT.023989._00001.pool.root.1':None,
+}
+
+job_list = []
+
+index = 0
+for lfn in files:
+    index += 1
+    job = JobSpec()
+    job.jobDefinitionID = (time.time()) % 10000
+    job.jobName = "%s_%d" % (str(uuid.uuid4()), index)
+    job.AtlasRelease = "Atlas-14.2.20"
+    job.homepackage = "AtlasProduction/14.2.20.1"
+    job.transformation = "csc_simul_reco_trf.py"
+    job.destinationDBlock = dataset_name
+    job.destinationSE = destination_se
+    job.computingSite = site
+    job.prodDBlock = "mc08.105031.Jimmy_jetsJ2.evgen.EVNT.e347_tid023986"
+    # job.prodDBlock        = 'mc08.105034.Jimmy_jetsJ5.evgen.EVNT.e347_tid023989'
+
+    job.prodSourceLabel = "test"
+    job.processingType = "test"
+    job.currentPriority = 10000
+    job.cloud = cloud
+
+    fileI = FileSpec()
+    fileI.dataset = job.prodDBlock
+    fileI.prodDBlock = job.prodDBlock
+    fileI.lfn = lfn
+    fileI.type = "input"
+    job.addFile(fileI)
+
+    fileD = FileSpec()
+    fileD.dataset = "ddo.000001.Atlas.Ideal.DBRelease.v050601"
+    fileD.prodDBlock = "ddo.000001.Atlas.Ideal.DBRelease.v050601"
+    fileD.lfn = "DBRelease-5.6.1.tar.gz"
+    fileD.type = "input"
+    job.addFile(fileD)
+
+    fileOA = FileSpec()
+    fileOA.lfn = f"{job.jobName}.AOD.pool.root"
+    fileOA.destinationDBlock = job.destinationDBlock
+    fileOA.destinationSE = job.destinationSE
+    fileOA.dataset = job.destinationDBlock
+    fileOA.destinationDBlockToken = "ATLASDATADISK"
+    fileOA.type = "output"
+    job.addFile(fileOA)
+
+    fileOE = FileSpec()
+    fileOE.lfn = f"{job.jobName}.ESD.pool.root"
+    fileOE.destinationDBlock = job.destinationDBlock
+    fileOE.destinationSE = job.destinationSE
+    fileOE.dataset = job.destinationDBlock
+    fileOE.destinationDBlockToken = "ATLASDATADISK"
+    fileOE.type = "output"
+    job.addFile(fileOE)
+
+    fileOL = FileSpec()
+    fileOL.lfn = f"{job.jobName}.job.log.tgz"
+    fileOL.destinationDBlock = job.destinationDBlock
+    fileOL.destinationSE = job.destinationSE
+    fileOL.dataset = job.destinationDBlock
+    fileOL.destinationDBlockToken = "ATLASDATADISK"
+    fileOL.type = "log"
+    job.addFile(fileOL)
+
+    job.jobParameters = (
+        "%s %s 30 500 3 ATLAS-GEO-02-01-00 3 3 QGSP_BERT jobConfig.VertexPosFastIDKiller.py FastSimulationJobTransforms/FastCaloSimAddCellsRecConfig.py,NoTrackSlimming.py %s OFF NONE NONE %s NONE"
+        % (fileI.lfn, fileOA.lfn, fileD.lfn, fileOE.lfn)
+    )
+
+    job_list.append(job)
+
+status, output = Client.submit_jobs(job_list)
+print("---------------------")
+print(f"Status: {status}. Output: {output}")

pandaserver/test/testSiteMap.py

@@ -0,0 +1,26 @@
+import sys
+
+from pandacommon.pandautils.thread_utils import GenericThread
+
+from pandaserver.brokerage import SiteMapper
+from pandaserver.config import panda_config
+from pandaserver.taskbuffer.TaskBuffer import taskBuffer
+
+requester_id = GenericThread().get_full_id(__name__, sys.modules[__name__].__file__)
+taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1, requester=requester_id)
+
+site_mapper = SiteMapper.SiteMapper(taskBuffer)
+
+"""
+x = site_mapper.getSite('ANALY_DESY-HH')
+print(x.ddm_input)
+print(x.ddm_output)
+print(x.setokens_input)
+print(x.setokens_output)
+
+x = site_mapper.getSite('BNL_PROD')
+print(x.ddm_input)
+print(x.ddm_output)
+print(x.setokens_input)
+print(x.setokens_output)
+"""

pandaserver/test/testUpdateWorkerPilotStatus.py

@@ -0,0 +1,9 @@
+from pandaserver.api.v1.http_client import HttpClient as HttpClientV1
+from pandaserver.api.v1.http_client import api_url_ssl as api_url_ssl_v1
+
+http_client = HttpClientV1()
+
+url = f"{api_url_ssl_v1}/pilot/update_worker_status"
+data = {"worker_id": 9139456, "harvester_id": "CERN_central_k8s", "status": "started"}
+status, output = http_client.post(url, data)
+print(f"Status: {status}. Output: {output}")