panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
from userinterface import Client
|
|
5
|
+
|
|
6
|
+
site = sys.argv[1]
|
|
7
|
+
|
|
8
|
+
dsName = "mc12_8TeV.189659.gg2VVPythia8_AU2CT10_ggH125p5_VV_2mu2numu_m2l4_2pt3.evgen.EVNT.e2872_tid01461041_00"
|
|
9
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
10
|
+
|
|
11
|
+
taskParamMap = {}
|
|
12
|
+
|
|
13
|
+
# taskParamMap['nFilesPerJob'] = 1
|
|
14
|
+
taskParamMap["nFiles"] = 1
|
|
15
|
+
taskParamMap["nEventsPerFile"] = 5000
|
|
16
|
+
taskParamMap["nEventsPerWorker"] = 10 # 5000
|
|
17
|
+
taskParamMap["nEventsPerJob"] = 1000
|
|
18
|
+
taskParamMap["nEvents"] = 1000
|
|
19
|
+
# taskParamMap['skipScout'] = True
|
|
20
|
+
taskParamMap["objectStore"] = "root^atlas-objectstore.cern.ch^/atlas/eventservice"
|
|
21
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
22
|
+
taskParamMap["userName"] = "pandasrv1"
|
|
23
|
+
taskParamMap["vo"] = "atlas"
|
|
24
|
+
taskParamMap["taskPriority"] = 900
|
|
25
|
+
# taskParamMap['reqID'] = reqIdx
|
|
26
|
+
taskParamMap["architecture"] = "x86_64-slc5-gcc43-opt"
|
|
27
|
+
taskParamMap["transUses"] = "Atlas-17.2.11"
|
|
28
|
+
taskParamMap["transHome"] = "AtlasProduction-17.2.11.8"
|
|
29
|
+
taskParamMap["transPath"] = "AtlasG4_tf.py"
|
|
30
|
+
taskParamMap["processingType"] = "simul"
|
|
31
|
+
taskParamMap["prodSourceLabel"] = "ptest"
|
|
32
|
+
taskParamMap["taskType"] = "prod"
|
|
33
|
+
taskParamMap["workingGroup"] = "AP_Higgs"
|
|
34
|
+
taskParamMap["coreCount"] = 1
|
|
35
|
+
taskParamMap["walltime"] = 1
|
|
36
|
+
taskParamMap["cloud"] = "US"
|
|
37
|
+
taskParamMap["site"] = site
|
|
38
|
+
taskParamMap["log"] = {
|
|
39
|
+
"dataset": logDatasetName,
|
|
40
|
+
"type": "template",
|
|
41
|
+
"param_type": "log",
|
|
42
|
+
"token": "ATLASDATADISK",
|
|
43
|
+
"offset": 1000,
|
|
44
|
+
"value": f"{logDatasetName}.${{SN}}.log.tgz",
|
|
45
|
+
}
|
|
46
|
+
outDatasetName = f"panda.jeditest.HITS.{uuid.uuid4()}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
taskParamMap["jobParameters"] = [
|
|
50
|
+
{
|
|
51
|
+
"type": "constant",
|
|
52
|
+
"value": "--AMITag s1831 --DBRelease=current --athenaopts=--preloadlib=${ATLASMKLLIBDIR_PRELOAD}/libimf.so --conditionsTag=OFLCOND-MC12-SIM-00",
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"type": "template",
|
|
56
|
+
"value": "firstEvent=${FIRSTEVENT}",
|
|
57
|
+
"param_type": "number",
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"type": "constant",
|
|
61
|
+
"value": "--geometryVersion=ATLAS-GEO-21-02-02_VALIDATION",
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"type": "template",
|
|
65
|
+
"param_type": "input",
|
|
66
|
+
"value": "--inputEvgenFile=${IN}",
|
|
67
|
+
"dataset": dsName,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "template",
|
|
71
|
+
"value": "--maxEvents=${MAXEVENTS}",
|
|
72
|
+
"param_type": "number",
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"type": "template",
|
|
76
|
+
"param_type": "output",
|
|
77
|
+
"token": "ATLASDATADISK",
|
|
78
|
+
"value": f" --outputHitsFile={outDatasetName}.${{SN}}.pool.root",
|
|
79
|
+
"dataset": outDatasetName,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"type": "constant",
|
|
83
|
+
"value": "--physicsList=QGSP_BERT --postInclude=RecJobTransforms/UseFrontierFallbackDBRelease.py --preInclude=SimulationJobOptions/preInclude.FrozenShowersFCalOnly.py,SimulationJobOptions/preInclude.BeamPipeKill.py",
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"type": "template",
|
|
87
|
+
"value": "--skipEvents=${SKIPEVENTS}",
|
|
88
|
+
"param_type": "number",
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"type": "template",
|
|
92
|
+
"value": "--randomSeed=${RNDMSEED}",
|
|
93
|
+
"param_type": "number",
|
|
94
|
+
},
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
taskParamMap["esmergeSpec"] = {}
|
|
98
|
+
taskParamMap["esmergeSpec"]["transPath"] = "Merge_trf.py"
|
|
99
|
+
taskParamMap["esmergeSpec"]["jobParameters"] = "aaa bbb"
|
|
100
|
+
|
|
101
|
+
print(Client.insertTaskParams(taskParamMap))
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from urllib.parse import quote
|
|
3
|
+
|
|
4
|
+
from userinterface import Client
|
|
5
|
+
|
|
6
|
+
taskParamMap = {}
|
|
7
|
+
|
|
8
|
+
taskParamMap["noInput"] = True
|
|
9
|
+
taskParamMap["nEventsPerJob"] = 1
|
|
10
|
+
taskParamMap["nEvents"] = 5
|
|
11
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
12
|
+
taskParamMap["userName"] = "pandasrv1"
|
|
13
|
+
taskParamMap["vo"] = "atlas"
|
|
14
|
+
taskParamMap["taskPriority"] = 1000
|
|
15
|
+
taskParamMap["reqID"] = 12345
|
|
16
|
+
taskParamMap["architecture"] = ""
|
|
17
|
+
taskParamMap["hpoWorkflow"] = True
|
|
18
|
+
taskParamMap["transUses"] = ""
|
|
19
|
+
taskParamMap["transHome"] = ""
|
|
20
|
+
taskParamMap["transPath"] = "http://pandaserver.cern.ch:25080/trf/user/runHPO-00-00-01"
|
|
21
|
+
taskParamMap["processingType"] = "simul"
|
|
22
|
+
taskParamMap["prodSourceLabel"] = "test"
|
|
23
|
+
taskParamMap["useLocalIO"] = 1
|
|
24
|
+
taskParamMap["taskType"] = "prod"
|
|
25
|
+
taskParamMap["workingGroup"] = "AP_HPO"
|
|
26
|
+
taskParamMap["coreCount"] = 1
|
|
27
|
+
taskParamMap["site"] = "BNL_PROD_UCORE"
|
|
28
|
+
taskParamMap["nucleus"] = "CERN-PROD"
|
|
29
|
+
taskParamMap["cloud"] = "WORLD"
|
|
30
|
+
|
|
31
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
32
|
+
outDatasetName = f"panda.jeditest.HPO.{uuid.uuid4()}"
|
|
33
|
+
|
|
34
|
+
taskParamMap["log"] = {
|
|
35
|
+
"dataset": logDatasetName,
|
|
36
|
+
"type": "template",
|
|
37
|
+
"param_type": "log",
|
|
38
|
+
"token": "ddd:.*DATADISK",
|
|
39
|
+
"destination": "(type=DATADISK)\(dontkeeplog=True)",
|
|
40
|
+
"offset": 1000,
|
|
41
|
+
"value": f"{logDatasetName}.${{SN}}.log.tgz",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
taskParamMap["hpoRequestData"] = {
|
|
45
|
+
"sandbox": "gitlab-registry.cern.ch/zhangruihpc/steeringcontainer:latest",
|
|
46
|
+
"executable": "docker",
|
|
47
|
+
"arguments": '/bin/bash -c "hpogrid generate --n_point=%NUM_POINTS ' "--max_point=%MAX_POINTS --infile=$PWD/%IN --outfile=$PWD/%OUT " '-l=nevergrad"',
|
|
48
|
+
"output_json": "output.json",
|
|
49
|
+
"max_points": 10,
|
|
50
|
+
"num_points_per_generation": 2,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
taskParamMap["container_name"] = "docker://gitlab-registry.cern.ch/zhangruihpc/evaluationcontainer:mlflow"
|
|
54
|
+
|
|
55
|
+
taskParamMap["jobParameters"] = [
|
|
56
|
+
{"type": "constant", "value": f"-o out.json -j \"\" -p \"{quote('bash ./exec_in_container.sh')}\""},
|
|
57
|
+
{"type": "constant", "value": "--writeInputToTxt IN_DATA:input_ds.json --inSampleFile input_sample.json"},
|
|
58
|
+
{"type": "constant", "value": "-a aaa.tgz --sourceURL https://aipanda048.cern.ch:25443"},
|
|
59
|
+
{"type": "constant", "value": "--inMap \"{'IN_DATA': ${IN_DATA/T}}\""},
|
|
60
|
+
{
|
|
61
|
+
"type": "template",
|
|
62
|
+
"param_type": "input",
|
|
63
|
+
"value": '-i "${IN_DATA/T}"',
|
|
64
|
+
"dataset": "mc16_13TeV.501103.MGPy8EG_StauStauDirect_220p0_1p0_TFilt.merge.EVNT.e8102_e7400_tid21342682_00",
|
|
65
|
+
"attributes": "nosplit,repeat",
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"type": "template",
|
|
69
|
+
"param_type": "output",
|
|
70
|
+
"token": "ATLASDATADISK",
|
|
71
|
+
"value": "$JEDITASKID.metrics.${SN}.tgz",
|
|
72
|
+
"dataset": outDatasetName,
|
|
73
|
+
"hidden": True,
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"type": "constant",
|
|
77
|
+
"value": "--outMetricsFile=${OUTPUT0}^metrics.tgz",
|
|
78
|
+
},
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
print(Client.insertTaskParams(taskParamMap))
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
|
|
4
|
+
from pandajedi.jediddm.DDMInterface import DDMInterface
|
|
5
|
+
from pandaserver.taskbuffer.DataCarousel import (
|
|
6
|
+
DataCarouselInterface,
|
|
7
|
+
DataCarouselRequestSpec,
|
|
8
|
+
DataCarouselRequestStatus,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
vo = "atlas"
|
|
12
|
+
task_id = int(sys.argv[1])
|
|
13
|
+
request_id = int(sys.argv[2])
|
|
14
|
+
|
|
15
|
+
print("set tbIF")
|
|
16
|
+
tbIF = JediTaskBufferInterface()
|
|
17
|
+
tbIF.setupInterface(max_size=1)
|
|
18
|
+
|
|
19
|
+
print("set ddmIF")
|
|
20
|
+
ddmIF = DDMInterface()
|
|
21
|
+
ddmIF.setupInterface()
|
|
22
|
+
|
|
23
|
+
print("set DCIF")
|
|
24
|
+
data_carousel_interface = DataCarouselInterface(tbIF, ddmIF.getInterface(vo))
|
|
25
|
+
if data_carousel_interface is None:
|
|
26
|
+
# data carousel interface is undefined
|
|
27
|
+
errStr = f"data carousel interface is undefined for vo={vo}"
|
|
28
|
+
print(errStr)
|
|
29
|
+
sys.exit(1)
|
|
30
|
+
|
|
31
|
+
print("query DB for data carousel request")
|
|
32
|
+
sql = (
|
|
33
|
+
f"SELECT req.* "
|
|
34
|
+
f"FROM ATLAS_PANDA.data_carousel_requests req, ATLAS_PANDA.data_carousel_relations rel "
|
|
35
|
+
f"WHERE req.request_id=rel.request_id "
|
|
36
|
+
f"AND rel.request_id=:request_id AND rel.task_id=:task_id "
|
|
37
|
+
)
|
|
38
|
+
var_map = {
|
|
39
|
+
":task_id": task_id,
|
|
40
|
+
":request_id": request_id,
|
|
41
|
+
}
|
|
42
|
+
res_list = tbIF.querySQL(sql, var_map)
|
|
43
|
+
|
|
44
|
+
if res_list:
|
|
45
|
+
for res in res_list:
|
|
46
|
+
print("got request; submit iDDS stage-in request")
|
|
47
|
+
# make request spec
|
|
48
|
+
dc_req_spec = DataCarouselRequestSpec()
|
|
49
|
+
dc_req_spec.pack(res)
|
|
50
|
+
# submit iDDS stage-in reqeust
|
|
51
|
+
ret = data_carousel_interface._submit_idds_stagein_request(task_id, dc_req_spec)
|
|
52
|
+
print(f"Done submit to iDDS; iDDS_requestID={ret}")
|
|
53
|
+
break
|
|
54
|
+
else:
|
|
55
|
+
print(f"Got no request: {res_list}")
|
|
56
|
+
sys.exit(0)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
from pandatools import Client
|
|
5
|
+
except ImportError:
|
|
6
|
+
from userinterface import Client
|
|
7
|
+
|
|
8
|
+
inFileList = ["file1", "file2", "file3"]
|
|
9
|
+
|
|
10
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
11
|
+
|
|
12
|
+
taskParamMap = {}
|
|
13
|
+
|
|
14
|
+
taskParamMap["nFilesPerJob"] = 1
|
|
15
|
+
taskParamMap["nFiles"] = len(inFileList)
|
|
16
|
+
# taskParamMap['nEventsPerInputFile'] = 10000
|
|
17
|
+
# taskParamMap['nEventsPerJob'] = 10000
|
|
18
|
+
# taskParamMap['nEvents'] = 25000
|
|
19
|
+
taskParamMap["noInput"] = True
|
|
20
|
+
taskParamMap["pfnList"] = inFileList
|
|
21
|
+
# taskParamMap['mergeOutput'] = True
|
|
22
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
23
|
+
taskParamMap["userName"] = "someone"
|
|
24
|
+
taskParamMap["vo"] = "wlcg"
|
|
25
|
+
taskParamMap["taskPriority"] = 900
|
|
26
|
+
# taskParamMap['reqID'] = reqIdx
|
|
27
|
+
taskParamMap["architecture"] = "power9"
|
|
28
|
+
taskParamMap["transUses"] = "A"
|
|
29
|
+
taskParamMap["transHome"] = "B"
|
|
30
|
+
taskParamMap["transPath"] = "executable"
|
|
31
|
+
taskParamMap["processingType"] = "step1"
|
|
32
|
+
taskParamMap["prodSourceLabel"] = "test"
|
|
33
|
+
taskParamMap["taskType"] = "test"
|
|
34
|
+
taskParamMap["workingGroup"] = "groupA"
|
|
35
|
+
# taskParamMap['coreCount'] = 1
|
|
36
|
+
# taskParamMap['walltime'] = 1
|
|
37
|
+
taskParamMap["cloud"] = "NA"
|
|
38
|
+
taskParamMap["site"] = "TEST_PQ"
|
|
39
|
+
taskParamMap["log"] = {
|
|
40
|
+
"dataset": logDatasetName,
|
|
41
|
+
"type": "template",
|
|
42
|
+
"param_type": "log",
|
|
43
|
+
"token": "local",
|
|
44
|
+
"destination": "local",
|
|
45
|
+
"value": f"{logDatasetName}.${{SN}}.log.tgz",
|
|
46
|
+
}
|
|
47
|
+
outDatasetName = f"panda.jeditest.{uuid.uuid4()}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
taskParamMap["jobParameters"] = [
|
|
51
|
+
{
|
|
52
|
+
"type": "constant",
|
|
53
|
+
"value": '-i "${IN/T}"',
|
|
54
|
+
},
|
|
55
|
+
{"type": "constant", "value": "ecmEnergy=8000 runNumber=12345"},
|
|
56
|
+
{
|
|
57
|
+
"type": "template",
|
|
58
|
+
"param_type": "output",
|
|
59
|
+
"token": "local",
|
|
60
|
+
"destination": "local",
|
|
61
|
+
"value": f"outputEVNTFile={outDatasetName}.${{SN}}.root",
|
|
62
|
+
"dataset": outDatasetName,
|
|
63
|
+
"offset": 1000,
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"type": "constant",
|
|
67
|
+
"value": "aaaa",
|
|
68
|
+
},
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
print(Client.insertTaskParams(taskParamMap))
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
from userinterface import Client
|
|
5
|
+
|
|
6
|
+
inFileList = ["file1"]
|
|
7
|
+
|
|
8
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
9
|
+
|
|
10
|
+
taskParamMap = {}
|
|
11
|
+
|
|
12
|
+
taskParamMap["nFilesPerJob"] = 1
|
|
13
|
+
taskParamMap["nFiles"] = len(inFileList)
|
|
14
|
+
# taskParamMap['nEventsPerInputFile'] = 10000
|
|
15
|
+
# taskParamMap['nEventsPerJob'] = 10000
|
|
16
|
+
# taskParamMap['nEvents'] = 25000
|
|
17
|
+
taskParamMap["noInput"] = True
|
|
18
|
+
taskParamMap["pfnList"] = inFileList
|
|
19
|
+
# taskParamMap['mergeOutput'] = True
|
|
20
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
21
|
+
taskParamMap["userName"] = "someone"
|
|
22
|
+
taskParamMap["vo"] = "wlcg"
|
|
23
|
+
taskParamMap["taskPriority"] = 900
|
|
24
|
+
# taskParamMap['reqID'] = reqIdx
|
|
25
|
+
taskParamMap["architecture"] = ""
|
|
26
|
+
taskParamMap["transUses"] = ""
|
|
27
|
+
taskParamMap["transHome"] = None
|
|
28
|
+
# taskParamMap['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c'
|
|
29
|
+
taskParamMap["transPath"] = "https://atlpan.web.cern.ch/atlpan/bash-c-enc"
|
|
30
|
+
taskParamMap["encJobParams"] = True
|
|
31
|
+
taskParamMap["processingType"] = "step1"
|
|
32
|
+
taskParamMap["prodSourceLabel"] = "test"
|
|
33
|
+
taskParamMap["taskType"] = "test"
|
|
34
|
+
taskParamMap["workingGroup"] = "lsst"
|
|
35
|
+
# taskParamMap['coreCount'] = 1
|
|
36
|
+
# taskParamMap['walltime'] = 1
|
|
37
|
+
taskParamMap["cloud"] = "LSST"
|
|
38
|
+
taskParamMap["site"] = "DOMA_LSST_GOOGLE_TEST"
|
|
39
|
+
# taskParamMap['site'] = 'DOMA_LSST_SLAC_TEST'
|
|
40
|
+
|
|
41
|
+
# taskParamMap['ramCount'] = 1000
|
|
42
|
+
taskParamMap["ramUnit"] = "MB"
|
|
43
|
+
|
|
44
|
+
"""
|
|
45
|
+
taskParamMap['log'] = {'dataset': logDatasetName,
|
|
46
|
+
'type':'template',
|
|
47
|
+
'param_type':'log',
|
|
48
|
+
'token':'local',
|
|
49
|
+
'destination':'local',
|
|
50
|
+
'value':'{0}.${{SN}}.log.tgz'.format(logDatasetName)}
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
taskParamMap["jobParameters"] = [
|
|
54
|
+
{
|
|
55
|
+
"type": "constant",
|
|
56
|
+
"value": "echo aaa; ls; echo",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"type": "constant",
|
|
60
|
+
"value": "\"'${IN/L}'\"",
|
|
61
|
+
},
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
print(Client.insertTaskParams(taskParamMap))
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import uuid
|
|
3
|
+
|
|
4
|
+
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
|
|
5
|
+
from pandaserver.taskbuffer.JediDatasetSpec import JediDatasetSpec
|
|
6
|
+
from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
|
|
7
|
+
|
|
8
|
+
tbIF = JediTaskBufferInterface()
|
|
9
|
+
tbIF.setupInterface()
|
|
10
|
+
|
|
11
|
+
task = JediTaskSpec()
|
|
12
|
+
task.jediTaskID = sys.argv[1]
|
|
13
|
+
task.taskName = f"pandatest.{uuid.uuid4()}"
|
|
14
|
+
task.status = "defined"
|
|
15
|
+
task.userName = "pandasrv1"
|
|
16
|
+
task.vo = "atlas"
|
|
17
|
+
task.prodSourceLabel = "managed"
|
|
18
|
+
task.taskPriority = 100
|
|
19
|
+
task.currentPriority = task.taskPriority
|
|
20
|
+
task.architecture = "i686-slc5-gcc43-opt"
|
|
21
|
+
task.transUses = "Atlas-17.2.7"
|
|
22
|
+
task.transHome = "AtlasProduction-17.2.8.10"
|
|
23
|
+
task.transPath = "Reco_trf.py"
|
|
24
|
+
task.workQueue_ID = 3
|
|
25
|
+
|
|
26
|
+
tbIF.insertTask_JEDI(task)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
ds = JediDatasetSpec()
|
|
30
|
+
ds.jediTaskID = task.jediTaskID
|
|
31
|
+
if len(sys.argv) > 2:
|
|
32
|
+
ds.datasetName = sys.argv[2]
|
|
33
|
+
else:
|
|
34
|
+
ds.datasetName = "data12_8TeV.00214651.physics_Egamma.merge.AOD.f489_m1261"
|
|
35
|
+
ds.type = "input"
|
|
36
|
+
ds.vo = task.vo
|
|
37
|
+
ds.cloud = "US"
|
|
38
|
+
ds.streamName = "IN"
|
|
39
|
+
ds.status = "defined"
|
|
40
|
+
ds.nFiles = 0
|
|
41
|
+
ds.nFilesUsed = 0
|
|
42
|
+
ds.nFilesFinished = 0
|
|
43
|
+
ds.nFilesFailed = 0
|
|
44
|
+
|
|
45
|
+
st, datasetID = tbIF.insertDataset_JEDI(ds)
|
|
46
|
+
|
|
47
|
+
ds = JediDatasetSpec()
|
|
48
|
+
ds.jediTaskID = task.jediTaskID
|
|
49
|
+
ds.datasetName = "ddo.000001.Atlas.Ideal.DBRelease.v220701"
|
|
50
|
+
ds.type = "input"
|
|
51
|
+
ds.vo = task.vo
|
|
52
|
+
ds.cloud = "US"
|
|
53
|
+
ds.streamName = "DBR"
|
|
54
|
+
ds.status = "defined"
|
|
55
|
+
ds.nFiles = 0
|
|
56
|
+
ds.nFilesUsed = 0
|
|
57
|
+
ds.nFilesFinished = 0
|
|
58
|
+
ds.nFilesFailed = 0
|
|
59
|
+
ds.masterID = datasetID
|
|
60
|
+
ds.attributes = "repeat,nosplit"
|
|
61
|
+
|
|
62
|
+
tbIF.insertDataset_JEDI(ds)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
ds = JediDatasetSpec()
|
|
66
|
+
ds.jediTaskID = task.jediTaskID
|
|
67
|
+
ds.datasetName = f"panda.jeditest.GEN.{uuid.uuid4()}"
|
|
68
|
+
ds.type = "output"
|
|
69
|
+
ds.vo = task.vo
|
|
70
|
+
ds.cloud = "US"
|
|
71
|
+
ds.streamName = "OUT"
|
|
72
|
+
ds.status = "defined"
|
|
73
|
+
ds.nFiles = 0
|
|
74
|
+
ds.nFilesUsed = 0
|
|
75
|
+
ds.nFilesFinished = 0
|
|
76
|
+
ds.nFilesFailed = 0
|
|
77
|
+
|
|
78
|
+
st, datasetID = tbIF.insertDataset_JEDI(ds)
|
|
79
|
+
|
|
80
|
+
tbIF.insertOutputTemplate_JEDI(
|
|
81
|
+
[
|
|
82
|
+
{
|
|
83
|
+
"jediTaskID": task.jediTaskID,
|
|
84
|
+
"datasetID": datasetID,
|
|
85
|
+
"filenameTemplate": f"{ds.datasetName}.${{SN}}.pool.root",
|
|
86
|
+
"serialNr": 1,
|
|
87
|
+
"streamName": "OUT",
|
|
88
|
+
"outtype": ds.type,
|
|
89
|
+
}
|
|
90
|
+
]
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
ds = JediDatasetSpec()
|
|
94
|
+
ds.jediTaskID = task.jediTaskID
|
|
95
|
+
ds.datasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
96
|
+
ds.type = "log"
|
|
97
|
+
ds.vo = task.vo
|
|
98
|
+
ds.cloud = "US"
|
|
99
|
+
ds.streamName = "LOG"
|
|
100
|
+
ds.status = "defined"
|
|
101
|
+
ds.nFiles = 0
|
|
102
|
+
ds.nFilesUsed = 0
|
|
103
|
+
ds.nFilesFinished = 0
|
|
104
|
+
ds.nFilesFailed = 0
|
|
105
|
+
|
|
106
|
+
st, datasetID = tbIF.insertDataset_JEDI(ds)
|
|
107
|
+
|
|
108
|
+
tbIF.insertOutputTemplate_JEDI(
|
|
109
|
+
[
|
|
110
|
+
{
|
|
111
|
+
"jediTaskID": task.jediTaskID,
|
|
112
|
+
"datasetID": datasetID,
|
|
113
|
+
"filenameTemplate": f"{ds.datasetName}.${{SN}}.log.tgz",
|
|
114
|
+
"serialNr": 1,
|
|
115
|
+
"streamName": "LOG",
|
|
116
|
+
"outtype": ds.type,
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
tbIF.insertJobParamsTemplate_JEDI(
|
|
122
|
+
task.jediTaskID,
|
|
123
|
+
'inputAODFile=${IN} maxEvents=1000 RunNumber=213816 autoConfiguration=everything preExec="from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag="BTagCalibALL-07-02"" DBRelease=${DBR} AMITag=p1462 outputNTUP_EMBLLDNFile=${OUT}',
|
|
124
|
+
)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
metaID = sys.argv[1]
|
|
5
|
+
except Exception:
|
|
6
|
+
metaID = None
|
|
7
|
+
import json
|
|
8
|
+
import uuid
|
|
9
|
+
|
|
10
|
+
taskParamMap = {}
|
|
11
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
12
|
+
taskParamMap["userName"] = "pandasrv1"
|
|
13
|
+
taskParamMap["vo"] = "atlas"
|
|
14
|
+
taskParamMap["taskPriority"] = 100
|
|
15
|
+
taskParamMap["architecture"] = "i686-slc5-gcc43-opt"
|
|
16
|
+
taskParamMap["transUses"] = "Atlas-17.2.7"
|
|
17
|
+
taskParamMap["transHome"] = "AtlasProduction-17.2.8.10"
|
|
18
|
+
taskParamMap["transPath"] = "Reco_trf.py"
|
|
19
|
+
taskParamMap["processingType"] = "reco"
|
|
20
|
+
taskParamMap["prodSourceLabel"] = "test"
|
|
21
|
+
taskParamMap["taskType"] = "prod"
|
|
22
|
+
taskParamMap["workingGroup"] = "AP_Higgs"
|
|
23
|
+
taskParamMap["coreCount"] = 1
|
|
24
|
+
taskParamMap["cloud"] = "US"
|
|
25
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
26
|
+
taskParamMap["log"] = {
|
|
27
|
+
"dataset": logDatasetName,
|
|
28
|
+
"type": "template",
|
|
29
|
+
"param_type": "log",
|
|
30
|
+
"token": "ATLASDATADISK",
|
|
31
|
+
"value": f"{logDatasetName}.${{SN}}.log.tgz",
|
|
32
|
+
}
|
|
33
|
+
outDatasetName = f"panda.jeditest.NTUP_EMBLLDN.{uuid.uuid4()}"
|
|
34
|
+
taskParamMap["jobParameters"] = [
|
|
35
|
+
{
|
|
36
|
+
"type": "template",
|
|
37
|
+
"param_type": "input",
|
|
38
|
+
"value": "inputAODFile=${IN}",
|
|
39
|
+
"dataset": "data12_8TeV.00214651.physics_Egamma.merge.AOD.f489_m1261",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"type": "constant",
|
|
43
|
+
"value": 'maxEvents=1000 RunNumber=213816 autoConfiguration=everything preExec="from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag="BTagCalibALL-07-02""',
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"type": "template",
|
|
47
|
+
"param_type": "input",
|
|
48
|
+
"value": "DBRelease=${DBR}",
|
|
49
|
+
"dataset": "ddo.000001.Atlas.Ideal.DBRelease.v220701",
|
|
50
|
+
"attributes": "repeat,nosplit",
|
|
51
|
+
},
|
|
52
|
+
{"type": "constant", "value": "AMITag=p1462"},
|
|
53
|
+
{
|
|
54
|
+
"type": "template",
|
|
55
|
+
"param_type": "output",
|
|
56
|
+
"token": "ATLASDATADISK",
|
|
57
|
+
"value": f"outputNTUP_EMBLLDNFile={outDatasetName}.${{SN}}.pool.root",
|
|
58
|
+
"dataset": outDatasetName,
|
|
59
|
+
},
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
jonStr = json.dumps(taskParamMap)
|
|
63
|
+
|
|
64
|
+
from pandajedi.jedicore.JediTaskBufferInterface import ( # noqa: E402
|
|
65
|
+
JediTaskBufferInterface,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
tbIF = JediTaskBufferInterface()
|
|
69
|
+
tbIF.setupInterface()
|
|
70
|
+
tbIF.insertTaskParams_JEDI(taskParamMap["vo"], taskParamMap["prodSourceLabel"], taskParamMap["userName"], taskParamMap["taskName"], jonStr)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
metaID = sys.argv[1]
|
|
5
|
+
except Exception:
|
|
6
|
+
metaID = None
|
|
7
|
+
import json
|
|
8
|
+
import uuid
|
|
9
|
+
|
|
10
|
+
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
|
|
11
|
+
|
|
12
|
+
tbIF = JediTaskBufferInterface()
|
|
13
|
+
tbIF.setupInterface()
|
|
14
|
+
|
|
15
|
+
reqIdx = 100000000
|
|
16
|
+
dsName = "mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00"
|
|
17
|
+
logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
|
|
18
|
+
|
|
19
|
+
taskParamMap = {}
|
|
20
|
+
|
|
21
|
+
taskParamMap["nFilesPerJob"] = 1
|
|
22
|
+
taskParamMap["nFiles"] = 2
|
|
23
|
+
taskParamMap["nEventsPerInputFile"] = 10000
|
|
24
|
+
taskParamMap["nEventsPerJob"] = 10000
|
|
25
|
+
# taskParamMap['nEvents'] = 25000
|
|
26
|
+
# taskParamMap['skipScout'] = True
|
|
27
|
+
taskParamMap["taskName"] = str(uuid.uuid4())
|
|
28
|
+
taskParamMap["userName"] = "pandasrv1"
|
|
29
|
+
taskParamMap["vo"] = "atlas"
|
|
30
|
+
taskParamMap["taskPriority"] = 900
|
|
31
|
+
# taskParamMap['reqID'] = reqIdx
|
|
32
|
+
taskParamMap["architecture"] = "i686-slc5-gcc43-opt"
|
|
33
|
+
taskParamMap["transUses"] = "Atlas-17.2.2"
|
|
34
|
+
taskParamMap["transHome"] = "AtlasProduction-17.2.2.2"
|
|
35
|
+
taskParamMap["transPath"] = "Generate_trf.py"
|
|
36
|
+
taskParamMap["processingType"] = "evgen"
|
|
37
|
+
taskParamMap["prodSourceLabel"] = "test"
|
|
38
|
+
taskParamMap["taskType"] = "prod"
|
|
39
|
+
taskParamMap["workingGroup"] = "AP_Higgs"
|
|
40
|
+
# taskParamMap['coreCount'] = 1
|
|
41
|
+
# taskParamMap['walltime'] = 1
|
|
42
|
+
taskParamMap["cloud"] = "FR"
|
|
43
|
+
# taskParamMap['site'] = 'FZK-LCG2'
|
|
44
|
+
taskParamMap["log"] = {
|
|
45
|
+
"dataset": logDatasetName,
|
|
46
|
+
"type": "template",
|
|
47
|
+
"param_type": "log",
|
|
48
|
+
"token": "ATLASDATADISK",
|
|
49
|
+
"offset": 1000,
|
|
50
|
+
"value": f"{logDatasetName}.${{SN}}.log.tgz",
|
|
51
|
+
}
|
|
52
|
+
outDatasetName = f"panda.jeditest.EVNT.{uuid.uuid4()}"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
taskParamMap["jobParameters"] = [
|
|
56
|
+
{
|
|
57
|
+
"type": "template",
|
|
58
|
+
"param_type": "input",
|
|
59
|
+
"value": "inputGenConfFile=${IN}",
|
|
60
|
+
"dataset": dsName,
|
|
61
|
+
},
|
|
62
|
+
{"type": "constant", "value": "ecmEnergy=8000 runNumber=12345"},
|
|
63
|
+
{
|
|
64
|
+
"type": "template",
|
|
65
|
+
"value": "maxEvents=${MAXEVENTS}",
|
|
66
|
+
"param_type": "number",
|
|
67
|
+
},
|
|
68
|
+
{"type": "template", "value": "randomSeed=${RNDMSEED}", "param_type": "number", "offset": 1000},
|
|
69
|
+
{
|
|
70
|
+
"type": "constant",
|
|
71
|
+
"value": "jobConfig=MC12JobOptions/MC12.161559.PowHegPythia8_AU2CT10_ggH145_tautaull.py",
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"type": "template",
|
|
75
|
+
"param_type": "output",
|
|
76
|
+
"token": "ATLASDATADISK",
|
|
77
|
+
"value": f"outputEVNTFile={outDatasetName}.${{SN}}.pool.root",
|
|
78
|
+
"dataset": outDatasetName,
|
|
79
|
+
"offset": 1000,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"type": "constant",
|
|
83
|
+
"value": "evgenJobOpts=MC12JobOpts-00-01-06_v1.tar.gz",
|
|
84
|
+
},
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
# params to merge output
|
|
88
|
+
taskParamMap["mergeOutput"] = True
|
|
89
|
+
taskParamMap["mergeSpec"] = {}
|
|
90
|
+
taskParamMap["mergeSpec"]["transPath"] = "HITSMerge_tf.py"
|
|
91
|
+
taskParamMap["mergeSpec"][
|
|
92
|
+
"jobParameters"
|
|
93
|
+
] = "--AMITag s1776 --DBRelease=current --autoConfiguration=everything --outputHitsFile=${OUTPUT0} --inputHitsFile=${TRN_OUTPUT0} --inputLogsFile=${TRN_LOG0}"
|
|
94
|
+
|
|
95
|
+
jonStr = json.dumps(taskParamMap)
|
|
96
|
+
|
|
97
|
+
tbIF.insertTaskParams_JEDI(taskParamMap["vo"], taskParamMap["prodSourceLabel"], taskParamMap["userName"], taskParamMap["taskName"], jonStr)
|