panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,1520 @@
|
|
|
1
|
+
# See the task state diagram for a better understanding of some of the actions https://panda-wms.readthedocs.io/en/latest/terminology/terminology.html#task
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
from typing import Any, Dict, List
|
|
7
|
+
|
|
8
|
+
from pandacommon.pandalogger.LogWrapper import LogWrapper
|
|
9
|
+
from pandacommon.pandalogger.PandaLogger import PandaLogger
|
|
10
|
+
from pandacommon.pandautils.PandaUtils import naive_utcnow
|
|
11
|
+
|
|
12
|
+
from pandaserver.api.v1.common import (
|
|
13
|
+
MESSAGE_TASK_ID,
|
|
14
|
+
extract_production_working_groups,
|
|
15
|
+
generate_response,
|
|
16
|
+
get_dn,
|
|
17
|
+
get_email_address,
|
|
18
|
+
get_fqan,
|
|
19
|
+
has_production_role,
|
|
20
|
+
request_validation,
|
|
21
|
+
)
|
|
22
|
+
from pandaserver.srvcore.CoreUtils import clean_user_id, make_reassign_comment
|
|
23
|
+
from pandaserver.srvcore.panda_request import PandaRequest
|
|
24
|
+
from pandaserver.taskbuffer import task_split_rules
|
|
25
|
+
from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
|
|
26
|
+
from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
|
|
27
|
+
|
|
28
|
+
_logger = PandaLogger().getLogger("api_task")
|
|
29
|
+
|
|
30
|
+
global_task_buffer = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def init_task_buffer(task_buffer: TaskBuffer) -> None:
|
|
34
|
+
"""
|
|
35
|
+
Initialize the task buffer. This method needs to be called before any other method in this module.
|
|
36
|
+
"""
|
|
37
|
+
global global_task_buffer
|
|
38
|
+
global_task_buffer = task_buffer
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
42
|
+
def retry(
|
|
43
|
+
req: PandaRequest,
|
|
44
|
+
task_id: int,
|
|
45
|
+
new_parameters: Dict | str = None,
|
|
46
|
+
no_child_retry: bool = False,
|
|
47
|
+
discard_events: bool = False,
|
|
48
|
+
disable_staging_mode: bool = False,
|
|
49
|
+
keep_gshare_priority: bool = False,
|
|
50
|
+
ignore_hard_exhausted: bool = False,
|
|
51
|
+
) -> Dict[str, Any]:
|
|
52
|
+
"""
|
|
53
|
+
Task retry
|
|
54
|
+
|
|
55
|
+
Retry a given task e.g. in exhausted state. Requires a secure connection without a production role to retry own tasks and with a production role to retry others' tasks.
|
|
56
|
+
|
|
57
|
+
API details:
|
|
58
|
+
HTTP Method: POST
|
|
59
|
+
Path: /v1/task/retry
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
req(PandaRequest): internally generated request object
|
|
63
|
+
task_id(int): JEDI Task ID
|
|
64
|
+
new_parameters(Dict, optional): a dictionary with the new parameters for rerunning the task. The new parameters are merged with the existing ones.
|
|
65
|
+
The parameters are the attributes in the JediTaskSpec object (https://github.com/PanDAWMS/panda-jedi/blob/master/pandajedi/jedicore/JediTaskSpec.py).
|
|
66
|
+
no_child_retry(bool, optional): if True, the child tasks are not retried. Defaults to False
|
|
67
|
+
discard_events(bool, optional): if True, events will be discarded. Defaults to False
|
|
68
|
+
disable_staging_mode(bool, optional): if True, the task skips staging state and directly goes to subsequent state. Defaults to False
|
|
69
|
+
keep_gshare_priority(bool, optional): if True, the task keeps current gshare and priority. Defaults to False
|
|
70
|
+
ignore_hard_exhausted(bool, optional): if True, the task ignores the limits for hard exhausted state and can be retried even if it is very faulty. Defaults to False
|
|
71
|
+
|
|
72
|
+
Returns:
|
|
73
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
74
|
+
"""
|
|
75
|
+
tmp_logger = LogWrapper(_logger, f"retry < task_id={task_id} >")
|
|
76
|
+
tmp_logger.debug("Start")
|
|
77
|
+
|
|
78
|
+
user = get_dn(req)
|
|
79
|
+
production_role = has_production_role(req)
|
|
80
|
+
|
|
81
|
+
# retry with new parameters
|
|
82
|
+
if new_parameters:
|
|
83
|
+
if isinstance(new_parameters, str):
|
|
84
|
+
new_parameters = json.loads(new_parameters)
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
# get original parameters
|
|
88
|
+
old_parameters_json = global_task_buffer.getTaskParamsPanda(task_id)
|
|
89
|
+
old_parameters = json.loads(old_parameters_json)
|
|
90
|
+
|
|
91
|
+
# update with new values
|
|
92
|
+
old_parameters.update(new_parameters)
|
|
93
|
+
final_task_parameters_json = json.dumps(old_parameters)
|
|
94
|
+
|
|
95
|
+
# retry with new parameters
|
|
96
|
+
ret = global_task_buffer.insertTaskParamsPanda(
|
|
97
|
+
final_task_parameters_json,
|
|
98
|
+
user,
|
|
99
|
+
production_role,
|
|
100
|
+
[],
|
|
101
|
+
properErrorCode=True,
|
|
102
|
+
allowActiveTask=True,
|
|
103
|
+
)
|
|
104
|
+
except Exception as e:
|
|
105
|
+
ret = 1, f"new parameter conversion failed with {str(e)}"
|
|
106
|
+
else:
|
|
107
|
+
# disable ignore_hard_exhausted if not production_role
|
|
108
|
+
if not production_role and ignore_hard_exhausted:
|
|
109
|
+
ignore_hard_exhausted = False
|
|
110
|
+
# get command qualifier
|
|
111
|
+
qualifier = JediTaskSpec.get_retry_command_qualifiers(no_child_retry, discard_events, disable_staging_mode, keep_gshare_priority, ignore_hard_exhausted)
|
|
112
|
+
qualifier = " ".join(qualifier)
|
|
113
|
+
# normal retry
|
|
114
|
+
ret = global_task_buffer.sendCommandTaskPanda(
|
|
115
|
+
task_id,
|
|
116
|
+
user,
|
|
117
|
+
production_role,
|
|
118
|
+
"retry",
|
|
119
|
+
properErrorCode=True,
|
|
120
|
+
comQualifier=qualifier,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
if ret[0] == 5:
|
|
124
|
+
# retry failed analysis jobs
|
|
125
|
+
job_ids = global_task_buffer.getJobdefIDsForFailedJob(task_id)
|
|
126
|
+
clean_id = clean_user_id(user)
|
|
127
|
+
for job_id in job_ids:
|
|
128
|
+
global_task_buffer.finalizePendingJobs(clean_id, job_id)
|
|
129
|
+
global_task_buffer.increaseAttemptNrPanda(task_id, 5)
|
|
130
|
+
return_str = f"retry has been triggered for failed jobs while the task is still {ret[1]}"
|
|
131
|
+
if not new_parameters:
|
|
132
|
+
ret = 0, return_str
|
|
133
|
+
else:
|
|
134
|
+
ret = 3, return_str
|
|
135
|
+
|
|
136
|
+
tmp_logger.debug("Done")
|
|
137
|
+
|
|
138
|
+
data, message = ret
|
|
139
|
+
success = True
|
|
140
|
+
return generate_response(success, message, data)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
144
|
+
def resume(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
145
|
+
"""
|
|
146
|
+
Task resume
|
|
147
|
+
|
|
148
|
+
Resume a given task. This transitions a paused or throttled task back to its previous active state. Resume can also be used to kick a task in staging state to the next state.
|
|
149
|
+
Requires a secure connection and production role.
|
|
150
|
+
|
|
151
|
+
API details:
|
|
152
|
+
HTTP Method: POST
|
|
153
|
+
Path: /v1/task/resume
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
req(PandaRequest): internally generated request object
|
|
157
|
+
task_id(int): JEDI Task ID
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
161
|
+
"""
|
|
162
|
+
tmp_logger = LogWrapper(_logger, f"resume < jediTaskID={task_id} >")
|
|
163
|
+
tmp_logger.debug("Start")
|
|
164
|
+
|
|
165
|
+
user = get_dn(req)
|
|
166
|
+
is_production_role = has_production_role(req)
|
|
167
|
+
|
|
168
|
+
# check task_id
|
|
169
|
+
try:
|
|
170
|
+
task_id = int(task_id)
|
|
171
|
+
except ValueError:
|
|
172
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
173
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
174
|
+
|
|
175
|
+
ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "resume", properErrorCode=True)
|
|
176
|
+
data, message = ret
|
|
177
|
+
success = data == 0
|
|
178
|
+
|
|
179
|
+
tmp_logger.debug("Done")
|
|
180
|
+
|
|
181
|
+
return generate_response(success, message, data)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
185
|
+
def release(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
186
|
+
"""
|
|
187
|
+
Task release
|
|
188
|
+
|
|
189
|
+
Release a given task by skipping iDDS for staging. Requires a secure connection and production role.
|
|
190
|
+
|
|
191
|
+
API details:
|
|
192
|
+
HTTP Method: POST
|
|
193
|
+
Path: /v1/task/release
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
req(PandaRequest): internally generated request object
|
|
197
|
+
task_id(int): JEDI Task ID
|
|
198
|
+
|
|
199
|
+
Returns:
|
|
200
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
tmp_logger = LogWrapper(_logger, f"release < task_id={task_id} >")
|
|
204
|
+
tmp_logger.debug("Start")
|
|
205
|
+
|
|
206
|
+
user = get_dn(req)
|
|
207
|
+
is_production_role = has_production_role(req)
|
|
208
|
+
|
|
209
|
+
# check task_id
|
|
210
|
+
try:
|
|
211
|
+
task_id = int(task_id)
|
|
212
|
+
except ValueError:
|
|
213
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
214
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
215
|
+
|
|
216
|
+
ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "release", properErrorCode=True)
|
|
217
|
+
data, message = ret
|
|
218
|
+
success = data == 0
|
|
219
|
+
|
|
220
|
+
tmp_logger.debug("Done")
|
|
221
|
+
|
|
222
|
+
return generate_response(success, message, data)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
# reassign task to site/cloud
|
|
226
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
227
|
+
def reassign(req: PandaRequest, task_id: int, site: str = None, cloud: str = None, nucleus: str = None, mode: str = None):
|
|
228
|
+
"""
|
|
229
|
+
Task reassign
|
|
230
|
+
|
|
231
|
+
Reassign a given task to a site, nucleus or cloud - depending on the parameters. Requires a secure connection.
|
|
232
|
+
|
|
233
|
+
API details:
|
|
234
|
+
HTTP Method: POST
|
|
235
|
+
Path: /v1/task/reassign
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
req(PandaRequest): internally generated request object
|
|
239
|
+
task_id(int): JEDI Task ID
|
|
240
|
+
site(str, optional): site name
|
|
241
|
+
cloud(str, optional): cloud name
|
|
242
|
+
nucleus(str, optional): nucleus name
|
|
243
|
+
mode(str, optional): `kill` (kills all jobs, default), `soft` (kills queued jobs) or `nokill` (doesn't kill jobs)
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
tmp_logger = LogWrapper(_logger, f"reassign < task_id={task_id} >")
|
|
250
|
+
tmp_logger.debug("Start")
|
|
251
|
+
|
|
252
|
+
# check task_id
|
|
253
|
+
try:
|
|
254
|
+
task_id = int(task_id)
|
|
255
|
+
except ValueError:
|
|
256
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
257
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
258
|
+
|
|
259
|
+
user = get_dn(req)
|
|
260
|
+
is_production_role = has_production_role(req)
|
|
261
|
+
|
|
262
|
+
# reassign to site, nucleus or cloud
|
|
263
|
+
comment = make_reassign_comment(site, cloud, nucleus, mode)
|
|
264
|
+
|
|
265
|
+
ret = global_task_buffer.sendCommandTaskPanda(
|
|
266
|
+
task_id,
|
|
267
|
+
user,
|
|
268
|
+
is_production_role,
|
|
269
|
+
"reassign",
|
|
270
|
+
comComment=comment,
|
|
271
|
+
properErrorCode=True,
|
|
272
|
+
)
|
|
273
|
+
data, message = ret
|
|
274
|
+
success = data == 0
|
|
275
|
+
|
|
276
|
+
tmp_logger.debug("Done")
|
|
277
|
+
|
|
278
|
+
return generate_response(success, message, data)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
282
|
+
def pause(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
283
|
+
"""
|
|
284
|
+
Task pause
|
|
285
|
+
|
|
286
|
+
Pause a given task. Requires a secure connection and production role.
|
|
287
|
+
|
|
288
|
+
API details:
|
|
289
|
+
HTTP Method: POST
|
|
290
|
+
Path: /v1/task/pause
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
req(PandaRequest): internally generated request object
|
|
294
|
+
task_id(int): JEDI Task ID
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
tmp_logger = LogWrapper(_logger, f"pause < task_id={task_id} >")
|
|
301
|
+
tmp_logger.debug("Start")
|
|
302
|
+
|
|
303
|
+
try:
|
|
304
|
+
task_id = int(task_id)
|
|
305
|
+
except ValueError:
|
|
306
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
307
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
308
|
+
|
|
309
|
+
user = get_dn(req)
|
|
310
|
+
is_production_role = has_production_role(req)
|
|
311
|
+
|
|
312
|
+
ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "pause", properErrorCode=True)
|
|
313
|
+
data, message = ret
|
|
314
|
+
success = data == 0
|
|
315
|
+
|
|
316
|
+
tmp_logger.debug("Done")
|
|
317
|
+
|
|
318
|
+
return generate_response(success, message, data)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
322
|
+
def kill(req: PandaRequest, task_id: int = None, broadcast: bool = False) -> Dict[str, Any]:
|
|
323
|
+
"""
|
|
324
|
+
Task kill
|
|
325
|
+
|
|
326
|
+
Kill a given task. Requires a secure connection.
|
|
327
|
+
|
|
328
|
+
API details:
|
|
329
|
+
HTTP Method: POST
|
|
330
|
+
Path: /v1/task/kill
|
|
331
|
+
|
|
332
|
+
Args:
|
|
333
|
+
req(PandaRequest): internally generated request object
|
|
334
|
+
task_id(int): JEDI Task ID
|
|
335
|
+
broadcast(bool, optional): broadcast kill command to pilots to kill the jobs
|
|
336
|
+
|
|
337
|
+
Returns:
|
|
338
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
339
|
+
"""
|
|
340
|
+
tmp_logger = LogWrapper(_logger, f"kill < task_id={task_id} >")
|
|
341
|
+
tmp_logger.debug("Start")
|
|
342
|
+
|
|
343
|
+
# check task_id
|
|
344
|
+
try:
|
|
345
|
+
task_id = int(task_id)
|
|
346
|
+
except ValueError:
|
|
347
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
348
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
349
|
+
|
|
350
|
+
user = get_dn(req)
|
|
351
|
+
is_production_role = has_production_role(req)
|
|
352
|
+
|
|
353
|
+
ret = global_task_buffer.sendCommandTaskPanda(
|
|
354
|
+
task_id,
|
|
355
|
+
user,
|
|
356
|
+
is_production_role,
|
|
357
|
+
"kill",
|
|
358
|
+
properErrorCode=True,
|
|
359
|
+
broadcast=broadcast,
|
|
360
|
+
)
|
|
361
|
+
data, message = ret
|
|
362
|
+
success = data == 0
|
|
363
|
+
|
|
364
|
+
tmp_logger.debug("Done")
|
|
365
|
+
|
|
366
|
+
return generate_response(success, message, data)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
370
|
+
def kill_unfinished_jobs(req: PandaRequest, task_id: int, code: int = None, use_email_as_id: bool = False):
|
|
371
|
+
"""
|
|
372
|
+
Kill all unfinished jobs in a task
|
|
373
|
+
|
|
374
|
+
Kills all unfinished jobs in a task. Requires a secure connection.
|
|
375
|
+
|
|
376
|
+
API details:
|
|
377
|
+
HTTP Method: POST
|
|
378
|
+
Path: /v1/task/kill_unfinished_jobs
|
|
379
|
+
|
|
380
|
+
Args:
|
|
381
|
+
req(PandaRequest): internally generated request object containing the env variables
|
|
382
|
+
task_id(int): JEDI task ID
|
|
383
|
+
code(int, optional): The kill code. Defaults to None.
|
|
384
|
+
```
|
|
385
|
+
code
|
|
386
|
+
2: expire
|
|
387
|
+
3: aborted
|
|
388
|
+
4: expire in waiting
|
|
389
|
+
7: retry by server
|
|
390
|
+
8: rebrokerage
|
|
391
|
+
9: force kill
|
|
392
|
+
10: fast rebrokerage in overloaded PQ
|
|
393
|
+
50: kill by JEDI
|
|
394
|
+
51: reassigned by JEDI
|
|
395
|
+
52: force kill by JEDI
|
|
396
|
+
55: killed since task is (almost) done
|
|
397
|
+
60: workload was terminated by the pilot without actual work
|
|
398
|
+
91: kill user jobs with prod role
|
|
399
|
+
99: force kill user jobs with prod role
|
|
400
|
+
```
|
|
401
|
+
use_email_as_id(bool, optional): Use the email as ID. Defaults to False.
|
|
402
|
+
|
|
403
|
+
Returns:
|
|
404
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. The data field contains a list of bools indicating the success of the kill operations.
|
|
405
|
+
"""
|
|
406
|
+
|
|
407
|
+
tmp_logger = LogWrapper(_logger, f"kill_unfinished_jobs")
|
|
408
|
+
|
|
409
|
+
# retrieve the user information
|
|
410
|
+
user = get_dn(req)
|
|
411
|
+
fqans = get_fqan(req)
|
|
412
|
+
is_production_manager = has_production_role(req)
|
|
413
|
+
|
|
414
|
+
if use_email_as_id:
|
|
415
|
+
email = get_email_address(user, tmp_logger)
|
|
416
|
+
if email:
|
|
417
|
+
user = email
|
|
418
|
+
|
|
419
|
+
tmp_logger.debug(f"Start user: {user} code: {code} is_production_manager: {is_production_manager} fqans: {fqans} task_id: {task_id}")
|
|
420
|
+
|
|
421
|
+
# Extract working groups with production role from FQANs
|
|
422
|
+
wg_prod_roles = extract_production_working_groups(fqans)
|
|
423
|
+
|
|
424
|
+
# get PandaIDs
|
|
425
|
+
job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id)
|
|
426
|
+
|
|
427
|
+
# kill jobs
|
|
428
|
+
ret = global_task_buffer.killJobs(job_ids, user, code, is_production_manager, wg_prod_roles, [])
|
|
429
|
+
tmp_logger.debug(f"Done with ret: {ret}")
|
|
430
|
+
return generate_response(True, data=ret)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
434
|
+
def finish(req: PandaRequest, task_id: int, soft: bool = False, broadcast: bool = False) -> Dict[str, Any]:
|
|
435
|
+
"""
|
|
436
|
+
Task finish
|
|
437
|
+
|
|
438
|
+
Finish a given task. Requires a secure connection.
|
|
439
|
+
|
|
440
|
+
API details:
|
|
441
|
+
HTTP Method: POST
|
|
442
|
+
Path: /v1/task/finish
|
|
443
|
+
|
|
444
|
+
Args:
|
|
445
|
+
req(PandaRequest): internally generated request object
|
|
446
|
+
task_id(int): JEDI Task ID
|
|
447
|
+
soft(bool, optional): soft finish
|
|
448
|
+
broadcast(bool, optional): broadcast finish command to pilots
|
|
449
|
+
|
|
450
|
+
Returns:
|
|
451
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
452
|
+
"""
|
|
453
|
+
tmp_logger = LogWrapper(_logger, f"finish < task_id={task_id} soft={soft} broadcast={broadcast} >")
|
|
454
|
+
tmp_logger.debug("Start")
|
|
455
|
+
|
|
456
|
+
qualifier = None
|
|
457
|
+
if soft:
|
|
458
|
+
qualifier = "soft"
|
|
459
|
+
|
|
460
|
+
user = get_dn(req)
|
|
461
|
+
is_production_role = has_production_role(req)
|
|
462
|
+
|
|
463
|
+
# check task_id
|
|
464
|
+
try:
|
|
465
|
+
task_id = int(task_id)
|
|
466
|
+
except ValueError:
|
|
467
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
468
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
469
|
+
|
|
470
|
+
ret = global_task_buffer.sendCommandTaskPanda(
|
|
471
|
+
task_id,
|
|
472
|
+
user,
|
|
473
|
+
is_production_role,
|
|
474
|
+
"finish",
|
|
475
|
+
properErrorCode=True,
|
|
476
|
+
comQualifier=qualifier,
|
|
477
|
+
broadcast=broadcast,
|
|
478
|
+
)
|
|
479
|
+
data, message = ret
|
|
480
|
+
success = data == 0
|
|
481
|
+
|
|
482
|
+
tmp_logger.debug("Done")
|
|
483
|
+
|
|
484
|
+
return generate_response(success, message, data)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
488
|
+
def reactivate(req: PandaRequest, task_id: int, keep_attempt_nr: bool = False, trigger_job_generation: bool = False) -> Dict[str, Any]:
|
|
489
|
+
"""
|
|
490
|
+
Reactivate task
|
|
491
|
+
|
|
492
|
+
Reactivate a given task, i.e. recycle a finished/done task. A reactivated task will generate new jobs and then go to done/finished.
|
|
493
|
+
Requires a secure connection and production role.
|
|
494
|
+
|
|
495
|
+
API details:
|
|
496
|
+
HTTP Method: POST
|
|
497
|
+
Path: /v1/task/reactivate
|
|
498
|
+
|
|
499
|
+
Args:
|
|
500
|
+
req(PandaRequest): internally generated request object
|
|
501
|
+
task_id(int): JEDI Task ID
|
|
502
|
+
keep_attempt_nr(bool, optional): keep the original attempt number
|
|
503
|
+
trigger_job_generation(bool, optional): trigger the job generation
|
|
504
|
+
|
|
505
|
+
Returns:
|
|
506
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
507
|
+
"""
|
|
508
|
+
tmp_logger = LogWrapper(_logger, f"reactivate < task_id={task_id} >")
|
|
509
|
+
tmp_logger.debug("Start")
|
|
510
|
+
|
|
511
|
+
try:
|
|
512
|
+
task_id = int(task_id)
|
|
513
|
+
except ValueError:
|
|
514
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
515
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
516
|
+
|
|
517
|
+
ret = global_task_buffer.reactivateTask(task_id, keep_attempt_nr, trigger_job_generation)
|
|
518
|
+
code, message = ret
|
|
519
|
+
success = code == 0
|
|
520
|
+
|
|
521
|
+
tmp_logger.debug("Done")
|
|
522
|
+
|
|
523
|
+
return generate_response(success, message, code)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
527
|
+
def avalanche(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
528
|
+
"""
|
|
529
|
+
Task avalanche
|
|
530
|
+
|
|
531
|
+
Avalanche a given task. This triggers the avalanche for tasks in scouting state or dynamically reconfigures the task to skip over the scouting state. Requires a secure connection and production role.
|
|
532
|
+
|
|
533
|
+
API details:
|
|
534
|
+
HTTP Method: POST
|
|
535
|
+
Path: /v1/task/avalanche
|
|
536
|
+
|
|
537
|
+
Args:
|
|
538
|
+
req(PandaRequest): internally generated request object
|
|
539
|
+
task_id(int): JEDI Task ID
|
|
540
|
+
|
|
541
|
+
Returns:
|
|
542
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
543
|
+
"""
|
|
544
|
+
tmp_logger = LogWrapper(_logger, f"avalanche < task_id={task_id} >")
|
|
545
|
+
tmp_logger.debug("Start")
|
|
546
|
+
|
|
547
|
+
user = get_dn(req)
|
|
548
|
+
is_production_role = has_production_role(req)
|
|
549
|
+
|
|
550
|
+
# check task_id
|
|
551
|
+
try:
|
|
552
|
+
task_id = int(task_id)
|
|
553
|
+
except ValueError:
|
|
554
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
555
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
556
|
+
|
|
557
|
+
ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "avalanche", properErrorCode=True)
|
|
558
|
+
data, message = ret
|
|
559
|
+
success = data == 0
|
|
560
|
+
|
|
561
|
+
tmp_logger.debug("Done")
|
|
562
|
+
|
|
563
|
+
return generate_response(success, message, data)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
567
|
+
def reload_input(req: PandaRequest, task_id: int, ignore_hard_exhausted: bool = False) -> Dict[str, Any]:
|
|
568
|
+
"""
|
|
569
|
+
Reload input
|
|
570
|
+
|
|
571
|
+
Request to reload the input for a given task. Requires a secure connection and production role.
|
|
572
|
+
|
|
573
|
+
API details:
|
|
574
|
+
HTTP Method: POST
|
|
575
|
+
Path: /v1/task/reload_input
|
|
576
|
+
|
|
577
|
+
Args:
|
|
578
|
+
req(PandaRequest): internally generated request object
|
|
579
|
+
task_id(int): JEDI Task ID
|
|
580
|
+
ignore_hard_exhausted(bool, optional): ignore the limits for hard exhausted
|
|
581
|
+
|
|
582
|
+
Returns:
|
|
583
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
584
|
+
"""
|
|
585
|
+
tmp_logger = LogWrapper(_logger, f"reload_input < task_id={task_id} >")
|
|
586
|
+
tmp_logger.debug("Start")
|
|
587
|
+
|
|
588
|
+
user = get_dn(req)
|
|
589
|
+
is_production_role = has_production_role(req)
|
|
590
|
+
|
|
591
|
+
# check task_id
|
|
592
|
+
try:
|
|
593
|
+
task_id = int(task_id)
|
|
594
|
+
except ValueError:
|
|
595
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
596
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
597
|
+
|
|
598
|
+
# allow ignore_hard_exhausted only for production role
|
|
599
|
+
if not is_production_role and ignore_hard_exhausted:
|
|
600
|
+
ignore_hard_exhausted = False
|
|
601
|
+
|
|
602
|
+
# specify ignore_hard_exhausted as a command qualifier in command's comment
|
|
603
|
+
com_qualifier = JediTaskSpec.get_retry_command_qualifiers(ignore_hard_exhausted=ignore_hard_exhausted)
|
|
604
|
+
com_comment = json.dumps([{}, com_qualifier])
|
|
605
|
+
|
|
606
|
+
ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "incexec", comComment=com_comment, properErrorCode=True)
|
|
607
|
+
data, message = ret
|
|
608
|
+
success = data == 0
|
|
609
|
+
|
|
610
|
+
tmp_logger.debug("Done")
|
|
611
|
+
|
|
612
|
+
return generate_response(success, message, data)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
616
|
+
def reassign_global_share(req: PandaRequest, task_id_list: List[int], share: str, reassign_running_jobs: bool) -> Dict[str, Any]:
|
|
617
|
+
"""
|
|
618
|
+
Reassign the global share of a task
|
|
619
|
+
|
|
620
|
+
Reassign the global share of a task. Requires a secure connection and production role.
|
|
621
|
+
|
|
622
|
+
API details:
|
|
623
|
+
HTTP Method: POST
|
|
624
|
+
Path: /v1/task/reassign_global_share
|
|
625
|
+
|
|
626
|
+
Args:
|
|
627
|
+
req(PandaRequest): internally generated request object
|
|
628
|
+
task_id_list(list): List of JEDI task IDs to reassign
|
|
629
|
+
share(str): destination share
|
|
630
|
+
reassign_running_jobs(bool): whether you want to reassign existing running jobs
|
|
631
|
+
|
|
632
|
+
Returns:
|
|
633
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
634
|
+
"""
|
|
635
|
+
|
|
636
|
+
tmp_logger = LogWrapper(_logger, f"reassign_global_share < task_id_list={task_id_list} share={share} reassign_running_jobs={reassign_running_jobs} >")
|
|
637
|
+
tmp_logger.debug("Start")
|
|
638
|
+
|
|
639
|
+
if not isinstance(task_id_list, list) or not isinstance(share, str):
|
|
640
|
+
tmp_logger.error("Failed due to invalid task list")
|
|
641
|
+
return generate_response(False, message="wrong parameters: task_ids must be list and share must be string")
|
|
642
|
+
|
|
643
|
+
code, message = global_task_buffer.reassignShare(task_id_list, share, reassign_running_jobs)
|
|
644
|
+
success = code == 0
|
|
645
|
+
|
|
646
|
+
tmp_logger.debug("Done")
|
|
647
|
+
|
|
648
|
+
return generate_response(success, message, code)
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
652
|
+
def enable_jumbo_jobs(req: PandaRequest, task_id: int, jumbo_jobs_total: int, jumbo_jobs_per_site: int = None):
|
|
653
|
+
"""
|
|
654
|
+
Enable Jumbo jobs
|
|
655
|
+
|
|
656
|
+
Enables the Jumbo jobs for a given task ID. Requires a secure connection and production role.
|
|
657
|
+
|
|
658
|
+
API details:
|
|
659
|
+
HTTP Method: POST
|
|
660
|
+
Path: /v1/task/enable_jumbo_jobs
|
|
661
|
+
|
|
662
|
+
Args:
|
|
663
|
+
req(PandaRequest): internally generated request object
|
|
664
|
+
task_id(int): JEDI task ID
|
|
665
|
+
jumbo_jobs_total(int): Total number of jumbo jobs
|
|
666
|
+
jumbo_jobs_per_site(int): Number of jumbo jobs per site. Defaults to `jumbo_jobs_total`.
|
|
667
|
+
|
|
668
|
+
Returns:
|
|
669
|
+
dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
670
|
+
"""
|
|
671
|
+
|
|
672
|
+
tmp_logger = LogWrapper(_logger, f"enable_jumbo_jobs < task_id={task_id} jumbo_jobs_total={jumbo_jobs_total} n_jumbo_jobs_per_site={jumbo_jobs_per_site} >")
|
|
673
|
+
tmp_logger.debug("Start")
|
|
674
|
+
|
|
675
|
+
if not jumbo_jobs_per_site:
|
|
676
|
+
jumbo_jobs_per_site = jumbo_jobs_total
|
|
677
|
+
|
|
678
|
+
code, message = global_task_buffer.enableJumboJobs(task_id, jumbo_jobs_total, jumbo_jobs_per_site)
|
|
679
|
+
if jumbo_jobs_total > 0 and code == 0:
|
|
680
|
+
tmp_logger.debug("Calling task avalanche")
|
|
681
|
+
avalanche(task_id)
|
|
682
|
+
|
|
683
|
+
success = code == 0
|
|
684
|
+
|
|
685
|
+
tmp_logger.debug("Done")
|
|
686
|
+
return generate_response(success, message, code)
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
690
|
+
def get_jumbo_job_datasets(req: PandaRequest, from_offset: int, to_offset: int = 0) -> Dict:
|
|
691
|
+
"""
|
|
692
|
+
Get jumbo job datasets
|
|
693
|
+
|
|
694
|
+
Gets a map of the jumbo-job-enabled tasks to their datasets, filtering by the last modification time (now - from_offset to now - to_offset).
|
|
695
|
+
Requires a secure connection.
|
|
696
|
+
|
|
697
|
+
API details:
|
|
698
|
+
HTTP Method: GET
|
|
699
|
+
Path: /v1/task/get_jumbo_job_datasets
|
|
700
|
+
|
|
701
|
+
Args:
|
|
702
|
+
req(PandaRequest): internally generated request object
|
|
703
|
+
from_offset(int): `now - from_offset` in days will serve as the floor for modification time (Previously called n_days)
|
|
704
|
+
to_offset(int, optional): `now - to_offset` in days will serve as the ceiling for modification time. Defaults to 0. (Previously called grace_period)
|
|
705
|
+
|
|
706
|
+
Returns:
|
|
707
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
708
|
+
When successful, the data field contains the dictionary of JEDI task IDs to datasets.
|
|
709
|
+
"""
|
|
710
|
+
tmp_logger = LogWrapper(_logger, f"get_jumbo_job_datasets")
|
|
711
|
+
|
|
712
|
+
tmp_logger.debug("Start")
|
|
713
|
+
jumbo_datasets = global_task_buffer.getJumboJobDatasets(from_offset, to_offset)
|
|
714
|
+
tmp_logger.debug("Done")
|
|
715
|
+
|
|
716
|
+
return generate_response(True, data=jumbo_datasets)
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
720
|
+
def enable_job_cloning(
|
|
721
|
+
req: PandaRequest,
|
|
722
|
+
jedi_task_id: int,
|
|
723
|
+
mode: str = None,
|
|
724
|
+
multiplicity: int = None,
|
|
725
|
+
num_sites: int = None,
|
|
726
|
+
) -> Dict[str, Any]:
|
|
727
|
+
"""
|
|
728
|
+
Enable job cloning
|
|
729
|
+
|
|
730
|
+
Enable job cloning for a given task. Requires secure connection and production role.
|
|
731
|
+
|
|
732
|
+
API details:
|
|
733
|
+
HTTP Method: POST
|
|
734
|
+
Path: /v1/task/enable_job_cloning
|
|
735
|
+
|
|
736
|
+
Args:
|
|
737
|
+
req(PandaRequest): internally generated request object
|
|
738
|
+
jedi_task_id(int): JEDI Task ID
|
|
739
|
+
mode(str, optional): mode of operation, runonce or storeonce
|
|
740
|
+
multiplicity(int, optional): number of clones to be created for each target
|
|
741
|
+
num_sites(int, optional): number of sites to be used for each target
|
|
742
|
+
|
|
743
|
+
Returns:
|
|
744
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
745
|
+
When there was an error, the message field contains the description.
|
|
746
|
+
"""
|
|
747
|
+
tmp_logger = LogWrapper(_logger, f"enable_job_cloning < jedi_task_id={jedi_task_id} >")
|
|
748
|
+
tmp_logger.debug("Start")
|
|
749
|
+
success, message = global_task_buffer.enable_job_cloning(jedi_task_id, mode, multiplicity, num_sites)
|
|
750
|
+
tmp_logger.debug("Done")
|
|
751
|
+
return generate_response(success, message)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
755
|
+
def disable_job_cloning(req: PandaRequest, jedi_task_id: int) -> Dict[str, Any]:
|
|
756
|
+
"""
|
|
757
|
+
Disable job cloning
|
|
758
|
+
|
|
759
|
+
Disable job cloning for a given task. Requires secure connection and production role.
|
|
760
|
+
|
|
761
|
+
API details:
|
|
762
|
+
HTTP Method: POST
|
|
763
|
+
Path: /v1/task/disable_job_cloning
|
|
764
|
+
|
|
765
|
+
Args:
|
|
766
|
+
req(PandaRequest): internally generated request object
|
|
767
|
+
jedi_task_id(int): JEDI Task ID
|
|
768
|
+
|
|
769
|
+
Returns:
|
|
770
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
771
|
+
When there was an error, the message field contains the description.
|
|
772
|
+
"""
|
|
773
|
+
tmp_logger = LogWrapper(_logger, f"disable_job_cloning < jedi_task_id={jedi_task_id} >")
|
|
774
|
+
tmp_logger.debug("Start")
|
|
775
|
+
success, message = global_task_buffer.disable_job_cloning(jedi_task_id)
|
|
776
|
+
tmp_logger.debug("Done")
|
|
777
|
+
return generate_response(success, message)
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
781
|
+
def increase_attempts(req: PandaRequest, task_id: int, increase: int) -> Dict[str, Any]:
|
|
782
|
+
"""
|
|
783
|
+
Increase possible task attempts
|
|
784
|
+
|
|
785
|
+
Increase possible task attempts. Requires secure connection and production role.
|
|
786
|
+
|
|
787
|
+
API details:
|
|
788
|
+
HTTP Method: POST
|
|
789
|
+
Path: /v1/task/increase_attempts
|
|
790
|
+
|
|
791
|
+
Args:
|
|
792
|
+
req(PandaRequest): internally generated request object
|
|
793
|
+
task_id(int): JEDI Task ID
|
|
794
|
+
increase(int): number of attempts to increase
|
|
795
|
+
|
|
796
|
+
Returns:
|
|
797
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
798
|
+
When there was an error, the message field contains the description and the data field contains the code.
|
|
799
|
+
"""
|
|
800
|
+
tmp_logger = LogWrapper(_logger, f"increase_attempt_number task_id={task_id}")
|
|
801
|
+
tmp_logger.debug("Start")
|
|
802
|
+
|
|
803
|
+
try:
|
|
804
|
+
task_id = int(task_id)
|
|
805
|
+
except Exception:
|
|
806
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
807
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
808
|
+
|
|
809
|
+
# check value for increase
|
|
810
|
+
try:
|
|
811
|
+
increase = int(increase)
|
|
812
|
+
if increase < 0:
|
|
813
|
+
raise ValueError
|
|
814
|
+
except Exception:
|
|
815
|
+
message = f"increase must be a positive integer, got {increase}"
|
|
816
|
+
tmp_logger.error(message)
|
|
817
|
+
return generate_response(False, message=message)
|
|
818
|
+
|
|
819
|
+
code, message = global_task_buffer.increaseAttemptNrPanda(task_id, increase)
|
|
820
|
+
success = code == 0
|
|
821
|
+
|
|
822
|
+
tmp_logger.debug("Done")
|
|
823
|
+
|
|
824
|
+
return generate_response(success, message, code)
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
828
|
+
def get_status(req, task_id):
|
|
829
|
+
"""
|
|
830
|
+
Get task status
|
|
831
|
+
|
|
832
|
+
Get the status of a given task. Requires secure connection.
|
|
833
|
+
|
|
834
|
+
API details:
|
|
835
|
+
HTTP Method: GET
|
|
836
|
+
Path: /v1/task/get_status
|
|
837
|
+
|
|
838
|
+
Args:
|
|
839
|
+
req(PandaRequest): internally generated request object
|
|
840
|
+
task_id(int): JEDI Task ID
|
|
841
|
+
|
|
842
|
+
Returns:
|
|
843
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
844
|
+
When successful, the data field contains the status of the task.
|
|
845
|
+
When there was an error, the message field contains the description.
|
|
846
|
+
"""
|
|
847
|
+
tmp_logger = LogWrapper(_logger, f"get_status < task_id={task_id} >")
|
|
848
|
+
tmp_logger.debug("Start")
|
|
849
|
+
|
|
850
|
+
try:
|
|
851
|
+
task_id = int(task_id)
|
|
852
|
+
except ValueError:
|
|
853
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
854
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
855
|
+
|
|
856
|
+
ret = global_task_buffer.getTaskStatus(task_id)
|
|
857
|
+
if not ret:
|
|
858
|
+
return generate_response(False, message="Task not found")
|
|
859
|
+
status = ret[0]
|
|
860
|
+
|
|
861
|
+
tmp_logger.debug("Done")
|
|
862
|
+
|
|
863
|
+
return generate_response(True, data=status)
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
@request_validation(_logger, request_method="GET", secure=True)
|
|
867
|
+
def get_details(req: PandaRequest, task_id: int, include_parameters: bool = False, include_status: bool = False):
|
|
868
|
+
"""
|
|
869
|
+
Get task details
|
|
870
|
+
|
|
871
|
+
Get the details of a given task. Requires secure connection.
|
|
872
|
+
|
|
873
|
+
API details:
|
|
874
|
+
HTTP Method: GET
|
|
875
|
+
Path: /v1/task/get_details
|
|
876
|
+
|
|
877
|
+
Args:
|
|
878
|
+
req(PandaRequest): internally generated request object
|
|
879
|
+
task_id(int): JEDI Task ID
|
|
880
|
+
include_parameters(bool, optional): flag to include task parameter information (Previously fullFlag)
|
|
881
|
+
include_status(bool, optional): flag to include status information (Previously withTaskInfo)
|
|
882
|
+
|
|
883
|
+
Returns:
|
|
884
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
885
|
+
When successful, the data field contains the task details.
|
|
886
|
+
When there was an error, the message field contains the description.
|
|
887
|
+
"""
|
|
888
|
+
tmp_logger = LogWrapper(_logger, f"get_details < task_id={task_id} include_parameters={include_parameters} include_status={include_status} >")
|
|
889
|
+
tmp_logger.debug("Start")
|
|
890
|
+
|
|
891
|
+
details = global_task_buffer.getJediTaskDetails(task_id, include_parameters, include_status)
|
|
892
|
+
if not details:
|
|
893
|
+
tmp_logger.error("Task not found or error retrieving the details")
|
|
894
|
+
return generate_response(False, message="Task not found or error retrieving the details")
|
|
895
|
+
|
|
896
|
+
tmp_logger.debug("Done")
|
|
897
|
+
|
|
898
|
+
return generate_response(True, data=details)
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
902
|
+
def change_attribute(req: PandaRequest, task_id: int, attribute_name: str, value: int) -> Dict[str, Any]:
|
|
903
|
+
"""
|
|
904
|
+
Change a task attribute
|
|
905
|
+
|
|
906
|
+
Change a task attribute within the list of valid attributes ("ramCount", "wallTime", "cpuTime", "coreCount").
|
|
907
|
+
Requires a secure connection and production role.
|
|
908
|
+
|
|
909
|
+
API details:
|
|
910
|
+
HTTP Method: POST
|
|
911
|
+
Path: /v1/task/change_attribute
|
|
912
|
+
|
|
913
|
+
Args:
|
|
914
|
+
req(PandaRequest): internally generated request object
|
|
915
|
+
task_id(int): JEDI task ID
|
|
916
|
+
attribute_name(str): attribute to change
|
|
917
|
+
value(int): value to set to the attribute
|
|
918
|
+
|
|
919
|
+
Returns:
|
|
920
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
|
|
921
|
+
Return code in the data field, 0 for success, others for failure.
|
|
922
|
+
"""
|
|
923
|
+
tmp_logger = LogWrapper(_logger, f"change_attribute < task_id={task_id} attribute_name={attribute_name} value={value} >")
|
|
924
|
+
tmp_logger.debug("Start")
|
|
925
|
+
|
|
926
|
+
# check task_id
|
|
927
|
+
try:
|
|
928
|
+
task_id = int(task_id)
|
|
929
|
+
except ValueError:
|
|
930
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
931
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
932
|
+
|
|
933
|
+
# check if attribute_name is valid
|
|
934
|
+
valid_attributes = ["ramCount", "wallTime", "cpuTime", "coreCount"]
|
|
935
|
+
if attribute_name not in valid_attributes:
|
|
936
|
+
tmp_logger.error("Failed due to invalid attribute_name")
|
|
937
|
+
return generate_response(False, message=f"{attribute_name} is not a valid attribute. Valid attributes are {valid_attributes}")
|
|
938
|
+
|
|
939
|
+
n_tasks_changed = global_task_buffer.changeTaskAttributePanda(task_id, attribute_name, value)
|
|
940
|
+
if n_tasks_changed is None: # method excepted
|
|
941
|
+
tmp_logger.error("Failed due to exception while changing the attribute")
|
|
942
|
+
return generate_response(False, message="Exception while changing the attribute")
|
|
943
|
+
|
|
944
|
+
if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
|
|
945
|
+
tmp_logger.error("Failed due to task not found")
|
|
946
|
+
return generate_response(False, message="Task not found")
|
|
947
|
+
|
|
948
|
+
tmp_logger.debug("Done")
|
|
949
|
+
return generate_response(True, message=f"{n_tasks_changed} tasks changed")
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
953
|
+
def change_modification_time(req: PandaRequest, task_id: int, hour_offset: int) -> Dict[str, Any]:
|
|
954
|
+
"""
|
|
955
|
+
Change task modification time
|
|
956
|
+
|
|
957
|
+
Change the modification time for a task to `now() + hour_offset`. Requires a secure connection and production role.
|
|
958
|
+
|
|
959
|
+
API details:
|
|
960
|
+
HTTP Method: POST
|
|
961
|
+
Path: /v1/task/change_modification_time
|
|
962
|
+
|
|
963
|
+
Args:
|
|
964
|
+
req(PandaRequest): internally generated request object
|
|
965
|
+
task_id(int): JEDI task ID
|
|
966
|
+
hour_offset(int): number of hours to add to the current time. Use a negative value (e.g. -12) to trigger task brokerage.
|
|
967
|
+
|
|
968
|
+
Returns:
|
|
969
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
970
|
+
"""
|
|
971
|
+
tmp_logger = LogWrapper(_logger, f"change_modification_time < task_id={task_id} hour_offset={hour_offset} >")
|
|
972
|
+
tmp_logger.debug("Start")
|
|
973
|
+
|
|
974
|
+
# check offset
|
|
975
|
+
try:
|
|
976
|
+
new_modification_time = datetime.datetime.now() + datetime.timedelta(hours=hour_offset)
|
|
977
|
+
except ValueError:
|
|
978
|
+
tmp_logger.error("Failed due to invalid hour_offset")
|
|
979
|
+
return generate_response(False, message=f"failed to convert {hour_offset} to time")
|
|
980
|
+
|
|
981
|
+
n_tasks_changed = global_task_buffer.changeTaskAttributePanda(task_id, "modificationTime", new_modification_time)
|
|
982
|
+
if n_tasks_changed is None: # method excepted
|
|
983
|
+
tmp_logger.error("Failed due to exception while changing the attribute")
|
|
984
|
+
return generate_response(False, message="Exception while changing the attribute")
|
|
985
|
+
|
|
986
|
+
if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
|
|
987
|
+
tmp_logger.error("Failed due to task not found")
|
|
988
|
+
return generate_response(False, message="Task not found")
|
|
989
|
+
|
|
990
|
+
tmp_logger.debug("Done")
|
|
991
|
+
return generate_response(True, message=f"{n_tasks_changed} tasks changed")
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
995
|
+
def change_priority(req: PandaRequest, task_id: int, priority: int):
|
|
996
|
+
"""
|
|
997
|
+
Change priority
|
|
998
|
+
|
|
999
|
+
Change the priority of a given task. Requires a secure connection and production role.
|
|
1000
|
+
|
|
1001
|
+
API details:
|
|
1002
|
+
HTTP Method: POST
|
|
1003
|
+
Path: /v1/task/change_priority
|
|
1004
|
+
|
|
1005
|
+
Args:
|
|
1006
|
+
req(PandaRequest): internally generated request object
|
|
1007
|
+
task_id(int): JEDI task ID
|
|
1008
|
+
priority(int): new priority for the task
|
|
1009
|
+
|
|
1010
|
+
Returns:
|
|
1011
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
1012
|
+
True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
1013
|
+
"""
|
|
1014
|
+
tmp_logger = LogWrapper(_logger, f"change_priority < task_id={task_id} priority={priority} >")
|
|
1015
|
+
tmp_logger.debug("Start")
|
|
1016
|
+
|
|
1017
|
+
# check task_id
|
|
1018
|
+
try:
|
|
1019
|
+
task_id = int(task_id)
|
|
1020
|
+
except ValueError:
|
|
1021
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1022
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1023
|
+
|
|
1024
|
+
# check priority
|
|
1025
|
+
try:
|
|
1026
|
+
priority = int(priority)
|
|
1027
|
+
except ValueError:
|
|
1028
|
+
tmp_logger.error("Failed due to invalid priority")
|
|
1029
|
+
return generate_response(False, message="priority must be an integer")
|
|
1030
|
+
|
|
1031
|
+
n_tasks_changed = global_task_buffer.changeTaskPriorityPanda(task_id, priority)
|
|
1032
|
+
|
|
1033
|
+
if n_tasks_changed is None: # method excepted
|
|
1034
|
+
tmp_logger.error("Failed due to exception while changing the priority")
|
|
1035
|
+
return generate_response(False, message="Exception while changing the priority")
|
|
1036
|
+
|
|
1037
|
+
if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
|
|
1038
|
+
tmp_logger.error("Failed due to task not found")
|
|
1039
|
+
return generate_response(False, message="Task not found")
|
|
1040
|
+
|
|
1041
|
+
tmp_logger.debug("Done")
|
|
1042
|
+
|
|
1043
|
+
return generate_response(True, message=f"{n_tasks_changed} tasks changed")
|
|
1044
|
+
|
|
1045
|
+
|
|
1046
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
1047
|
+
def change_split_rule(req: PandaRequest, task_id: int, attribute_name: str, value: str) -> Dict[str, Any]:
|
|
1048
|
+
"""
|
|
1049
|
+
Change the split rule
|
|
1050
|
+
|
|
1051
|
+
Change the split rule for a task by modifying or adding an `attribute_name=value pair`. Requires a secure connection and production role.
|
|
1052
|
+
|
|
1053
|
+
API details:
|
|
1054
|
+
HTTP Method: POST
|
|
1055
|
+
Path: /v1/task/change_split_rule
|
|
1056
|
+
|
|
1057
|
+
Args:
|
|
1058
|
+
req(PandaRequest): internally generated request object
|
|
1059
|
+
task_id(int): JEDI task ID
|
|
1060
|
+
attribute_name(str): split rule attribute to change. The allowed attributes are defined in `task_split_rules.changeable_split_rule_tags`
|
|
1061
|
+
value(str): value to set to the attribute
|
|
1062
|
+
|
|
1063
|
+
Returns:
|
|
1064
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
|
|
1065
|
+
Return code in the data field, 0 for success, others for failure.
|
|
1066
|
+
"""
|
|
1067
|
+
tmp_logger = LogWrapper(_logger, f"change_split_rule < task_id={task_id} attribute_name={attribute_name} value={value} >")
|
|
1068
|
+
tmp_logger.debug("Start")
|
|
1069
|
+
|
|
1070
|
+
# check task_id
|
|
1071
|
+
try:
|
|
1072
|
+
task_id = int(task_id)
|
|
1073
|
+
except ValueError:
|
|
1074
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1075
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1076
|
+
|
|
1077
|
+
# check attribute
|
|
1078
|
+
if attribute_name not in task_split_rules.changeable_split_rule_tags:
|
|
1079
|
+
tmp_logger.error("Failed due to invalid attribute_name")
|
|
1080
|
+
return generate_response(
|
|
1081
|
+
False, message=f"{attribute_name} is not a valid attribute. Valid attributes are {task_split_rules.changeable_split_rule_tags}", data=2
|
|
1082
|
+
)
|
|
1083
|
+
|
|
1084
|
+
n_tasks_changed = global_task_buffer.changeTaskSplitRulePanda(task_id, attribute_name, value)
|
|
1085
|
+
if n_tasks_changed is None: # method excepted
|
|
1086
|
+
tmp_logger.error("Failed due to exception while changing the split rule")
|
|
1087
|
+
return generate_response(False, message="Exception while changing the split rule")
|
|
1088
|
+
|
|
1089
|
+
if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
|
|
1090
|
+
tmp_logger.error("Failed due to task not found")
|
|
1091
|
+
return generate_response(False, message="Task not found")
|
|
1092
|
+
|
|
1093
|
+
tmp_logger.debug("Done")
|
|
1094
|
+
|
|
1095
|
+
return generate_response(True, message=f"{n_tasks_changed} tasks changed")
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1099
|
+
def get_tasks_modified_since(req, since: str, dn: str = None, full: bool = False, min_task_id: int = None, prod_source_label: str = "user") -> Dict[str, Any]:
|
|
1100
|
+
"""
|
|
1101
|
+
Get tasks modified since
|
|
1102
|
+
|
|
1103
|
+
Get the tasks with `modificationtime > since`. Requires a secure connection.
|
|
1104
|
+
|
|
1105
|
+
API details:
|
|
1106
|
+
HTTP Method: GET
|
|
1107
|
+
Path: /v1/task/get_tasks_modified_since
|
|
1108
|
+
|
|
1109
|
+
Args:
|
|
1110
|
+
req(PandaRequest): internally generated request object
|
|
1111
|
+
since(str): time in the format `%Y-%m-%d %H:%M:%S`, e.g. `2024-12-18 14:30:45`. The tasks with `modificationtime > since` will be returned
|
|
1112
|
+
dn(str, optional): user DN
|
|
1113
|
+
full(bool, optional): flag to include full task information. If `full=False` the basic fields are `jediTaskID, modificationTime, status, processingType, transUses, transHome, architecture, reqID, creationDate, site, cloud, taskName`
|
|
1114
|
+
min_task_id(int, optional): minimum task ID
|
|
1115
|
+
prod_source_label(str, optional): task type (e.g. `user`, `managed`, `test`, etc.)
|
|
1116
|
+
|
|
1117
|
+
Returns:
|
|
1118
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
1119
|
+
"""
|
|
1120
|
+
tmp_logger = LogWrapper(_logger, "get_tasks_modified_since")
|
|
1121
|
+
tmp_logger.debug("Start")
|
|
1122
|
+
|
|
1123
|
+
if not dn:
|
|
1124
|
+
dn = get_dn(req)
|
|
1125
|
+
|
|
1126
|
+
tmp_logger.debug(f"parameters dn:{dn} since:{since} full:{full} min_task_id:{min_task_id} prod_source_label:{prod_source_label}")
|
|
1127
|
+
|
|
1128
|
+
tasks = global_task_buffer.getJediTasksInTimeRange(dn, since, full, min_task_id, prod_source_label)
|
|
1129
|
+
|
|
1130
|
+
tmp_logger.debug("Done")
|
|
1131
|
+
|
|
1132
|
+
return generate_response(True, data=tasks)
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1136
|
+
def get_datasets_and_files(req, task_id, dataset_types: List[str] = ("input", "pseudo_input")) -> Dict[str, Any]:
|
|
1137
|
+
"""
|
|
1138
|
+
Get datasets and files
|
|
1139
|
+
|
|
1140
|
+
Get the files in the datasets associated to a given task. You can filter passing a list of dataset types. The return format is:
|
|
1141
|
+
```
|
|
1142
|
+
[
|
|
1143
|
+
{
|
|
1144
|
+
"dataset": {
|
|
1145
|
+
"name": dataset_name,
|
|
1146
|
+
"id": dataset_id
|
|
1147
|
+
},
|
|
1148
|
+
"files": [
|
|
1149
|
+
{
|
|
1150
|
+
"lfn": lfn,
|
|
1151
|
+
"scope": file_scope,
|
|
1152
|
+
"id": file_id,
|
|
1153
|
+
"status": status
|
|
1154
|
+
},
|
|
1155
|
+
...
|
|
1156
|
+
]
|
|
1157
|
+
},
|
|
1158
|
+
...
|
|
1159
|
+
]
|
|
1160
|
+
```
|
|
1161
|
+
Requires a secure connection.
|
|
1162
|
+
|
|
1163
|
+
API details:
|
|
1164
|
+
HTTP Method: GET
|
|
1165
|
+
Path: /v1/task/get_datasets_and_files
|
|
1166
|
+
|
|
1167
|
+
Args:
|
|
1168
|
+
req(PandaRequest): internally generated request object
|
|
1169
|
+
task_id(int): JEDI task ID
|
|
1170
|
+
dataset_types(List, optional): list of dataset types, defaults to `["input", "pseudo_input"]`
|
|
1171
|
+
|
|
1172
|
+
Returns:
|
|
1173
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
1174
|
+
"""
|
|
1175
|
+
tmp_logger = LogWrapper(_logger, f"get_datasets_and_files < task_id={task_id} dataset_types={dataset_types} >")
|
|
1176
|
+
tmp_logger.debug("Start")
|
|
1177
|
+
|
|
1178
|
+
data = global_task_buffer.get_files_in_datasets(task_id, dataset_types)
|
|
1179
|
+
if data is None:
|
|
1180
|
+
tmp_logger.error("Failed due to exception while gathering files")
|
|
1181
|
+
return generate_response(False, message="Database exception while gathering files")
|
|
1182
|
+
|
|
1183
|
+
if data == []:
|
|
1184
|
+
tmp_logger.error("Failed due to no data found for the task")
|
|
1185
|
+
return generate_response(False, message="No data found for the task")
|
|
1186
|
+
|
|
1187
|
+
tmp_logger.debug("Done")
|
|
1188
|
+
|
|
1189
|
+
return generate_response(True, data=data)
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1193
|
+
def get_job_ids(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
1194
|
+
"""
|
|
1195
|
+
Get job IDs
|
|
1196
|
+
|
|
1197
|
+
Get a list with the job IDs `[job_id, ...]` (in any status) associated to a given task. Requires a secure connection.
|
|
1198
|
+
|
|
1199
|
+
API details:
|
|
1200
|
+
HTTP Method: GET
|
|
1201
|
+
Path: /v1/task/get_job_ids
|
|
1202
|
+
|
|
1203
|
+
Args:
|
|
1204
|
+
req(PandaRequest): internally generated request object
|
|
1205
|
+
task_id(int): JEDI task ID
|
|
1206
|
+
|
|
1207
|
+
Returns:
|
|
1208
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
1209
|
+
"""
|
|
1210
|
+
tmp_logger = LogWrapper(_logger, f"get_job_ids < task_id={task_id} >")
|
|
1211
|
+
tmp_logger.debug("Start")
|
|
1212
|
+
|
|
1213
|
+
try:
|
|
1214
|
+
task_id = int(task_id)
|
|
1215
|
+
except ValueError:
|
|
1216
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1217
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1218
|
+
|
|
1219
|
+
job_id_list = global_task_buffer.getPandaIDsWithTaskID(task_id)
|
|
1220
|
+
|
|
1221
|
+
tmp_logger.debug("Done")
|
|
1222
|
+
|
|
1223
|
+
return generate_response(True, data=job_id_list)
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
1227
|
+
def submit(req: PandaRequest, task_parameters: Dict, parent_tid: int = None) -> Dict[str, Any]:
|
|
1228
|
+
"""
|
|
1229
|
+
Register task
|
|
1230
|
+
|
|
1231
|
+
Insert the task parameters to register a task. Requires a secure connection.
|
|
1232
|
+
|
|
1233
|
+
API details:
|
|
1234
|
+
HTTP Method: POST
|
|
1235
|
+
Path: /v1/task/submit
|
|
1236
|
+
|
|
1237
|
+
Args:
|
|
1238
|
+
req(PandaRequest): internally generated request object
|
|
1239
|
+
task_parameters(dict): Dictionary with all the required task parameters. The parameters are the attributes in the JediTaskSpec object (https://github.com/PanDAWMS/panda-jedi/blob/master/pandajedi/jedicore/JediTaskSpec.py).
|
|
1240
|
+
parent_tid(int, optional): Parent task ID
|
|
1241
|
+
|
|
1242
|
+
Returns:
|
|
1243
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
|
|
1244
|
+
Return code in the data field, 0 for success, others for failure.
|
|
1245
|
+
"""
|
|
1246
|
+
tmp_log = LogWrapper(_logger, f"submit {naive_utcnow().isoformat('/')}")
|
|
1247
|
+
tmp_log.debug("Start")
|
|
1248
|
+
|
|
1249
|
+
user = get_dn(req)
|
|
1250
|
+
is_production_role = has_production_role(req)
|
|
1251
|
+
fqans = get_fqan(req)
|
|
1252
|
+
|
|
1253
|
+
tmp_log.debug(f"user={user} is_production_role={is_production_role} FQAN:{str(fqans)} parent_tid={parent_tid}")
|
|
1254
|
+
ret = global_task_buffer.insertTaskParamsPanda(task_parameters, user, is_production_role, fqans, properErrorCode=True, parent_tid=parent_tid, decode=False)
|
|
1255
|
+
|
|
1256
|
+
code, message = ret
|
|
1257
|
+
success = code in (0, 3)
|
|
1258
|
+
if not success:
|
|
1259
|
+
return generate_response(False, message=message, data=code)
|
|
1260
|
+
|
|
1261
|
+
# Extract the task ID from the message
|
|
1262
|
+
task_id = None
|
|
1263
|
+
match = re.search(r"jediTaskID=(\d+)", message)
|
|
1264
|
+
if match:
|
|
1265
|
+
try:
|
|
1266
|
+
task_id = int(match.group(1)) # Convert to an integer
|
|
1267
|
+
tmp_log.debug(f"Created task with task_id: {task_id}")
|
|
1268
|
+
except ValueError:
|
|
1269
|
+
tmp_log.error("Failed to extract the task ID from the message")
|
|
1270
|
+
task_id = None
|
|
1271
|
+
|
|
1272
|
+
tmp_log.debug("Done")
|
|
1273
|
+
|
|
1274
|
+
return generate_response(True, message=message, data=task_id)
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
@request_validation(_logger, request_method="GET")
|
|
1278
|
+
def get_task_parameters(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
1279
|
+
"""
|
|
1280
|
+
Get task parameters
|
|
1281
|
+
|
|
1282
|
+
Get a dictionary with the task parameters used to create a task.
|
|
1283
|
+
|
|
1284
|
+
API details:
|
|
1285
|
+
HTTP Method: GET
|
|
1286
|
+
Path: /v1/task/get_task_parameters
|
|
1287
|
+
|
|
1288
|
+
Args:
|
|
1289
|
+
req(PandaRequest): internally generated request object
|
|
1290
|
+
task_id(int): JEDI task ID
|
|
1291
|
+
|
|
1292
|
+
Returns:
|
|
1293
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
|
|
1294
|
+
|
|
1295
|
+
"""
|
|
1296
|
+
tmp_logger = LogWrapper(_logger, f"get_task_parameters < task_id={task_id} >")
|
|
1297
|
+
tmp_logger.debug("Start")
|
|
1298
|
+
|
|
1299
|
+
# validate the task id
|
|
1300
|
+
try:
|
|
1301
|
+
task_id = int(task_id)
|
|
1302
|
+
except Exception:
|
|
1303
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1304
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1305
|
+
|
|
1306
|
+
# get the parameters
|
|
1307
|
+
task_parameters_str = global_task_buffer.getTaskParamsMap(task_id)
|
|
1308
|
+
if not task_parameters_str:
|
|
1309
|
+
tmp_logger.error("Failed due to task not found")
|
|
1310
|
+
return generate_response(False, message="Task not found")
|
|
1311
|
+
|
|
1312
|
+
# decode the parameters
|
|
1313
|
+
try:
|
|
1314
|
+
task_parameters = json.loads(task_parameters_str)
|
|
1315
|
+
except json.JSONDecodeError as e:
|
|
1316
|
+
tmp_logger.error("Failed due to error decoding the task parameters")
|
|
1317
|
+
return generate_response(False, message=f"Error decoding the task parameters: {str(e)}")
|
|
1318
|
+
|
|
1319
|
+
tmp_logger.debug("Done")
|
|
1320
|
+
|
|
1321
|
+
return generate_response(True, data=task_parameters)
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1325
|
+
def get_detailed_info(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
1326
|
+
"""
|
|
1327
|
+
Get detailed task info
|
|
1328
|
+
|
|
1329
|
+
Get all task fields together with jobParamsTemplate and taskParams for a given task.
|
|
1330
|
+
The record is read without locking. Requires a secure connection.
|
|
1331
|
+
|
|
1332
|
+
API details:
|
|
1333
|
+
HTTP Method: GET
|
|
1334
|
+
Path: /v1/task/get_detailed_info
|
|
1335
|
+
|
|
1336
|
+
Args:
|
|
1337
|
+
req(PandaRequest): internally generated request object
|
|
1338
|
+
task_id(int): JEDI task ID
|
|
1339
|
+
|
|
1340
|
+
Returns:
|
|
1341
|
+
dict: The system response ``{"success": success, "message": message, "data": data}``.
|
|
1342
|
+
On success the ``data`` field contains a dictionary with all JediTaskSpec
|
|
1343
|
+
attributes plus additional information.
|
|
1344
|
+
On failure ``success`` is False and ``message`` contains the error description.
|
|
1345
|
+
"""
|
|
1346
|
+
tmp_logger = LogWrapper(_logger, f"get_detailed_info < task_id={task_id} >")
|
|
1347
|
+
tmp_logger.debug("Start")
|
|
1348
|
+
|
|
1349
|
+
try:
|
|
1350
|
+
task_id = int(task_id)
|
|
1351
|
+
except Exception:
|
|
1352
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1353
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1354
|
+
|
|
1355
|
+
task_info = global_task_buffer.get_task_details_json(task_id)
|
|
1356
|
+
if task_info is None:
|
|
1357
|
+
tmp_logger.error("Task not found or error retrieving task info")
|
|
1358
|
+
return generate_response(False, message="Task not found or error retrieving task info")
|
|
1359
|
+
|
|
1360
|
+
tmp_logger.debug("Done")
|
|
1361
|
+
return generate_response(True, data=task_info)
|
|
1362
|
+
|
|
1363
|
+
|
|
1364
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1365
|
+
def get_parent_detailed_info(req: PandaRequest, task_id: int) -> Dict[str, Any]:
|
|
1366
|
+
"""
|
|
1367
|
+
Get detailed parent task info
|
|
1368
|
+
|
|
1369
|
+
Resolve the parent task from a given child task and return only the parent
|
|
1370
|
+
detailed information. Requires a secure connection.
|
|
1371
|
+
|
|
1372
|
+
API details:
|
|
1373
|
+
HTTP Method: GET
|
|
1374
|
+
Path: /v1/task/get_parent_detailed_info
|
|
1375
|
+
|
|
1376
|
+
Args:
|
|
1377
|
+
req(PandaRequest): internally generated request object
|
|
1378
|
+
task_id(int): child JEDI task ID
|
|
1379
|
+
|
|
1380
|
+
Returns:
|
|
1381
|
+
dict: The system response ``{"success": success, "message": message, "data": data}``.
|
|
1382
|
+
On success the ``data`` field contains the parent task details.
|
|
1383
|
+
On failure ``success`` is False with a message distinguishing:
|
|
1384
|
+
child-not-found, no-parent, parent-not-found, or retrieval error.
|
|
1385
|
+
"""
|
|
1386
|
+
tmp_logger = LogWrapper(_logger, f"get_parent_detailed_info < child_task_id={task_id} >")
|
|
1387
|
+
tmp_logger.debug("Start")
|
|
1388
|
+
|
|
1389
|
+
try:
|
|
1390
|
+
task_id = int(task_id)
|
|
1391
|
+
except Exception:
|
|
1392
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1393
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1394
|
+
|
|
1395
|
+
# Resolve parent and retrieve details in one DB-helper call.
|
|
1396
|
+
resolve_status, parent_task_id, task_info = global_task_buffer.get_task_details_json(
|
|
1397
|
+
task_id,
|
|
1398
|
+
resolve_parent=True,
|
|
1399
|
+
include_resolve_status=True,
|
|
1400
|
+
)
|
|
1401
|
+
tmp_logger.debug(f"resolve_parent status={resolve_status} child={task_id} parent={parent_task_id}")
|
|
1402
|
+
|
|
1403
|
+
if resolve_status == "child_not_found":
|
|
1404
|
+
tmp_logger.error(f"Child task not found child={task_id} parent={parent_task_id}")
|
|
1405
|
+
return generate_response(False, message="Child task not found")
|
|
1406
|
+
if resolve_status == "no_parent":
|
|
1407
|
+
tmp_logger.error(f"No valid parent task child={task_id} parent={parent_task_id}")
|
|
1408
|
+
return generate_response(False, message="No parent task found")
|
|
1409
|
+
if resolve_status == "parent_not_found":
|
|
1410
|
+
tmp_logger.error(f"Parent task not found child={task_id} parent={parent_task_id}")
|
|
1411
|
+
return generate_response(False, message="Parent task not found")
|
|
1412
|
+
if resolve_status in ["target_not_found", "error"] or task_info is None:
|
|
1413
|
+
tmp_logger.error(f"Parent task retrieval failed status={resolve_status} child={task_id} parent={parent_task_id}")
|
|
1414
|
+
return generate_response(False, message="Parent task details not found or retrieval error")
|
|
1415
|
+
|
|
1416
|
+
tmp_logger.debug(f"Done child={task_id} parent={parent_task_id}")
|
|
1417
|
+
return generate_response(True, data=task_info)
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1421
|
+
def get_scout_job_descriptions(req: PandaRequest, task_id: int) -> Dict:
|
|
1422
|
+
"""
|
|
1423
|
+
Get scout job descriptions for a task.
|
|
1424
|
+
|
|
1425
|
+
Resolves all PanDA job IDs associated with a JEDI task, retrieves their job
|
|
1426
|
+
descriptions including archive lookup, and returns only scout jobs. Scout
|
|
1427
|
+
jobs are identified by the ``sj`` token in the comma-separated
|
|
1428
|
+
``specialHandling`` field.
|
|
1429
|
+
Requires a secure connection.
|
|
1430
|
+
|
|
1431
|
+
API details:
|
|
1432
|
+
HTTP Method: GET
|
|
1433
|
+
Path: /v1/task/get_scout_job_descriptions
|
|
1434
|
+
|
|
1435
|
+
Args:
|
|
1436
|
+
req(PandaRequest): internally generated request object containing the env variables.
|
|
1437
|
+
task_id (int): JEDI task ID.
|
|
1438
|
+
|
|
1439
|
+
Returns:
|
|
1440
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
1441
|
+
When successful, the data field contains a list of scout job descriptions.
|
|
1442
|
+
If no jobs are found or no scout jobs exist, returns success=True with empty list.
|
|
1443
|
+
"""
|
|
1444
|
+
tmp_logger = LogWrapper(_logger, f"get_scout_job_descriptions task_id={task_id}")
|
|
1445
|
+
tmp_logger.debug("Start")
|
|
1446
|
+
|
|
1447
|
+
try:
|
|
1448
|
+
task_id = int(task_id)
|
|
1449
|
+
except Exception:
|
|
1450
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1451
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1452
|
+
|
|
1453
|
+
job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id, scout_only=True)
|
|
1454
|
+
if not job_ids:
|
|
1455
|
+
tmp_logger.debug("No scout jobs found for task")
|
|
1456
|
+
return generate_response(True, data=[])
|
|
1457
|
+
|
|
1458
|
+
tmp_logger.debug(f"Found {len(job_ids)} scout job IDs for task")
|
|
1459
|
+
|
|
1460
|
+
max_ids = 5500
|
|
1461
|
+
if len(job_ids) > max_ids:
|
|
1462
|
+
tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
|
|
1463
|
+
job_ids = job_ids[:max_ids]
|
|
1464
|
+
|
|
1465
|
+
scout_jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
|
|
1466
|
+
|
|
1467
|
+
tmp_logger.debug(f"Done with {len(scout_jobs)} scout jobs")
|
|
1468
|
+
return generate_response(True, data=scout_jobs)
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
1472
|
+
def get_job_descriptions(req: PandaRequest, task_id: int, unsuccessful_only: bool = False) -> Dict:
|
|
1473
|
+
"""
|
|
1474
|
+
Get job descriptions for a task.
|
|
1475
|
+
|
|
1476
|
+
Resolves all PanDA job IDs associated with a JEDI task, retrieves their job
|
|
1477
|
+
descriptions including archive lookup, and returns them. Optionally filters
|
|
1478
|
+
to only unsuccessful jobs (status ``failed``, ``cancelled``, or ``closed``).
|
|
1479
|
+
Requires a secure connection.
|
|
1480
|
+
|
|
1481
|
+
API details:
|
|
1482
|
+
HTTP Method: GET
|
|
1483
|
+
Path: /v1/task/get_job_descriptions
|
|
1484
|
+
|
|
1485
|
+
Args:
|
|
1486
|
+
req(PandaRequest): internally generated request object containing the env variables.
|
|
1487
|
+
task_id (int): JEDI task ID.
|
|
1488
|
+
unsuccessful_only (bool): When True, return only failed, cancelled, or closed jobs.
|
|
1489
|
+
Defaults to False.
|
|
1490
|
+
|
|
1491
|
+
Returns:
|
|
1492
|
+
dict: The system response `{"success": success, "message": message, "data": data}`.
|
|
1493
|
+
When successful, the data field contains a list of job descriptions.
|
|
1494
|
+
If no jobs are found, returns success=True with empty list.
|
|
1495
|
+
"""
|
|
1496
|
+
tmp_logger = LogWrapper(_logger, f"get_job_descriptions task_id={task_id} unsuccessful_only={unsuccessful_only}")
|
|
1497
|
+
tmp_logger.debug("Start")
|
|
1498
|
+
|
|
1499
|
+
try:
|
|
1500
|
+
task_id = int(task_id)
|
|
1501
|
+
except Exception:
|
|
1502
|
+
tmp_logger.error("Failed due to invalid task_id")
|
|
1503
|
+
return generate_response(False, message=MESSAGE_TASK_ID)
|
|
1504
|
+
|
|
1505
|
+
job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id, unsuccessful_only=unsuccessful_only)
|
|
1506
|
+
if not job_ids:
|
|
1507
|
+
tmp_logger.debug("No jobs found for task")
|
|
1508
|
+
return generate_response(True, data=[])
|
|
1509
|
+
|
|
1510
|
+
tmp_logger.debug(f"Found {len(job_ids)} job IDs for task")
|
|
1511
|
+
|
|
1512
|
+
max_ids = 5500
|
|
1513
|
+
if len(job_ids) > max_ids:
|
|
1514
|
+
tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
|
|
1515
|
+
job_ids = job_ids[:max_ids]
|
|
1516
|
+
|
|
1517
|
+
jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
|
|
1518
|
+
|
|
1519
|
+
tmp_logger.debug(f"Done with {len(jobs)} jobs")
|
|
1520
|
+
return generate_response(True, data=jobs)
|