panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import socket
|
|
4
|
+
from threading import Lock
|
|
5
|
+
from typing import List
|
|
6
|
+
|
|
7
|
+
from pandacommon.pandalogger.LogWrapper import LogWrapper
|
|
8
|
+
from pandacommon.pandalogger.PandaLogger import PandaLogger
|
|
9
|
+
from pandacommon.pandautils.PandaUtils import naive_utcnow
|
|
10
|
+
|
|
11
|
+
from pandaserver.api.v1.common import generate_response, get_dn, request_validation
|
|
12
|
+
from pandaserver.config import panda_config
|
|
13
|
+
from pandaserver.proxycache import panda_proxy_cache, token_cache
|
|
14
|
+
from pandaserver.srvcore import CoreUtils
|
|
15
|
+
from pandaserver.srvcore.CoreUtils import clean_user_id, get_bare_dn
|
|
16
|
+
from pandaserver.srvcore.panda_request import PandaRequest
|
|
17
|
+
from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
|
|
18
|
+
|
|
19
|
+
_logger = PandaLogger().getLogger("api_cred_management")
|
|
20
|
+
|
|
21
|
+
# These global variables are initialized in the init_task_buffer method
|
|
22
|
+
global_task_buffer = None
|
|
23
|
+
global_dispatch_parameter_cache = None
|
|
24
|
+
|
|
25
|
+
# These global variables don't depend on DB access and can be initialized here
|
|
26
|
+
global_proxy_cache = panda_proxy_cache.MyProxyInterface()
|
|
27
|
+
global_token_cache = token_cache.TokenCache()
|
|
28
|
+
|
|
29
|
+
global_lock = Lock()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def init_task_buffer(task_buffer: TaskBuffer) -> None:
|
|
33
|
+
"""
|
|
34
|
+
Initialize the task buffer. This method needs to be called before any other method in this module.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
with global_lock:
|
|
38
|
+
global global_task_buffer
|
|
39
|
+
global_task_buffer = task_buffer
|
|
40
|
+
|
|
41
|
+
# This variable depends on having an initialized task buffer
|
|
42
|
+
global global_dispatch_parameter_cache
|
|
43
|
+
global_dispatch_parameter_cache = CoreUtils.CachedObject("dispatcher_params", 60 * 10, _get_dispatch_parameters, _logger)
|
|
44
|
+
|
|
45
|
+
global global_token_cache_config
|
|
46
|
+
global_token_cache_config = _read_token_cache_configuration()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _read_token_cache_configuration():
|
|
50
|
+
# config of token cacher
|
|
51
|
+
try:
|
|
52
|
+
with open(panda_config.token_cache_config) as f:
|
|
53
|
+
return json.load(f)
|
|
54
|
+
except Exception:
|
|
55
|
+
return {}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _get_dispatch_parameters():
|
|
59
|
+
"""
|
|
60
|
+
Wrapper function around taskBuffer.get_special_dispatch_params to convert list to set since task buffer cannot return set
|
|
61
|
+
"""
|
|
62
|
+
parameters = global_task_buffer.get_special_dispatch_params()
|
|
63
|
+
for client_name, client_data in parameters["tokenKeys"].items():
|
|
64
|
+
client_data["fullList"] = set(client_data["fullList"])
|
|
65
|
+
|
|
66
|
+
return True, parameters
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _validate_user_permissions(compact_name, tokenized=False) -> dict:
|
|
70
|
+
allowed_names = global_dispatch_parameter_cache.get("allowProxy", [])
|
|
71
|
+
|
|
72
|
+
# The user is allowed to get a proxy or token
|
|
73
|
+
if compact_name in allowed_names:
|
|
74
|
+
return True, ""
|
|
75
|
+
|
|
76
|
+
# Permission denied for user
|
|
77
|
+
secret = "proxy" if not tokenized else "access token"
|
|
78
|
+
tmp_msg = f"failed: '{compact_name}' not in authorized users with 'p' in {panda_config.schemaMETA}.USERS.GRIDPREF to get {secret}"
|
|
79
|
+
|
|
80
|
+
return False, tmp_msg
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@request_validation(_logger, secure=True, request_method="POST")
|
|
84
|
+
def set_user_secrets(req: PandaRequest, key: str = None, value: str = None) -> dict:
|
|
85
|
+
"""
|
|
86
|
+
Set user secrets
|
|
87
|
+
|
|
88
|
+
Set a key-value pair to store in PanDA. Requires a secure connection.
|
|
89
|
+
|
|
90
|
+
API details:
|
|
91
|
+
HTTP Method: POST
|
|
92
|
+
Path: /v1/creds/set_user_secrets
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
req(PandaRequest): internally generated request object
|
|
96
|
+
key(str): Key to reference the secret
|
|
97
|
+
value(str): Value of the secret
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
dict: The system response. True for success, False for failure, and an error message.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
tmp_logger = LogWrapper(_logger, f"set_user_secret-{naive_utcnow().isoformat('/')}")
|
|
104
|
+
|
|
105
|
+
# Get client DN and clean it. The request validation has already verified that the DN is present.
|
|
106
|
+
dn = get_dn(req)
|
|
107
|
+
owner = clean_user_id(dn)
|
|
108
|
+
tmp_logger.debug(f"Setting secret for client={owner}")
|
|
109
|
+
success, message = global_task_buffer.set_user_secret(owner, key, value)
|
|
110
|
+
return generate_response(success, message)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
114
|
+
def get_user_secrets(req: PandaRequest, keys: List[str] = None) -> dict:
|
|
115
|
+
"""
|
|
116
|
+
Get user secrets
|
|
117
|
+
|
|
118
|
+
Get the secrets for a user identified by a list of keys. Requires a secure connection.
|
|
119
|
+
|
|
120
|
+
API details:
|
|
121
|
+
HTTP Method: POST
|
|
122
|
+
Path: /v1/creds/get_user_secrets
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
req(PandaRequest): internally generated request object
|
|
126
|
+
keys(list of str, optional): List of keys to reference the secrets to retrieve
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the user secrets. When unsuccessful, the message field contains the error message.
|
|
130
|
+
"""
|
|
131
|
+
tmp_logger = LogWrapper(_logger, f"get_user_secrets-{naive_utcnow().isoformat('/')}")
|
|
132
|
+
|
|
133
|
+
# Get client DN and clean it. The request validation has already verified that the DN is present.
|
|
134
|
+
dn = get_dn(req)
|
|
135
|
+
owner = clean_user_id(dn)
|
|
136
|
+
tmp_logger.debug(f"Start for client={owner}")
|
|
137
|
+
|
|
138
|
+
# The task buffer method expects a comma-separated string of keys
|
|
139
|
+
if keys is None:
|
|
140
|
+
keys_str = ""
|
|
141
|
+
else:
|
|
142
|
+
keys_str = ",".join(keys)
|
|
143
|
+
|
|
144
|
+
success, data_or_message = global_task_buffer.get_user_secrets(owner, keys_str)
|
|
145
|
+
message, data = "", {}
|
|
146
|
+
if success:
|
|
147
|
+
data = data_or_message
|
|
148
|
+
else:
|
|
149
|
+
message = data_or_message
|
|
150
|
+
tmp_logger.debug(f"Done for client={owner}")
|
|
151
|
+
return generate_response(success, message, data)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
155
|
+
def get_key_pair(req: PandaRequest, public_key_name: str, private_key_name: str) -> dict:
|
|
156
|
+
"""
|
|
157
|
+
Get key pair
|
|
158
|
+
|
|
159
|
+
This function retrieves the distinguished name (DN) from the request and uses it to get a key pair. Requires a secure connection.
|
|
160
|
+
|
|
161
|
+
API details:
|
|
162
|
+
HTTP Method: GET
|
|
163
|
+
Path: /v1/creds/get_key_pair
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
req(PandaRequest): internally generated request object
|
|
167
|
+
public_key_name(str): The name of the public key.
|
|
168
|
+
private_key_name(str): The name of the private key.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the user secrets. When unsuccessful, the message field contains the error message.
|
|
172
|
+
"""
|
|
173
|
+
tmp_logger = LogWrapper(_logger, f"get_key_pair {public_key_name}/{private_key_name}")
|
|
174
|
+
tmp_logger.debug("Start")
|
|
175
|
+
|
|
176
|
+
real_dn = get_dn(req)
|
|
177
|
+
|
|
178
|
+
# get compact DN
|
|
179
|
+
compact_dn = clean_user_id(real_dn)
|
|
180
|
+
|
|
181
|
+
# check permission
|
|
182
|
+
global_dispatch_parameter_cache.update()
|
|
183
|
+
allowed_keys = global_dispatch_parameter_cache.get("allowKeyPair", [])
|
|
184
|
+
if compact_dn not in allowed_keys:
|
|
185
|
+
# permission denied
|
|
186
|
+
tmp_msg = f"Failed since '{compact_dn}' not authorized with 'k' in {panda_config.schemaMETA}.USERS.GRIDPREF"
|
|
187
|
+
tmp_logger.debug(tmp_msg)
|
|
188
|
+
return generate_response(False, tmp_msg)
|
|
189
|
+
|
|
190
|
+
# retrieve the key pairs
|
|
191
|
+
key_pair_dict = global_dispatch_parameter_cache.get("keyPair", {})
|
|
192
|
+
|
|
193
|
+
if public_key_name not in key_pair_dict: # public key is missing
|
|
194
|
+
tmp_msg = f"Failed for '{compact_dn}' since {public_key_name} is missing on {socket.getfqdn()}"
|
|
195
|
+
tmp_logger.debug(tmp_msg)
|
|
196
|
+
return generate_response(False, tmp_msg)
|
|
197
|
+
|
|
198
|
+
if private_key_name not in key_pair_dict: # private key is missing
|
|
199
|
+
tmp_msg = f"Failed for '{compact_dn}' since {private_key_name} is missing on {socket.getfqdn()}"
|
|
200
|
+
tmp_logger.debug(tmp_msg)
|
|
201
|
+
return generate_response(False, tmp_msg)
|
|
202
|
+
|
|
203
|
+
# key pair is available
|
|
204
|
+
data = {"public_key": key_pair_dict[public_key_name], "private_key": key_pair_dict[private_key_name]}
|
|
205
|
+
tmp_logger.debug(f"Done with key-pair for '{compact_dn}'")
|
|
206
|
+
|
|
207
|
+
return generate_response(True, data=data)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
211
|
+
def get_proxy(req: PandaRequest, role: str = None, dn: str = None) -> dict:
|
|
212
|
+
"""
|
|
213
|
+
Get proxy
|
|
214
|
+
|
|
215
|
+
Get the x509 proxy certificate for a user with a role. Requires a secure connection.
|
|
216
|
+
|
|
217
|
+
API details:
|
|
218
|
+
HTTP Method: GET
|
|
219
|
+
Path: /v1/creds/get_proxy
|
|
220
|
+
|
|
221
|
+
Args:
|
|
222
|
+
req(PandaRequest): internally generated request object
|
|
223
|
+
role(str, optional): The role of the user. Defaults to None.
|
|
224
|
+
dn(str, optional): The distinguished name of the user. Defaults to None.
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the x509 proxy. When unsuccessful, the message field contains the error message.
|
|
228
|
+
"""
|
|
229
|
+
tmp_logger = LogWrapper(_logger, f"get_proxy PID={os.getpid()}")
|
|
230
|
+
|
|
231
|
+
# Set the requested DN or, if not set, the requestor DN
|
|
232
|
+
real_dn = get_dn(req)
|
|
233
|
+
target_dn = dn if dn else real_dn
|
|
234
|
+
|
|
235
|
+
# Default roles
|
|
236
|
+
if role == "":
|
|
237
|
+
role = None
|
|
238
|
+
|
|
239
|
+
tmp_logger.debug(f"Start for DN='{real_dn}' role={role}")
|
|
240
|
+
|
|
241
|
+
# check permission
|
|
242
|
+
global_dispatch_parameter_cache.update()
|
|
243
|
+
real_dn_compact = clean_user_id(real_dn)
|
|
244
|
+
allowed, tmp_msg = _validate_user_permissions(real_dn_compact, tokenized=False)
|
|
245
|
+
if not allowed:
|
|
246
|
+
tmp_logger.debug(tmp_msg)
|
|
247
|
+
return generate_response(False, tmp_msg)
|
|
248
|
+
|
|
249
|
+
# remove redundant extensions and retrieve the proxy
|
|
250
|
+
target_dn_bare = get_bare_dn(target_dn, keep_digits=False)
|
|
251
|
+
output = global_proxy_cache.retrieve(target_dn_bare, role=role)
|
|
252
|
+
|
|
253
|
+
# proxy not found
|
|
254
|
+
if output is None:
|
|
255
|
+
tmp_msg = f"'proxy' not found for {target_dn_bare}"
|
|
256
|
+
tmp_logger.debug(tmp_msg)
|
|
257
|
+
return generate_response(False, tmp_msg)
|
|
258
|
+
|
|
259
|
+
data = {"user_proxy": output}
|
|
260
|
+
tmp_logger.debug(f"Done")
|
|
261
|
+
return generate_response(True, data=data)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
265
|
+
def get_access_token(req: PandaRequest, client_name: str, token_key: str = None) -> dict:
|
|
266
|
+
"""
|
|
267
|
+
Get access token
|
|
268
|
+
|
|
269
|
+
Get the OAuth access token for the specified client. Requires a secure connection.
|
|
270
|
+
|
|
271
|
+
API details:
|
|
272
|
+
HTTP Method: GET
|
|
273
|
+
Path: /v1/creds/get_access_token
|
|
274
|
+
|
|
275
|
+
Args:
|
|
276
|
+
req(PandaRequest): internally generated request object
|
|
277
|
+
client_name(str): client_name for the token as defined in token_cache_config
|
|
278
|
+
token_key(str, optional): key to get the token from the token cache. Defaults to None.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the access token. When unsuccessful, the message field contains the error message.
|
|
282
|
+
"""
|
|
283
|
+
|
|
284
|
+
tmp_logger = LogWrapper(_logger, f"get_proxy PID={os.getpid()}")
|
|
285
|
+
|
|
286
|
+
# Set the requested DN or, if not set, the requestor DN
|
|
287
|
+
real_dn = get_dn(req)
|
|
288
|
+
target_dn = client_name if client_name else real_dn
|
|
289
|
+
|
|
290
|
+
tmp_logger.debug(f"Start for DN='{real_dn}' target_dn='{target_dn}' token_key={token_key}")
|
|
291
|
+
|
|
292
|
+
# check permission
|
|
293
|
+
global_dispatch_parameter_cache.update()
|
|
294
|
+
real_dn_compact = clean_user_id(real_dn)
|
|
295
|
+
allowed, tmp_msg = _validate_user_permissions(real_dn_compact, tokenized=True)
|
|
296
|
+
if not allowed:
|
|
297
|
+
tmp_logger.debug(tmp_msg)
|
|
298
|
+
return generate_response(False, tmp_msg)
|
|
299
|
+
|
|
300
|
+
# check if the token key is valid
|
|
301
|
+
if target_dn in global_token_cache_config and global_token_cache_config[target_dn].get("use_token_key") is True:
|
|
302
|
+
if (
|
|
303
|
+
target_dn not in global_dispatch_parameter_cache["tokenKeys"]
|
|
304
|
+
or token_key not in global_dispatch_parameter_cache["tokenKeys"][target_dn]["fullList"]
|
|
305
|
+
):
|
|
306
|
+
tmp_msg = f"failed since token key is invalid for {target_dn}"
|
|
307
|
+
tmp_logger.debug(tmp_msg)
|
|
308
|
+
return generate_response(False, tmp_msg)
|
|
309
|
+
|
|
310
|
+
# remove redundant extensions
|
|
311
|
+
target_dn_bare = get_bare_dn(target_dn, keep_digits=False)
|
|
312
|
+
|
|
313
|
+
# get token
|
|
314
|
+
output = global_token_cache.get_access_token(target_dn_bare)
|
|
315
|
+
|
|
316
|
+
# access token not found
|
|
317
|
+
if output is None:
|
|
318
|
+
tmp_msg = f"'token' not found for {target_dn_bare}"
|
|
319
|
+
tmp_logger.debug(tmp_msg)
|
|
320
|
+
return generate_response(False, tmp_msg)
|
|
321
|
+
|
|
322
|
+
data = {"user_proxy": output}
|
|
323
|
+
tmp_logger.debug("Done")
|
|
324
|
+
return generate_response(True, data=data)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@request_validation(_logger, secure=True, request_method="GET")
|
|
328
|
+
def get_token_key(req: PandaRequest, client_name: str) -> dict:
|
|
329
|
+
"""
|
|
330
|
+
Get token key
|
|
331
|
+
|
|
332
|
+
This function retrieves the distinguished name (DN) from the request and uses it to get a token key for the specified client. Requires a secure connection.
|
|
333
|
+
|
|
334
|
+
API details:
|
|
335
|
+
HTTP Method: GET
|
|
336
|
+
Path: /v1/creds/get_token_key
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
req(PandaRequest): internally generated request object
|
|
340
|
+
client_name (str): The name of the client requesting the token key
|
|
341
|
+
|
|
342
|
+
Returns:
|
|
343
|
+
dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the token key. When unsuccessful, the message field contains the error message.
|
|
344
|
+
"""
|
|
345
|
+
tmp_logger = LogWrapper(_logger, f"get_token_key client={client_name} PID={os.getpid()}")
|
|
346
|
+
tmp_logger.debug("Start")
|
|
347
|
+
|
|
348
|
+
# get DN and clean it
|
|
349
|
+
real_dn = get_dn(req)
|
|
350
|
+
compact_name = clean_user_id(real_dn)
|
|
351
|
+
|
|
352
|
+
global_dispatch_parameter_cache.update()
|
|
353
|
+
|
|
354
|
+
# check if user is allowed to retrieved token keys
|
|
355
|
+
allowed_users = global_dispatch_parameter_cache.get("allowTokenKey", [])
|
|
356
|
+
if compact_name not in allowed_users:
|
|
357
|
+
# permission denied
|
|
358
|
+
tmp_msg = f"Denied since '{compact_name}' not authorized with 't' in {panda_config.schemaMETA}.USERS.GRIDPREF"
|
|
359
|
+
tmp_logger.debug(tmp_msg)
|
|
360
|
+
return generate_response(False, tmp_msg)
|
|
361
|
+
|
|
362
|
+
# token key is missing
|
|
363
|
+
if client_name not in global_dispatch_parameter_cache["tokenKeys"]:
|
|
364
|
+
tmp_msg = f"Token key is missing for '{client_name}'"
|
|
365
|
+
tmp_logger.debug(tmp_msg)
|
|
366
|
+
return generate_response(False, tmp_msg)
|
|
367
|
+
|
|
368
|
+
# send token key
|
|
369
|
+
data = {"tokenKey": global_dispatch_parameter_cache["tokenKeys"][client_name]["latest"]}
|
|
370
|
+
tmp_logger.debug(f"Done getting token key for '{compact_name}'")
|
|
371
|
+
|
|
372
|
+
return generate_response(True, data=data)
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
3
|
+
from threading import Lock
|
|
4
|
+
from typing import Any, Dict, List
|
|
5
|
+
|
|
6
|
+
from pandacommon.pandalogger.LogWrapper import LogWrapper
|
|
7
|
+
from pandacommon.pandalogger.PandaLogger import PandaLogger
|
|
8
|
+
from pandacommon.pandautils.PandaUtils import naive_utcnow
|
|
9
|
+
|
|
10
|
+
from pandaserver.api.v1.common import (
|
|
11
|
+
MESSAGE_DATABASE,
|
|
12
|
+
TIME_OUT,
|
|
13
|
+
TimedMethod,
|
|
14
|
+
generate_response,
|
|
15
|
+
get_dn,
|
|
16
|
+
request_validation,
|
|
17
|
+
)
|
|
18
|
+
from pandaserver.srvcore.panda_request import PandaRequest
|
|
19
|
+
from pandaserver.taskbuffer.DataCarousel import (
|
|
20
|
+
DataCarouselInterface,
|
|
21
|
+
DataCarouselRequestStatus,
|
|
22
|
+
)
|
|
23
|
+
from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
|
|
24
|
+
|
|
25
|
+
_logger = PandaLogger().getLogger("api_data_carousel")
|
|
26
|
+
|
|
27
|
+
# These global variables are initialized in the init_task_buffer method
|
|
28
|
+
global_task_buffer = None
|
|
29
|
+
global_dcif = None
|
|
30
|
+
|
|
31
|
+
# These global variables don't depend on DB access and can be initialized here
|
|
32
|
+
# global_proxy_cache = panda_proxy_cache.MyProxyInterface()
|
|
33
|
+
# global_token_cache = token_cache.TokenCache()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def init_task_buffer(task_buffer: TaskBuffer) -> None:
|
|
37
|
+
"""
|
|
38
|
+
Initialize the task buffer and other interfaces. This method needs to be called before any other method in this module.
|
|
39
|
+
"""
|
|
40
|
+
global global_task_buffer
|
|
41
|
+
global_task_buffer = task_buffer
|
|
42
|
+
|
|
43
|
+
global global_dcif
|
|
44
|
+
global_dcif = DataCarouselInterface(global_task_buffer)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
48
|
+
def change_staging_destination(req: PandaRequest, request_id: int | None = None, dataset: str | None = None) -> dict:
|
|
49
|
+
"""
|
|
50
|
+
Change destination of staging
|
|
51
|
+
|
|
52
|
+
The current active staging request will be cancelled, and a new request will be created with the newly selected destination RSE, excluding the original destination.
|
|
53
|
+
The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
|
|
54
|
+
Requires a secure connection production role.
|
|
55
|
+
|
|
56
|
+
API details:
|
|
57
|
+
HTTP Method: POST
|
|
58
|
+
Path: /v1/data_carousel/change_staging_destination
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
req(PandaRequest): internally generated request object
|
|
62
|
+
request_id (int|None): request_id of the staging request, e.g. `123`
|
|
63
|
+
dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
|
|
67
|
+
"""
|
|
68
|
+
tmp_logger = LogWrapper(_logger, f"change_staging_destination request_id={request_id} dataset={dataset}")
|
|
69
|
+
tmp_logger.debug("Start")
|
|
70
|
+
success, message, data = False, "", None
|
|
71
|
+
dc_req_spec_resubmitted = None
|
|
72
|
+
to_submit_idds = False
|
|
73
|
+
time_start = naive_utcnow()
|
|
74
|
+
|
|
75
|
+
dc_req_spec = None
|
|
76
|
+
if request_id is not None:
|
|
77
|
+
# specified by request_id
|
|
78
|
+
dc_req_spec = global_dcif.get_request_by_id(request_id)
|
|
79
|
+
elif request_id is None and dataset is not None:
|
|
80
|
+
# specified by dataset
|
|
81
|
+
dc_req_spec = global_dcif.get_request_by_dataset(dataset)
|
|
82
|
+
|
|
83
|
+
if dc_req_spec is not None:
|
|
84
|
+
dc_req_spec_resubmitted, err_msg = global_dcif.resubmit_request(dc_req_spec, submit_idds_request=False, exclude_prev_dst=True)
|
|
85
|
+
if not dc_req_spec_resubmitted or err_msg:
|
|
86
|
+
err_msg = f"failed to resubmit request_id={dc_req_spec.request_id} : {err_msg}"
|
|
87
|
+
tmp_logger.error(err_msg)
|
|
88
|
+
success, message = False, err_msg
|
|
89
|
+
else:
|
|
90
|
+
to_submit_idds = True
|
|
91
|
+
else:
|
|
92
|
+
err_msg = f"failed to get corresponding request"
|
|
93
|
+
tmp_logger.error(err_msg)
|
|
94
|
+
success, message = False, err_msg
|
|
95
|
+
|
|
96
|
+
if dc_req_spec_resubmitted and dc_req_spec_resubmitted.status == DataCarouselRequestStatus.staging:
|
|
97
|
+
success = True
|
|
98
|
+
data = {"request_id": dc_req_spec.request_id, "new_request_id": dc_req_spec_resubmitted.request_id, "dataset": dc_req_spec_resubmitted.dataset}
|
|
99
|
+
message = "new request resubmitted, destination changed"
|
|
100
|
+
if to_submit_idds:
|
|
101
|
+
new_request_id = dc_req_spec_resubmitted.request_id
|
|
102
|
+
task_id_list = global_dcif._get_related_tasks(new_request_id)
|
|
103
|
+
if task_id_list:
|
|
104
|
+
tmp_logger.debug(f"related tasks: {task_id_list}")
|
|
105
|
+
with ThreadPoolExecutor() as thread_pool:
|
|
106
|
+
thread_pool.map((lambda task_id: global_dcif._submit_idds_stagein_request(task_id, dc_req_spec_resubmitted)), task_id_list)
|
|
107
|
+
tmp_logger.debug(f"submitted corresponding iDDS requests for related tasks")
|
|
108
|
+
message += "; submitted iDDS requests"
|
|
109
|
+
|
|
110
|
+
else:
|
|
111
|
+
err_msg = f"failed to get related tasks; skipped to submit iDDS requests"
|
|
112
|
+
tmp_logger.warning(err_msg)
|
|
113
|
+
message += f"; {err_msg}"
|
|
114
|
+
|
|
115
|
+
time_delta = naive_utcnow() - time_start
|
|
116
|
+
tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
|
|
117
|
+
|
|
118
|
+
return generate_response(success, message, data)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
122
|
+
def change_staging_source(
|
|
123
|
+
req: PandaRequest,
|
|
124
|
+
request_id: int | None = None,
|
|
125
|
+
dataset: str | None = None,
|
|
126
|
+
cancel_fts: bool = False,
|
|
127
|
+
change_src_expr: bool = False,
|
|
128
|
+
source_rse: str | None = None,
|
|
129
|
+
) -> dict:
|
|
130
|
+
"""
|
|
131
|
+
Change source of staging
|
|
132
|
+
|
|
133
|
+
If the request is queued, its source_rse will be rechosen, excluding the original source.
|
|
134
|
+
If the request is staging, the source_replica_expression of its DDM rule is unset so new source can be tried.
|
|
135
|
+
Only effective on queued or staging requests.
|
|
136
|
+
The requests can be specified by request_id or dataset (if both exist, request_id is taken).
|
|
137
|
+
Requires a secure connection production role.
|
|
138
|
+
|
|
139
|
+
API details:
|
|
140
|
+
HTTP Method: POST
|
|
141
|
+
Path: /v1/data_carousel/change_staging_source
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
req(PandaRequest): internally generated request object
|
|
145
|
+
request_id (int|None): request_id of the staging request, e.g. `123`
|
|
146
|
+
dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
|
|
147
|
+
cancel_fts (bool): whether to cancel current FTS requests on DDM, False by default
|
|
148
|
+
change_src_expr (bool): whether to change source_replica_expression of the DDM rule by replacing old source with new one, instead of just dropping old source
|
|
149
|
+
source_rse (str|None): if set, use this source RSE instead of choosing one randomly, also force change_src_expr to be True; default is None
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
|
|
153
|
+
"""
|
|
154
|
+
tmp_logger = LogWrapper(
|
|
155
|
+
_logger,
|
|
156
|
+
f"change_staging_source request_id={request_id} dataset={dataset} cancel_fts={cancel_fts} change_src_expr={change_src_expr} source_rse={source_rse}",
|
|
157
|
+
)
|
|
158
|
+
tmp_logger.debug("Start")
|
|
159
|
+
success, message, data = False, "", None
|
|
160
|
+
time_start = naive_utcnow()
|
|
161
|
+
|
|
162
|
+
dc_req_spec = None
|
|
163
|
+
if request_id is not None:
|
|
164
|
+
# specified by request_id
|
|
165
|
+
dc_req_spec = global_dcif.get_request_by_id(request_id)
|
|
166
|
+
elif request_id is None and dataset is not None:
|
|
167
|
+
# specified by dataset
|
|
168
|
+
dc_req_spec = global_dcif.get_request_by_dataset(dataset)
|
|
169
|
+
|
|
170
|
+
if dc_req_spec is not None:
|
|
171
|
+
status = dc_req_spec.status
|
|
172
|
+
orig_source_rse = dc_req_spec.source_rse
|
|
173
|
+
if status not in [DataCarouselRequestStatus.queued, DataCarouselRequestStatus.staging]:
|
|
174
|
+
err_msg = f"request_id={dc_req_spec.request_id} status={status} not queued or staging; skipped"
|
|
175
|
+
tmp_logger.warning(err_msg)
|
|
176
|
+
success, message = False, err_msg
|
|
177
|
+
else:
|
|
178
|
+
ret, dc_req_spec, err_msg = global_dcif.change_request_source_rse(dc_req_spec, cancel_fts, change_src_expr, source_rse)
|
|
179
|
+
if not ret:
|
|
180
|
+
err_msg = f"failed to change source request_id={dc_req_spec.request_id} : {err_msg}"
|
|
181
|
+
tmp_logger.error(err_msg)
|
|
182
|
+
success, message = False, err_msg
|
|
183
|
+
else:
|
|
184
|
+
success = True
|
|
185
|
+
if dc_req_spec.status == DataCarouselRequestStatus.queued or change_src_expr:
|
|
186
|
+
message = f"status={status} changed source_rse from {orig_source_rse} to {dc_req_spec.source_rse}"
|
|
187
|
+
else:
|
|
188
|
+
message = f"status={status} source replica expression is dropped"
|
|
189
|
+
data = {
|
|
190
|
+
"request_id": dc_req_spec.request_id,
|
|
191
|
+
"dataset": dc_req_spec.dataset,
|
|
192
|
+
"source_rse": dc_req_spec.source_rse,
|
|
193
|
+
"ddm_rule_id": dc_req_spec.ddm_rule_id,
|
|
194
|
+
}
|
|
195
|
+
else:
|
|
196
|
+
err_msg = f"failed to get corresponding request"
|
|
197
|
+
tmp_logger.error(err_msg)
|
|
198
|
+
success, message = False, err_msg
|
|
199
|
+
|
|
200
|
+
time_delta = naive_utcnow() - time_start
|
|
201
|
+
tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
|
|
202
|
+
|
|
203
|
+
return generate_response(success, message, data)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@request_validation(_logger, secure=True, production=True, request_method="POST")
|
|
207
|
+
def force_to_staging(req: PandaRequest, request_id: int | None = None, dataset: str | None = None) -> dict:
|
|
208
|
+
"""
|
|
209
|
+
Force to staging
|
|
210
|
+
|
|
211
|
+
The request will skip the queue and go to staging immediately (will submit DDM rules).
|
|
212
|
+
Only effective on queued requests.
|
|
213
|
+
The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
|
|
214
|
+
Requires a secure connection production role.
|
|
215
|
+
|
|
216
|
+
API details:
|
|
217
|
+
HTTP Method: POST
|
|
218
|
+
Path: /v1/data_carousel/force_to_staging
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
req(PandaRequest): internally generated request object
|
|
222
|
+
request_id (int|None): request_id of the staging request, e.g. `123`
|
|
223
|
+
dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
|
|
227
|
+
"""
|
|
228
|
+
tmp_logger = LogWrapper(_logger, f"force_to_staging request_id={request_id} dataset={dataset}")
|
|
229
|
+
tmp_logger.debug("Start")
|
|
230
|
+
success, message, data = False, "", None
|
|
231
|
+
time_start = naive_utcnow()
|
|
232
|
+
|
|
233
|
+
dc_req_spec = None
|
|
234
|
+
if request_id is not None:
|
|
235
|
+
# specified by request_id
|
|
236
|
+
dc_req_spec = global_dcif.get_request_by_id(request_id)
|
|
237
|
+
elif request_id is None and dataset is not None:
|
|
238
|
+
# specified by dataset
|
|
239
|
+
dc_req_spec = global_dcif.get_request_by_dataset(dataset)
|
|
240
|
+
|
|
241
|
+
if dc_req_spec is not None:
|
|
242
|
+
is_ok, err_msg, dc_req_spec = global_dcif.stage_request(dc_req_spec)
|
|
243
|
+
if not is_ok:
|
|
244
|
+
err_msg = f"failed to stage request_id={dc_req_spec.request_id} : {err_msg}"
|
|
245
|
+
tmp_logger.error(err_msg)
|
|
246
|
+
success, message = False, err_msg
|
|
247
|
+
else:
|
|
248
|
+
success = True
|
|
249
|
+
message = f"status has become {dc_req_spec.status}"
|
|
250
|
+
data = {
|
|
251
|
+
"request_id": dc_req_spec.request_id,
|
|
252
|
+
"dataset": dc_req_spec.dataset,
|
|
253
|
+
"status": dc_req_spec.status,
|
|
254
|
+
"ddm_rule_id": dc_req_spec.ddm_rule_id,
|
|
255
|
+
}
|
|
256
|
+
else:
|
|
257
|
+
err_msg = f"failed to get corresponding request"
|
|
258
|
+
tmp_logger.error(err_msg)
|
|
259
|
+
success, message = False, err_msg
|
|
260
|
+
|
|
261
|
+
time_delta = naive_utcnow() - time_start
|
|
262
|
+
tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
|
|
263
|
+
|
|
264
|
+
return generate_response(success, message, data)
|