panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
# split rules
|
|
2
|
+
split_rule_dict = {
|
|
3
|
+
"allowEmptyInput": "AE",
|
|
4
|
+
"allowIncompleteInDS": "AI",
|
|
5
|
+
"addNthFieldToLFN": "AN",
|
|
6
|
+
"allowPartialFinish": "AP",
|
|
7
|
+
"altStageOut": "AT",
|
|
8
|
+
"avoidVP": "AV",
|
|
9
|
+
"maxCoreCount": "CC",
|
|
10
|
+
"cloudAsVO": "CV",
|
|
11
|
+
"ddmBackEnd": "DE",
|
|
12
|
+
"disableAutoFinish": "DF",
|
|
13
|
+
"disableReassign": "DI",
|
|
14
|
+
"debugMode": "DM",
|
|
15
|
+
"disableAutoRetry": "DR",
|
|
16
|
+
"dynamicNumEvents": "DY",
|
|
17
|
+
"nEsConsumers": "EC",
|
|
18
|
+
"nEventsPerInput": "EI",
|
|
19
|
+
"encJobParams": "EJ",
|
|
20
|
+
"nEventsPerWorker": "ES",
|
|
21
|
+
"firstContentsFeed": "FC",
|
|
22
|
+
"failGoalUnreached": "FG",
|
|
23
|
+
"fineGrainedProc": "FP",
|
|
24
|
+
"firstEvent": "FT",
|
|
25
|
+
"fullChain": "FU",
|
|
26
|
+
"groupBoundaryID": "GB",
|
|
27
|
+
"hpoWorkflow": "HO",
|
|
28
|
+
"instantiateTmplSite": "IA",
|
|
29
|
+
"inFilePosEvtNum": "IF",
|
|
30
|
+
"ipStack": "IK",
|
|
31
|
+
"allowInputLAN": "IL",
|
|
32
|
+
"ignoreMissingInDS": "IM",
|
|
33
|
+
"intermediateTask": "IN",
|
|
34
|
+
"ipConnectivity": "IP",
|
|
35
|
+
"inputPreStaging": "IS",
|
|
36
|
+
"instantiateTmpl": "IT",
|
|
37
|
+
"allowInputWAN": "IW",
|
|
38
|
+
"noLoopingCheck": "LC",
|
|
39
|
+
"useLocalIO": "LI",
|
|
40
|
+
"limitedSites": "LS",
|
|
41
|
+
"loadXML": "LX",
|
|
42
|
+
"minCpuEfficiency": "MC",
|
|
43
|
+
"messageDriven": "MD",
|
|
44
|
+
"mergeEsOnOS": "ME",
|
|
45
|
+
"nMaxFilesPerJob": "MF",
|
|
46
|
+
"maxJumboPerSite": "MJ",
|
|
47
|
+
"maxNumJobs": "MN",
|
|
48
|
+
"mergeOutput": "MO",
|
|
49
|
+
"multiStepExec": "MS",
|
|
50
|
+
"maxWalltime": "MW",
|
|
51
|
+
"maxEventsPerJob": "MX",
|
|
52
|
+
"noExecStrCnv": "NC",
|
|
53
|
+
"notDiscardEvents": "ND",
|
|
54
|
+
"nEventsPerJob": "NE",
|
|
55
|
+
"nFilesPerJob": "NF",
|
|
56
|
+
"nGBPerJob": "NG",
|
|
57
|
+
"noInputPooling": "NI",
|
|
58
|
+
"nJumboJobs": "NJ",
|
|
59
|
+
"noAutoPause": "NP",
|
|
60
|
+
"nSitesPerJob": "NS",
|
|
61
|
+
"nChunksToWait": "NT",
|
|
62
|
+
"noWaitParent": "NW",
|
|
63
|
+
"orderInputBy": "OI",
|
|
64
|
+
"orderByLB": "OL",
|
|
65
|
+
"onSiteMerging": "OM",
|
|
66
|
+
"osMatching": "OS",
|
|
67
|
+
"onlyTagsForFC": "OT",
|
|
68
|
+
"pushStatusChanges": "PC",
|
|
69
|
+
"pushJob": "PJ",
|
|
70
|
+
"pfnList": "PL",
|
|
71
|
+
"putLogToOS": "PO",
|
|
72
|
+
"runUntilClosed": "RC",
|
|
73
|
+
"registerDatasets": "RD",
|
|
74
|
+
"registerEsFiles": "RE",
|
|
75
|
+
"respectLB": "RL",
|
|
76
|
+
"retryModuleRules": "RM",
|
|
77
|
+
"reuseSecOnDemand": "RO",
|
|
78
|
+
"releasePerLB": "RP",
|
|
79
|
+
"respectSplitRule": "RR",
|
|
80
|
+
"randomSeed": "RS",
|
|
81
|
+
"retryRamOffset": "RX",
|
|
82
|
+
"retryRamStep": "RY",
|
|
83
|
+
"retryRamMax": "RZ",
|
|
84
|
+
"resurrectConsumers": "SC",
|
|
85
|
+
"switchEStoNormal": "SE",
|
|
86
|
+
"stayOutputOnSite": "SO",
|
|
87
|
+
"scoutSuccessRate": "SS",
|
|
88
|
+
"useSecrets": "ST",
|
|
89
|
+
"segmentedWork": "SW",
|
|
90
|
+
"totNumJobs": "TJ",
|
|
91
|
+
"tgtMaxOutputForNG": "TN",
|
|
92
|
+
"t1Weight": "TW",
|
|
93
|
+
"useBuild": "UB",
|
|
94
|
+
"useJobCloning": "UC",
|
|
95
|
+
"useRealNumEvents": "UE",
|
|
96
|
+
"useFileAsSourceLFN": "UF",
|
|
97
|
+
"usePrePro": "UP",
|
|
98
|
+
"useScout": "US",
|
|
99
|
+
"usePrefetcher": "UT",
|
|
100
|
+
"useExhausted": "UX",
|
|
101
|
+
"useZipToPin": "UZ",
|
|
102
|
+
"writeInputToFile": "WF",
|
|
103
|
+
"workflowHoldup": "WH",
|
|
104
|
+
"waitInput": "WI",
|
|
105
|
+
"maxAttemptES": "XA",
|
|
106
|
+
"decAttOnFailedES": "XF",
|
|
107
|
+
"maxAttemptEsJob": "XJ",
|
|
108
|
+
"nEventsPerMergeJob": "ZE",
|
|
109
|
+
"nFilesPerMergeJob": "ZF",
|
|
110
|
+
"nGBPerMergeJob": "ZG",
|
|
111
|
+
"nMaxFilesPerMergeJob": "ZM",
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
# changeable split rules
|
|
115
|
+
changeable_split_rule_names = [
|
|
116
|
+
"allowIncompleteInDS",
|
|
117
|
+
"t1Weight",
|
|
118
|
+
"nEsConsumers",
|
|
119
|
+
"nMaxFilesPerJob",
|
|
120
|
+
"nGBPerJob",
|
|
121
|
+
"noInputPooling",
|
|
122
|
+
"nFilesPerJob",
|
|
123
|
+
"nEventsPerWorker",
|
|
124
|
+
"nJumboJobs",
|
|
125
|
+
"avoidVP",
|
|
126
|
+
"allowInputLAN",
|
|
127
|
+
"useLocalIO",
|
|
128
|
+
"noLoopingCheck",
|
|
129
|
+
"maxCoreCount",
|
|
130
|
+
"onlyTagsForFC",
|
|
131
|
+
"useZipToPin",
|
|
132
|
+
"ignoreMissingInDS",
|
|
133
|
+
"noAutoPause",
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
changeable_split_rule_tags = [split_rule_dict[tmp_name] for tmp_name in changeable_split_rule_names]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# extract rules
|
|
140
|
+
def extract_rule_values(split_rules: str, rule_names: list, is_sub_rule: bool = False) -> dict:
|
|
141
|
+
"""
|
|
142
|
+
Extract rule values from split rule string
|
|
143
|
+
|
|
144
|
+
:param split_rules: a comma separated string
|
|
145
|
+
:param rule_names: a list of rule names
|
|
146
|
+
:param is_sub_rule: True to indicate the rule is a subrule
|
|
147
|
+
:return: dict of rule names and values
|
|
148
|
+
"""
|
|
149
|
+
if split_rules is None:
|
|
150
|
+
split_rules = ""
|
|
151
|
+
ret = {}
|
|
152
|
+
if is_sub_rule:
|
|
153
|
+
rule_separator = "|"
|
|
154
|
+
key_value_separator = ":"
|
|
155
|
+
else:
|
|
156
|
+
rule_separator = ","
|
|
157
|
+
key_value_separator = "="
|
|
158
|
+
for tmp_rule in split_rules.split(rule_separator):
|
|
159
|
+
for tmp_name in rule_names:
|
|
160
|
+
if tmp_name in split_rule_dict and tmp_rule.startswith(split_rule_dict[tmp_name] + key_value_separator):
|
|
161
|
+
ret[tmp_name] = tmp_rule.split(key_value_separator)[-1]
|
|
162
|
+
break
|
|
163
|
+
for tmp_name in rule_names:
|
|
164
|
+
if tmp_name not in ret:
|
|
165
|
+
ret[tmp_name] = None
|
|
166
|
+
return ret
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# replace a rule
|
|
170
|
+
def replace_rule(split_rules: str, rule_name: str, rule_value: int | str, is_sub_rule=False):
|
|
171
|
+
"""
|
|
172
|
+
Replace a rule in the split rule string
|
|
173
|
+
|
|
174
|
+
:param split_rules: s comma separated string
|
|
175
|
+
:param rule_name: rule name
|
|
176
|
+
:param rule_value: rule value
|
|
177
|
+
:param is_sub_rule: True to indicate the rule is a subrule
|
|
178
|
+
:return: string of split rules
|
|
179
|
+
"""
|
|
180
|
+
if rule_name not in split_rule_dict:
|
|
181
|
+
return split_rules
|
|
182
|
+
if split_rules is None:
|
|
183
|
+
split_rules = ""
|
|
184
|
+
if is_sub_rule:
|
|
185
|
+
rule_separator = "|"
|
|
186
|
+
key_value_separator = ":"
|
|
187
|
+
else:
|
|
188
|
+
rule_separator = ","
|
|
189
|
+
key_value_separator = "="
|
|
190
|
+
tmp_str = ""
|
|
191
|
+
for tmp_rule in split_rules.split(rule_separator):
|
|
192
|
+
if tmp_rule.startswith(split_rule_dict[rule_name] + key_value_separator):
|
|
193
|
+
continue
|
|
194
|
+
if tmp_str != "":
|
|
195
|
+
tmp_str += rule_separator
|
|
196
|
+
tmp_str += tmp_rule
|
|
197
|
+
if tmp_str != "":
|
|
198
|
+
tmp_str += rule_separator
|
|
199
|
+
tmp_str += split_rule_dict[rule_name] + key_value_separator + str(rule_value)
|
|
200
|
+
return tmp_str
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
# remove a rule
|
|
204
|
+
def remove_rule(split_rules: str, rule_token: str, is_sub_rule: bool = False):
|
|
205
|
+
"""
|
|
206
|
+
Remove a rule from the split rule string
|
|
207
|
+
|
|
208
|
+
:param split_rules: s comma separated string
|
|
209
|
+
:param rule_token: rule token
|
|
210
|
+
:param is_sub_rule: TRue to indicate the rule is a subrule
|
|
211
|
+
:return: string of split rules
|
|
212
|
+
"""
|
|
213
|
+
if split_rules is None:
|
|
214
|
+
split_rules = ""
|
|
215
|
+
if is_sub_rule:
|
|
216
|
+
rule_separator = "|"
|
|
217
|
+
key_value_separator = ":"
|
|
218
|
+
else:
|
|
219
|
+
rule_separator = ","
|
|
220
|
+
key_value_separator = "="
|
|
221
|
+
tmp_str = ""
|
|
222
|
+
for tmp_rule in split_rules.split(rule_separator):
|
|
223
|
+
if tmp_rule.startswith(rule_token + key_value_separator):
|
|
224
|
+
continue
|
|
225
|
+
if tmp_str != "":
|
|
226
|
+
tmp_str += rule_separator
|
|
227
|
+
tmp_str += tmp_rule
|
|
228
|
+
return tmp_str
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
# decode split rule string: replace two-letter tags with human-readable names
|
|
232
|
+
def decode_split_rule(split_rules: str) -> str:
|
|
233
|
+
"""
|
|
234
|
+
Convert a compact split-rule string into a human-readable one by replacing
|
|
235
|
+
two-letter tags with their full names from split_rule_dict.
|
|
236
|
+
|
|
237
|
+
The input is a comma-separated string of ``TAG=value`` pairs, e.g.::
|
|
238
|
+
|
|
239
|
+
"AE=1,AN=1,2,3,AI=2"
|
|
240
|
+
|
|
241
|
+
Special case: ``AN`` (addNthFieldToLFN) may carry a comma-separated list
|
|
242
|
+
as its value (e.g. ``AN=1,2,3``). All other tags treat ``=`` as the sole
|
|
243
|
+
separator and never embed commas in their values.
|
|
244
|
+
|
|
245
|
+
:param split_rules: compact comma-separated split-rule string, or None/empty
|
|
246
|
+
:return: human-readable comma-separated string, e.g.
|
|
247
|
+
``"allowEmptyInput=1,addNthFieldToLFN=1,2,3,allowIncompleteInDS=2"``
|
|
248
|
+
Unknown tags are kept as-is.
|
|
249
|
+
"""
|
|
250
|
+
if not split_rules:
|
|
251
|
+
return split_rules or ""
|
|
252
|
+
|
|
253
|
+
# build reverse map: two-letter tag -> full name
|
|
254
|
+
tag_to_name = {tag: name for name, tag in split_rule_dict.items()}
|
|
255
|
+
|
|
256
|
+
# The AN tag is the only one whose value may contain commas.
|
|
257
|
+
an_tag = split_rule_dict["addNthFieldToLFN"] # "AN"
|
|
258
|
+
|
|
259
|
+
# Split on commas, then re-group tokens into (tag, value) pairs.
|
|
260
|
+
# A token starts a new rule when it matches "XX=..." where XX is a known
|
|
261
|
+
# two-letter tag. Otherwise, it is appended to the current rule's value.
|
|
262
|
+
tokens = split_rules.split(",")
|
|
263
|
+
rules = [] # list of [tag, value_parts...]
|
|
264
|
+
for token in tokens:
|
|
265
|
+
# Check whether this token begins a new TAG=value entry
|
|
266
|
+
eq_pos = token.find("=")
|
|
267
|
+
if eq_pos == 2:
|
|
268
|
+
candidate_tag = token[:2]
|
|
269
|
+
if candidate_tag in tag_to_name or candidate_tag == an_tag:
|
|
270
|
+
# Start a new rule
|
|
271
|
+
rules.append([candidate_tag, token[3:]]) # token[3:] = value after "XX="
|
|
272
|
+
continue
|
|
273
|
+
# Not a recognized new rule:
|
|
274
|
+
# - if there are existing rules and the last one is AN, treat as value continuation
|
|
275
|
+
# - otherwise keep the token verbatim as its own entry
|
|
276
|
+
if rules and rules[-1][0] == an_tag:
|
|
277
|
+
rules[-1].append(token)
|
|
278
|
+
else:
|
|
279
|
+
# unknown / non-standard token — keep as-is using a sentinel
|
|
280
|
+
rules.append([None, token])
|
|
281
|
+
|
|
282
|
+
# Reconstruct using full names; rejoin multipart AN values with commas
|
|
283
|
+
result_parts = []
|
|
284
|
+
for parts in rules:
|
|
285
|
+
tag = parts[0]
|
|
286
|
+
if tag is None:
|
|
287
|
+
# unknown / non-standard token kept verbatim
|
|
288
|
+
result_parts.append(parts[1])
|
|
289
|
+
else:
|
|
290
|
+
value = "_".join(parts[1:])
|
|
291
|
+
name = tag_to_name.get(tag, tag) # fall back to raw tag if unknown
|
|
292
|
+
result_parts.append(f"{name}={value}")
|
|
293
|
+
|
|
294
|
+
return ",".join(result_parts)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# remove a rule with name
|
|
298
|
+
def remove_rule_with_name(split_rules: str, rule_name: str, is_sub_rule: bool = False) -> str:
|
|
299
|
+
"""
|
|
300
|
+
Remove a rule from the split rule string using the rule name
|
|
301
|
+
|
|
302
|
+
:param split_rules: a comma separated string
|
|
303
|
+
:param rule_name: rule name
|
|
304
|
+
:param is_sub_rule: True to indicate the rule is a subrule
|
|
305
|
+
:return: string of split rules
|
|
306
|
+
"""
|
|
307
|
+
split_rules = remove_rule(split_rules, split_rule_dict[rule_name], is_sub_rule)
|
|
308
|
+
return split_rules
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os.path
|
|
3
|
+
import sys
|
|
4
|
+
import tempfile
|
|
5
|
+
import traceback
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from idds.client.clientmanager import ClientManager
|
|
9
|
+
from idds.common.utils import get_rest_host
|
|
10
|
+
from pandacommon.pandalogger.LogWrapper import LogWrapper
|
|
11
|
+
|
|
12
|
+
# import PandaLogger before idds modules not to change message levels of other modules
|
|
13
|
+
from pandacommon.pandalogger.PandaLogger import PandaLogger
|
|
14
|
+
from ruamel.yaml import YAML
|
|
15
|
+
|
|
16
|
+
from pandaserver.srvcore.CoreUtils import clean_user_id, commands_get_status_output
|
|
17
|
+
from pandaserver.srvcore.MailUtils import MailUtils
|
|
18
|
+
from pandaserver.workflow import pcwl_utils, workflow_utils
|
|
19
|
+
from pandaserver.workflow.snakeparser import Parser
|
|
20
|
+
|
|
21
|
+
_logger = PandaLogger().getLogger("workflow_processor")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
SUPPORTED_WORKFLOW_LANGUAGES = ["cwl", "snakemake"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# process workflow
|
|
28
|
+
class WorkflowProcessor(object):
|
|
29
|
+
# constructor
|
|
30
|
+
def __init__(self, task_buffer=None, log_stream=None):
|
|
31
|
+
self.taskBuffer = task_buffer
|
|
32
|
+
self.log = _logger
|
|
33
|
+
|
|
34
|
+
# process a file
|
|
35
|
+
def process(
|
|
36
|
+
self,
|
|
37
|
+
file_name,
|
|
38
|
+
to_delete=False,
|
|
39
|
+
test_mode=False,
|
|
40
|
+
get_log=False,
|
|
41
|
+
dump_workflow=False,
|
|
42
|
+
):
|
|
43
|
+
is_fatal = False
|
|
44
|
+
request_id = None
|
|
45
|
+
dump_str = None
|
|
46
|
+
try:
|
|
47
|
+
with open(file_name) as f:
|
|
48
|
+
ops = json.load(f)
|
|
49
|
+
user_name = clean_user_id(ops["userName"])
|
|
50
|
+
base_platform = ops["data"].get("base_platform")
|
|
51
|
+
for task_type in ops["data"]["taskParams"]:
|
|
52
|
+
ops["data"]["taskParams"][task_type]["userName"] = user_name
|
|
53
|
+
if base_platform:
|
|
54
|
+
ops["data"]["taskParams"][task_type]["basePlatform"] = base_platform
|
|
55
|
+
log_token = f"< id=\"{user_name}\" test={test_mode} outDS={ops['data']['outDS']} >"
|
|
56
|
+
tmpLog = LogWrapper(self.log, log_token)
|
|
57
|
+
tmpLog.info(f"start {file_name}")
|
|
58
|
+
sandbox_url = os.path.join(ops["data"]["sourceURL"], "cache", ops["data"]["sandbox"])
|
|
59
|
+
# IO through json files
|
|
60
|
+
ops_file = tempfile.NamedTemporaryFile(delete=False, mode="w")
|
|
61
|
+
json.dump(ops, ops_file)
|
|
62
|
+
ops_file.close()
|
|
63
|
+
# execute main in another process to avoid chdir mess
|
|
64
|
+
tmp_stat, tmp_out = commands_get_status_output(
|
|
65
|
+
f"python {__file__} {sandbox_url} '{log_token}' {dump_workflow} {ops_file.name} '{user_name}' {test_mode}"
|
|
66
|
+
)
|
|
67
|
+
if tmp_stat:
|
|
68
|
+
is_OK = False
|
|
69
|
+
tmpLog.error(f"main execution failed with {tmp_stat}:{tmp_out}")
|
|
70
|
+
else:
|
|
71
|
+
with open(tmp_out.split("\n")[-1]) as tmp_out_file:
|
|
72
|
+
is_OK, is_fatal, request_id, dump_str = json.load(tmp_out_file)
|
|
73
|
+
try:
|
|
74
|
+
os.remove(tmp_out)
|
|
75
|
+
except Exception:
|
|
76
|
+
pass
|
|
77
|
+
if is_OK:
|
|
78
|
+
tmpLog.info(f"is_OK={is_OK} request_id={request_id}")
|
|
79
|
+
else:
|
|
80
|
+
tmpLog.info(f"is_OK={is_OK} is_fatal={is_fatal} request_id={request_id}")
|
|
81
|
+
if to_delete or (not test_mode and (is_OK or is_fatal)):
|
|
82
|
+
if dump_str:
|
|
83
|
+
dump_str = dump_str + tmpLog.dumpToString()
|
|
84
|
+
else:
|
|
85
|
+
dump_str = tmpLog.dumpToString()
|
|
86
|
+
tmpLog.debug(f"delete {file_name}")
|
|
87
|
+
try:
|
|
88
|
+
os.remove(file_name)
|
|
89
|
+
except Exception:
|
|
90
|
+
pass
|
|
91
|
+
# send notification
|
|
92
|
+
if not test_mode and self.taskBuffer is not None:
|
|
93
|
+
toAdder = self.taskBuffer.getEmailAddr(user_name)
|
|
94
|
+
if toAdder is None or toAdder.startswith("notsend"):
|
|
95
|
+
tmpLog.debug("skip to send notification since suppressed")
|
|
96
|
+
else:
|
|
97
|
+
# message
|
|
98
|
+
if is_OK:
|
|
99
|
+
mailSubject = f"PANDA Notification for Workflow {ops['data']['outDS']}"
|
|
100
|
+
mailBody = f"Hello,\n\nWorkflow:{ops['data']['outDS']} has been accepted with RequestID:{request_id}\n\n"
|
|
101
|
+
else:
|
|
102
|
+
mailSubject = f"PANDA WARNING for Workflow={ops['data']['outDS']}"
|
|
103
|
+
mailBody = "Hello,\n\nWorkflow {} was not accepted\n\n".format(ops["data"]["outDS"], request_id)
|
|
104
|
+
mailBody += f"Reason : {dump_str}\n"
|
|
105
|
+
# send
|
|
106
|
+
tmpSM = MailUtils().send(toAdder, mailSubject, mailBody)
|
|
107
|
+
tmpLog.debug(f"sent message with {tmpSM}")
|
|
108
|
+
except Exception as e:
|
|
109
|
+
is_OK = False
|
|
110
|
+
tmpLog.error(f"failed to run with {str(e)} {traceback.format_exc()}")
|
|
111
|
+
if get_log:
|
|
112
|
+
ret_val = {"status": is_OK, "request_id": request_id}
|
|
113
|
+
if is_OK:
|
|
114
|
+
ret_val["log"] = dump_str
|
|
115
|
+
else:
|
|
116
|
+
if dump_str is None:
|
|
117
|
+
ret_val["log"] = tmpLog.dumpToString()
|
|
118
|
+
else:
|
|
119
|
+
ret_val["log"] = dump_str
|
|
120
|
+
return ret_val
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# execute chdir in another process
|
|
124
|
+
def core_exec(sandbox_url, log_token, dump_workflow, ops_file, user_name, test_mode):
|
|
125
|
+
tmpLog = LogWrapper(_logger, log_token)
|
|
126
|
+
is_OK = True
|
|
127
|
+
is_fatal = False
|
|
128
|
+
request_id = None
|
|
129
|
+
if dump_workflow == "True":
|
|
130
|
+
dump_workflow = True
|
|
131
|
+
else:
|
|
132
|
+
dump_workflow = False
|
|
133
|
+
if test_mode == "True":
|
|
134
|
+
test_mode = True
|
|
135
|
+
else:
|
|
136
|
+
test_mode = False
|
|
137
|
+
try:
|
|
138
|
+
with open(ops_file) as f:
|
|
139
|
+
ops = json.load(f)
|
|
140
|
+
try:
|
|
141
|
+
os.remove(ops_file)
|
|
142
|
+
except Exception:
|
|
143
|
+
pass
|
|
144
|
+
# go to temp dir
|
|
145
|
+
cur_dir = os.getcwd()
|
|
146
|
+
with tempfile.TemporaryDirectory() as tmp_dirname:
|
|
147
|
+
os.chdir(tmp_dirname)
|
|
148
|
+
# download sandbox
|
|
149
|
+
tmpLog.info(f"downloading sandbox from {sandbox_url}")
|
|
150
|
+
with requests.get(sandbox_url, allow_redirects=True, verify=False, stream=True) as r:
|
|
151
|
+
if r.status_code == 400:
|
|
152
|
+
tmpLog.error("not found")
|
|
153
|
+
is_fatal = True
|
|
154
|
+
is_OK = False
|
|
155
|
+
elif r.status_code != 200:
|
|
156
|
+
tmpLog.error(f"bad HTTP response {r.status_code}")
|
|
157
|
+
is_OK = False
|
|
158
|
+
# extract sandbox
|
|
159
|
+
if is_OK:
|
|
160
|
+
with open(ops["data"]["sandbox"], "wb") as fs:
|
|
161
|
+
for chunk in r.raw.stream(1024, decode_content=False):
|
|
162
|
+
if chunk:
|
|
163
|
+
fs.write(chunk)
|
|
164
|
+
fs.close()
|
|
165
|
+
tmp_stat, tmp_out = commands_get_status_output(f"tar xvfz {ops['data']['sandbox']}")
|
|
166
|
+
if tmp_stat != 0:
|
|
167
|
+
tmpLog.error(tmp_out)
|
|
168
|
+
dump_str = f"failed to extract {ops['data']['sandbox']}"
|
|
169
|
+
tmpLog.error(dump_str)
|
|
170
|
+
is_fatal = True
|
|
171
|
+
is_OK = False
|
|
172
|
+
# parse workflow files
|
|
173
|
+
if is_OK:
|
|
174
|
+
tmpLog.info("parse workflow")
|
|
175
|
+
workflow_name = None
|
|
176
|
+
if (wf_lang := ops["data"]["language"]) in SUPPORTED_WORKFLOW_LANGUAGES:
|
|
177
|
+
if wf_lang == "cwl":
|
|
178
|
+
workflow_name = ops["data"].get("workflow_name")
|
|
179
|
+
nodes, root_in = pcwl_utils.parse_workflow_file(ops["data"]["workflowSpecFile"], tmpLog)
|
|
180
|
+
with open(ops["data"]["workflowInputFile"]) as workflow_input:
|
|
181
|
+
yaml = YAML(typ="safe", pure=True)
|
|
182
|
+
data = yaml.load(workflow_input)
|
|
183
|
+
elif wf_lang == "snakemake":
|
|
184
|
+
parser = Parser(ops["data"]["workflowSpecFile"], logger=tmpLog)
|
|
185
|
+
nodes, root_in = parser.parse_nodes()
|
|
186
|
+
data = dict()
|
|
187
|
+
# resolve nodes
|
|
188
|
+
s_id, t_nodes, nodes = workflow_utils.resolve_nodes(nodes, root_in, data, 0, set(), ops["data"]["outDS"], tmpLog)
|
|
189
|
+
workflow_utils.set_workflow_outputs(nodes)
|
|
190
|
+
id_node_map = workflow_utils.get_node_id_map(nodes)
|
|
191
|
+
[node.resolve_params(ops["data"]["taskParams"], id_node_map) for node in nodes]
|
|
192
|
+
dump_str = "the description was internally converted as follows\n" + workflow_utils.dump_nodes(nodes)
|
|
193
|
+
tmpLog.info(dump_str)
|
|
194
|
+
for node in nodes:
|
|
195
|
+
s_check, o_check = node.verify()
|
|
196
|
+
tmp_str = f"Verification failure in ID:{node.id} {o_check}"
|
|
197
|
+
if not s_check:
|
|
198
|
+
tmpLog.error(tmp_str)
|
|
199
|
+
dump_str += tmp_str
|
|
200
|
+
dump_str += "\n"
|
|
201
|
+
is_fatal = True
|
|
202
|
+
is_OK = False
|
|
203
|
+
else:
|
|
204
|
+
dump_str = "{} is not supported to describe the workflow"
|
|
205
|
+
tmpLog.error(dump_str)
|
|
206
|
+
is_fatal = True
|
|
207
|
+
is_OK = False
|
|
208
|
+
# convert to workflow
|
|
209
|
+
if is_OK:
|
|
210
|
+
(
|
|
211
|
+
workflow_to_submit,
|
|
212
|
+
dump_str_list,
|
|
213
|
+
) = workflow_utils.convert_nodes_to_workflow(nodes, workflow_name=workflow_name)
|
|
214
|
+
try:
|
|
215
|
+
if workflow_to_submit:
|
|
216
|
+
if not test_mode:
|
|
217
|
+
tmpLog.info("submit workflow")
|
|
218
|
+
wm = ClientManager(host=get_rest_host())
|
|
219
|
+
request_id = wm.submit(workflow_to_submit, username=user_name, use_dataset_name=False)
|
|
220
|
+
try:
|
|
221
|
+
request_id = int(request_id)
|
|
222
|
+
except Exception:
|
|
223
|
+
# wrong request_id
|
|
224
|
+
is_fatal = True
|
|
225
|
+
is_OK = False
|
|
226
|
+
else:
|
|
227
|
+
dump_str = "workflow is empty"
|
|
228
|
+
tmpLog.error(dump_str)
|
|
229
|
+
is_fatal = True
|
|
230
|
+
is_OK = False
|
|
231
|
+
except Exception as e:
|
|
232
|
+
dump_str = f"failed to submit the workflow with {str(e)}"
|
|
233
|
+
tmpLog.error(f"{dump_str} {traceback.format_exc()}")
|
|
234
|
+
if dump_workflow:
|
|
235
|
+
tmpLog.debug("\n" + "".join(dump_str_list))
|
|
236
|
+
os.chdir(cur_dir)
|
|
237
|
+
except Exception as e:
|
|
238
|
+
is_OK = False
|
|
239
|
+
is_fatal = True
|
|
240
|
+
tmpLog.error(f"failed to run with {str(e)} {traceback.format_exc()}")
|
|
241
|
+
|
|
242
|
+
with tempfile.NamedTemporaryFile(delete=False, mode="w") as tmp_json:
|
|
243
|
+
json.dump([is_OK, is_fatal, request_id, tmpLog.dumpToString()], tmp_json)
|
|
244
|
+
print(tmp_json.name)
|
|
245
|
+
sys.exit(0)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
if __name__ == "__main__":
|
|
249
|
+
core_exec(*sys.argv[1:])
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# wrapper for Oracle Connection
|
|
2
|
+
class WrappedOracleConn(object):
|
|
3
|
+
def __init__(self, conn):
|
|
4
|
+
self.orig_conn = conn
|
|
5
|
+
|
|
6
|
+
def __getattribute__(self, item):
|
|
7
|
+
try:
|
|
8
|
+
return object.__getattribute__(self.orig_conn, item)
|
|
9
|
+
except Exception:
|
|
10
|
+
pass
|
|
11
|
+
return object.__getattribute__(self, item)
|
|
12
|
+
|
|
13
|
+
# override context manager protocol not to close connection
|
|
14
|
+
def __enter__(self):
|
|
15
|
+
return self.orig_conn.__enter__()
|
|
16
|
+
|
|
17
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
18
|
+
self.orig_conn.commit()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Checking DB schema version for PanDA Server.
|
|
3
|
+
If there is an issue and the pandaEmailNotification var is
|
|
4
|
+
defined in panda_config, it will send an email notification.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from packaging import version
|
|
8
|
+
|
|
9
|
+
from pandaserver.config import panda_config
|
|
10
|
+
from pandaserver.srvcore.MailUtils import MailUtils
|
|
11
|
+
from pandaserver.taskbuffer import PandaDBSchemaInfo
|
|
12
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
13
|
+
|
|
14
|
+
proxyS = DBProxy()
|
|
15
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
16
|
+
|
|
17
|
+
sql = "select major || '.' || minor || '.' || patch from ATLAS_PANDA.pandadb_version where component = 'PanDA'"
|
|
18
|
+
|
|
19
|
+
res = proxyS.querySQL(sql)
|
|
20
|
+
dbVersion = res[0][0]
|
|
21
|
+
|
|
22
|
+
serverDBVersion = PandaDBSchemaInfo.PandaDBSchemaInfo().method()
|
|
23
|
+
|
|
24
|
+
if version.parse(dbVersion) >= version.parse(serverDBVersion):
|
|
25
|
+
print("DB schema check: OK")
|
|
26
|
+
else:
|
|
27
|
+
message_body = (
|
|
28
|
+
f"There is an issue with {panda_config.pserveralias}. "
|
|
29
|
+
f"PanDA DB schema installed is {dbVersion} while PanDA Server requires "
|
|
30
|
+
f"version {serverDBVersion} to be installed. Please check the official docs "
|
|
31
|
+
f"for instructions on how to upgrade the schema."
|
|
32
|
+
)
|
|
33
|
+
print(message_body)
|
|
34
|
+
|
|
35
|
+
if "pandaEmailNotification" in panda_config.__dict__:
|
|
36
|
+
MailUtils().send(
|
|
37
|
+
panda_config.pandaEmailNotification,
|
|
38
|
+
f"PanDA DB Version installed is not correct for {panda_config.pserveralias}",
|
|
39
|
+
message_body,
|
|
40
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import optparse
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
# password
|
|
5
|
+
from pandaserver.config import panda_config
|
|
6
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
7
|
+
|
|
8
|
+
option_parser = optparse.OptionParser(conflict_handler="resolve")
|
|
9
|
+
option_parser.add_option("--user", action="store", dest="user", default=None, help="prodUserName")
|
|
10
|
+
option_parser.add_option(
|
|
11
|
+
"--unban",
|
|
12
|
+
action="store_const",
|
|
13
|
+
const=True,
|
|
14
|
+
dest="unban",
|
|
15
|
+
default=False,
|
|
16
|
+
help="unban the user",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
options, args = option_parser.parse_args()
|
|
20
|
+
|
|
21
|
+
if options.user is None:
|
|
22
|
+
print("--user=<prodUserName> is required")
|
|
23
|
+
sys.exit(1)
|
|
24
|
+
|
|
25
|
+
proxyS = DBProxy()
|
|
26
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
27
|
+
|
|
28
|
+
prodUserName = sys.argv[1]
|
|
29
|
+
|
|
30
|
+
varMap = {}
|
|
31
|
+
varMap[":name"] = options.user
|
|
32
|
+
if options.unban:
|
|
33
|
+
varMap[":status"] = None
|
|
34
|
+
else:
|
|
35
|
+
varMap[":status"] = "disabled"
|
|
36
|
+
|
|
37
|
+
sql = "UPDATE ATLAS_PANDAMETA.users SET status=:status WHERE name=:name"
|
|
38
|
+
|
|
39
|
+
status, res = proxyS.querySQLS(sql, varMap)
|
|
40
|
+
if res is None:
|
|
41
|
+
print("Failed with database error")
|
|
42
|
+
else:
|
|
43
|
+
print(f"{res} rows updated")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
# password
|
|
4
|
+
from pandaserver.config import panda_config
|
|
5
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
6
|
+
|
|
7
|
+
proxyS = DBProxy()
|
|
8
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
9
|
+
|
|
10
|
+
varMap = {}
|
|
11
|
+
varMap[":prodSourceLabel"] = "managed"
|
|
12
|
+
varMap[":taskID"] = sys.argv[1]
|
|
13
|
+
varMap[":prio"] = sys.argv[2]
|
|
14
|
+
sql = "UPDATE %s SET currentPriority=currentPriority+:prio WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID"
|
|
15
|
+
for table in [
|
|
16
|
+
"ATLAS_PANDA.jobsActive4",
|
|
17
|
+
"ATLAS_PANDA.jobsDefined4",
|
|
18
|
+
]:
|
|
19
|
+
status, res = proxyS.querySQLS(sql % table, varMap)
|