panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import optparse
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
# password
|
|
5
|
+
from pandaserver.config import panda_config
|
|
6
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
7
|
+
from pandaserver.userinterface import Client
|
|
8
|
+
|
|
9
|
+
option_parser = optparse.OptionParser(conflict_handler="resolve")
|
|
10
|
+
option_parser.add_option(
|
|
11
|
+
"-9",
|
|
12
|
+
action="store_const",
|
|
13
|
+
const=True,
|
|
14
|
+
dest="forceKill",
|
|
15
|
+
default=False,
|
|
16
|
+
help="kill jobs even if they are still running",
|
|
17
|
+
)
|
|
18
|
+
option_parser.add_option(
|
|
19
|
+
"--noRunning",
|
|
20
|
+
action="store_const",
|
|
21
|
+
const=True,
|
|
22
|
+
dest="noRunning",
|
|
23
|
+
default=True,
|
|
24
|
+
help="kill jobs if they are not in running or transferring (ON by default)",
|
|
25
|
+
)
|
|
26
|
+
option_parser.add_option(
|
|
27
|
+
"--killAny",
|
|
28
|
+
action="store_const",
|
|
29
|
+
const=True,
|
|
30
|
+
dest="killAny",
|
|
31
|
+
default=False,
|
|
32
|
+
help="kill jobs in any status",
|
|
33
|
+
)
|
|
34
|
+
option_parser.add_option(
|
|
35
|
+
"--prodSourceLabel",
|
|
36
|
+
action="store",
|
|
37
|
+
dest="prodSourceLabel",
|
|
38
|
+
default="managed",
|
|
39
|
+
help="prodSourceLabel",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
options, args = option_parser.parse_args()
|
|
43
|
+
|
|
44
|
+
proxyS = DBProxy()
|
|
45
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
46
|
+
|
|
47
|
+
jobs = []
|
|
48
|
+
|
|
49
|
+
varMap = {}
|
|
50
|
+
varMap[":prodSourceLabel"] = options.prodSourceLabel
|
|
51
|
+
varMap[":taskID"] = args[0]
|
|
52
|
+
if not options.noRunning or options.killAny:
|
|
53
|
+
sql = "SELECT PandaID FROM %s WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID ORDER BY PandaID"
|
|
54
|
+
else:
|
|
55
|
+
sql = "SELECT PandaID FROM %s WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID AND NOT jobStatus IN (:js1,:js2,:js3) ORDER BY PandaID"
|
|
56
|
+
varMap[":js1"] = "running"
|
|
57
|
+
varMap[":js2"] = "transferring"
|
|
58
|
+
varMap[":js3"] = "holding"
|
|
59
|
+
for table in [
|
|
60
|
+
"ATLAS_PANDA.jobsActive4",
|
|
61
|
+
"ATLAS_PANDA.jobsDefined4",
|
|
62
|
+
]:
|
|
63
|
+
status, res = proxyS.querySQLS(sql % table, varMap)
|
|
64
|
+
if res is not None:
|
|
65
|
+
for (id,) in res:
|
|
66
|
+
if id not in jobs:
|
|
67
|
+
jobs.append(id)
|
|
68
|
+
|
|
69
|
+
print(f"The number of jobs to be killed for prodSourceLabel={options.prodSourceLabel} taskID={args[0]}: {len(jobs)}")
|
|
70
|
+
if len(jobs):
|
|
71
|
+
nJob = 100
|
|
72
|
+
iJob = 0
|
|
73
|
+
while iJob < len(jobs):
|
|
74
|
+
print(f"kill {str(jobs[iJob:iJob + nJob])}")
|
|
75
|
+
if options.forceKill:
|
|
76
|
+
Client.kill_jobs(jobs[iJob : iJob + nJob], 9)
|
|
77
|
+
else:
|
|
78
|
+
Client.kill_jobs(jobs[iJob : iJob + nJob])
|
|
79
|
+
iJob += nJob
|
|
80
|
+
time.sleep(1)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import optparse
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
# password
|
|
5
|
+
from pandaserver.config import panda_config
|
|
6
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
7
|
+
from pandaserver.userinterface import Client
|
|
8
|
+
|
|
9
|
+
option_parser = optparse.OptionParser(conflict_handler="resolve")
|
|
10
|
+
option_parser.add_option("--user", action="store", dest="user", default=None, help="prodUserName")
|
|
11
|
+
option_parser.add_option("--jobID", action="store", dest="jobID", default=None, help="jobDefinitionID")
|
|
12
|
+
option_parser.add_option(
|
|
13
|
+
"--jobsetID",
|
|
14
|
+
action="store",
|
|
15
|
+
dest="jobsetID",
|
|
16
|
+
default=None,
|
|
17
|
+
help="jobsetID, or 'all' to kill all jobs",
|
|
18
|
+
)
|
|
19
|
+
option_parser.add_option(
|
|
20
|
+
"--prodSourceLabel",
|
|
21
|
+
action="store",
|
|
22
|
+
dest="prodSourceLabel",
|
|
23
|
+
default=None,
|
|
24
|
+
help="additional prodSourceLabel",
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
options, args = option_parser.parse_args()
|
|
29
|
+
|
|
30
|
+
if options.user is None:
|
|
31
|
+
print("--user=<prodUserName> is required")
|
|
32
|
+
sys.exit(1)
|
|
33
|
+
if options.jobID is None and options.jobsetID is None:
|
|
34
|
+
print("--jobID=<jobDefinitionID> or --jobsetID=<jobsetID or 'all'> is required")
|
|
35
|
+
sys.exit(1)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
proxyS = DBProxy()
|
|
39
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
40
|
+
|
|
41
|
+
prodUserName = sys.argv[1]
|
|
42
|
+
|
|
43
|
+
varMap = {}
|
|
44
|
+
varMap[":src1"] = "user"
|
|
45
|
+
varMap[":src2"] = "panda"
|
|
46
|
+
varMap[":prodUserName"] = options.user
|
|
47
|
+
srcSQL = "(:src1,:src2"
|
|
48
|
+
if options.jobID is not None:
|
|
49
|
+
varMap[":jobDefinitionID"] = options.jobID
|
|
50
|
+
if options.jobsetID not in (None, "all"):
|
|
51
|
+
varMap[":jobsetID"] = options.jobsetID
|
|
52
|
+
if options.prodSourceLabel is not None:
|
|
53
|
+
varMap[":src3"] = options.prodSourceLabel
|
|
54
|
+
srcSQL += ",:src3"
|
|
55
|
+
srcSQL += ")"
|
|
56
|
+
|
|
57
|
+
jobs = []
|
|
58
|
+
tables = [
|
|
59
|
+
"ATLAS_PANDA.jobsActive4",
|
|
60
|
+
"ATLAS_PANDA.jobsDefined4",
|
|
61
|
+
]
|
|
62
|
+
for table in tables:
|
|
63
|
+
sql = f"SELECT PandaID FROM {table} WHERE prodUserName=:prodUserName AND prodSourceLabel IN {srcSQL} "
|
|
64
|
+
if options.jobID is not None:
|
|
65
|
+
sql += "AND jobDefinitionID=:jobDefinitionID "
|
|
66
|
+
if options.jobsetID not in (None, "all"):
|
|
67
|
+
sql += "AND jobsetID=:jobsetID "
|
|
68
|
+
sql += "ORDER BY PandaID "
|
|
69
|
+
status, res = proxyS.querySQLS(sql, varMap)
|
|
70
|
+
if res is not None:
|
|
71
|
+
for (id,) in res:
|
|
72
|
+
if id not in jobs:
|
|
73
|
+
jobs.append(id)
|
|
74
|
+
if len(jobs):
|
|
75
|
+
iJob = 0
|
|
76
|
+
nJob = 1000
|
|
77
|
+
while iJob < len(jobs):
|
|
78
|
+
subJobs = jobs[iJob : iJob + nJob]
|
|
79
|
+
print(f"kill {str(subJobs)} {iJob}/{len(jobs)}")
|
|
80
|
+
Client.kill_jobs(subJobs, code=9)
|
|
81
|
+
iJob += nJob
|
|
82
|
+
else:
|
|
83
|
+
print("no job was killed")
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import time
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
from pandaserver.taskbuffer.FileSpec import FileSpec
|
|
6
|
+
from pandaserver.taskbuffer.JobSpec import JobSpec
|
|
7
|
+
from pandaserver.userinterface import Client
|
|
8
|
+
|
|
9
|
+
prodUserNameDefault = "unknown-user"
|
|
10
|
+
prodUserName = None
|
|
11
|
+
prodUserNameDP = None
|
|
12
|
+
prodUserNamePipeline = None
|
|
13
|
+
site = "ANALY_BNL-LSST"
|
|
14
|
+
PIPELINE_TASK = None
|
|
15
|
+
PIPELINE_PROCESSINSTANCE = None
|
|
16
|
+
PIPELINE_EXECUTIONNUMBER = None
|
|
17
|
+
PIPELINE_STREAM = None
|
|
18
|
+
lsstJobParams = ""
|
|
19
|
+
|
|
20
|
+
for idx, argv in enumerate(sys.argv):
|
|
21
|
+
if argv == "--site":
|
|
22
|
+
try:
|
|
23
|
+
site = sys.argv[idx + 1]
|
|
24
|
+
except Exception:
|
|
25
|
+
site = "ANALY_BNL-LSST"
|
|
26
|
+
if argv == "-DP_USER":
|
|
27
|
+
try:
|
|
28
|
+
prodUserNameDP = sys.argv[idx + 1]
|
|
29
|
+
if len(lsstJobParams):
|
|
30
|
+
lsstJobParams += "|"
|
|
31
|
+
lsstJobParams += f"DP_USER={str(prodUserNameDP)}"
|
|
32
|
+
except Exception:
|
|
33
|
+
prodUserNameDP = None
|
|
34
|
+
if argv == "-PIPELINE_USER":
|
|
35
|
+
try:
|
|
36
|
+
prodUserNamePipeline = sys.argv[idx + 1]
|
|
37
|
+
if len(lsstJobParams):
|
|
38
|
+
lsstJobParams += "|"
|
|
39
|
+
lsstJobParams += f"PIPELINE_USER={str(prodUserNamePipeline)}"
|
|
40
|
+
except Exception:
|
|
41
|
+
prodUserNamePipeline = None
|
|
42
|
+
if argv == "-PIPELINE_TASK":
|
|
43
|
+
try:
|
|
44
|
+
PIPELINE_TASK = sys.argv[idx + 1]
|
|
45
|
+
if len(lsstJobParams):
|
|
46
|
+
lsstJobParams += "|"
|
|
47
|
+
lsstJobParams += f"PIPELINE_TASK={str(PIPELINE_TASK)}"
|
|
48
|
+
except Exception:
|
|
49
|
+
PIPELINE_TASK = None
|
|
50
|
+
if argv == "-PIPELINE_PROCESSINSTANCE":
|
|
51
|
+
try:
|
|
52
|
+
PIPELINE_PROCESSINSTANCE = int(sys.argv[idx + 1])
|
|
53
|
+
if len(lsstJobParams):
|
|
54
|
+
lsstJobParams += "|"
|
|
55
|
+
lsstJobParams += f"PIPELINE_PROCESSINSTANCE={str(PIPELINE_PROCESSINSTANCE)}"
|
|
56
|
+
except Exception:
|
|
57
|
+
PIPELINE_PROCESSINSTANCE = None
|
|
58
|
+
if argv == "-PIPELINE_EXECUTIONNUMBER":
|
|
59
|
+
try:
|
|
60
|
+
PIPELINE_EXECUTIONNUMBER = int(sys.argv[idx + 1])
|
|
61
|
+
if len(lsstJobParams):
|
|
62
|
+
lsstJobParams += "|"
|
|
63
|
+
lsstJobParams += f"PIPELINE_EXECUTIONNUMBER={str(PIPELINE_EXECUTIONNUMBER)}"
|
|
64
|
+
except Exception:
|
|
65
|
+
PIPELINE_EXECUTIONNUMBER = None
|
|
66
|
+
if argv == "-PIPELINE_STREAM":
|
|
67
|
+
try:
|
|
68
|
+
PIPELINE_STREAM = int(sys.argv[idx + 1])
|
|
69
|
+
if len(lsstJobParams):
|
|
70
|
+
lsstJobParams += "|"
|
|
71
|
+
lsstJobParams += f"PIPELINE_STREAM={str(PIPELINE_STREAM)}"
|
|
72
|
+
except Exception:
|
|
73
|
+
PIPELINE_STREAM = None
|
|
74
|
+
|
|
75
|
+
# DP_USER and PIPELINE_USER preference
|
|
76
|
+
if prodUserNameDP is not None:
|
|
77
|
+
prodUserName = prodUserNameDP
|
|
78
|
+
elif prodUserNamePipeline is not None:
|
|
79
|
+
prodUserName = prodUserNamePipeline
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# site = sys.argv[1]
|
|
83
|
+
# site = 'ANALY_BNL-LSST' #orig
|
|
84
|
+
# site = 'BNL-LSST'
|
|
85
|
+
# site = 'SWT2_CPB-LSST'
|
|
86
|
+
# site = 'UTA_SWT2-LSST'
|
|
87
|
+
# site = 'ANALY_SWT2_CPB-LSST'
|
|
88
|
+
|
|
89
|
+
destName = None
|
|
90
|
+
|
|
91
|
+
if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_PROCESSINSTANCE is not None:
|
|
92
|
+
datasetName = f"panda.lsst.user.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(prodUserName)}"
|
|
93
|
+
else:
|
|
94
|
+
datasetName = f"panda.lsst.user.jschovan.{str(uuid.uuid4())}"
|
|
95
|
+
|
|
96
|
+
if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_EXECUTIONNUMBER is not None and PIPELINE_STREAM is not None:
|
|
97
|
+
jobName = f"job.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(PIPELINE_EXECUTIONNUMBER)}.{str(prodUserName)}.{str(PIPELINE_STREAM)}"
|
|
98
|
+
else:
|
|
99
|
+
jobName = f"{str(uuid.uuid4())}"
|
|
100
|
+
|
|
101
|
+
if PIPELINE_STREAM is not None:
|
|
102
|
+
jobDefinitionID = PIPELINE_STREAM
|
|
103
|
+
else:
|
|
104
|
+
jobDefinitionID = int(time.time()) % 10000
|
|
105
|
+
job = JobSpec()
|
|
106
|
+
job.jobDefinitionID = jobDefinitionID
|
|
107
|
+
job.jobName = jobName
|
|
108
|
+
job.transformation = "http://pandawms.org/pandawms-jobcache/lsst-trf.sh"
|
|
109
|
+
job.destinationDBlock = datasetName
|
|
110
|
+
job.destinationSE = "local"
|
|
111
|
+
job.currentPriority = 1000
|
|
112
|
+
job.prodSourceLabel = "panda"
|
|
113
|
+
job.jobParameters = f' --lsstJobParams="{lsstJobParams}" '
|
|
114
|
+
if prodUserName is not None:
|
|
115
|
+
job.prodUserName = prodUserName
|
|
116
|
+
else:
|
|
117
|
+
job.prodUserName = prodUserNameDefault
|
|
118
|
+
if PIPELINE_PROCESSINSTANCE is not None:
|
|
119
|
+
job.taskID = PIPELINE_PROCESSINSTANCE
|
|
120
|
+
if PIPELINE_EXECUTIONNUMBER is not None:
|
|
121
|
+
job.attemptNr = PIPELINE_EXECUTIONNUMBER
|
|
122
|
+
if PIPELINE_TASK is not None:
|
|
123
|
+
job.processingType = PIPELINE_TASK
|
|
124
|
+
job.computingSite = site
|
|
125
|
+
job.VO = "lsst"
|
|
126
|
+
|
|
127
|
+
fileOL = FileSpec()
|
|
128
|
+
fileOL.lfn = f"{job.jobName}.job.log.tgz"
|
|
129
|
+
fileOL.destinationDBlock = job.destinationDBlock
|
|
130
|
+
fileOL.destinationSE = job.destinationSE
|
|
131
|
+
fileOL.dataset = job.destinationDBlock
|
|
132
|
+
fileOL.type = "log"
|
|
133
|
+
job.addFile(fileOL)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
s, o = Client.submit_jobs([job])
|
|
137
|
+
print(s)
|
|
138
|
+
for x in o:
|
|
139
|
+
print(f"PandaID={x[0]}")
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import time
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
from pandaserver.taskbuffer.FileSpec import FileSpec
|
|
6
|
+
from pandaserver.taskbuffer.JobSpec import JobSpec
|
|
7
|
+
from pandaserver.userinterface import Client
|
|
8
|
+
|
|
9
|
+
prodUserNameDefault = "jschovan"
|
|
10
|
+
prodUserName = None
|
|
11
|
+
prodUserNameDP = None
|
|
12
|
+
prodUserNamePipeline = None
|
|
13
|
+
# site = 'ANALY_BNL-LSST-OSGRPMtest'
|
|
14
|
+
# site = 'ANALY_BNL-LSST-gridgk01-mergetest'
|
|
15
|
+
site = "ANALY_BNL-LSST-mergetest"
|
|
16
|
+
PIPELINE_TASK = None
|
|
17
|
+
PIPELINE_PROCESSINSTANCE = None
|
|
18
|
+
PIPELINE_EXECUTIONNUMBER = None
|
|
19
|
+
PIPELINE_STREAM = None
|
|
20
|
+
lsstJobParams = ""
|
|
21
|
+
|
|
22
|
+
for idx, argv in enumerate(sys.argv):
|
|
23
|
+
if argv == "--site":
|
|
24
|
+
try:
|
|
25
|
+
site = sys.argv[idx + 1]
|
|
26
|
+
except Exception:
|
|
27
|
+
site = "ANALY_BNL-LSST-mergetest"
|
|
28
|
+
if argv == "-DP_USER":
|
|
29
|
+
try:
|
|
30
|
+
prodUserNameDP = sys.argv[idx + 1]
|
|
31
|
+
if len(lsstJobParams):
|
|
32
|
+
lsstJobParams += "|"
|
|
33
|
+
lsstJobParams += f"DP_USER={str(prodUserNameDP)}"
|
|
34
|
+
except Exception:
|
|
35
|
+
prodUserNameDP = None
|
|
36
|
+
if argv == "-PIPELINE_USER":
|
|
37
|
+
try:
|
|
38
|
+
prodUserNamePipeline = sys.argv[idx + 1]
|
|
39
|
+
if len(lsstJobParams):
|
|
40
|
+
lsstJobParams += "|"
|
|
41
|
+
lsstJobParams += f"PIPELINE_USER={str(prodUserNamePipeline)}"
|
|
42
|
+
except Exception:
|
|
43
|
+
prodUserNamePipeline = None
|
|
44
|
+
if argv == "-PIPELINE_TASK":
|
|
45
|
+
try:
|
|
46
|
+
PIPELINE_TASK = sys.argv[idx + 1]
|
|
47
|
+
if len(lsstJobParams):
|
|
48
|
+
lsstJobParams += "|"
|
|
49
|
+
lsstJobParams += f"PIPELINE_TASK={str(PIPELINE_TASK)}"
|
|
50
|
+
except Exception:
|
|
51
|
+
PIPELINE_TASK = None
|
|
52
|
+
if argv == "-PIPELINE_PROCESSINSTANCE":
|
|
53
|
+
try:
|
|
54
|
+
PIPELINE_PROCESSINSTANCE = int(sys.argv[idx + 1])
|
|
55
|
+
if len(lsstJobParams):
|
|
56
|
+
lsstJobParams += "|"
|
|
57
|
+
lsstJobParams += f"PIPELINE_PROCESSINSTANCE={str(PIPELINE_PROCESSINSTANCE)}"
|
|
58
|
+
except Exception:
|
|
59
|
+
PIPELINE_PROCESSINSTANCE = None
|
|
60
|
+
if argv == "-PIPELINE_EXECUTIONNUMBER":
|
|
61
|
+
try:
|
|
62
|
+
PIPELINE_EXECUTIONNUMBER = int(sys.argv[idx + 1])
|
|
63
|
+
if len(lsstJobParams):
|
|
64
|
+
lsstJobParams += "|"
|
|
65
|
+
lsstJobParams += f"PIPELINE_EXECUTIONNUMBER={str(PIPELINE_EXECUTIONNUMBER)}"
|
|
66
|
+
except Exception:
|
|
67
|
+
PIPELINE_EXECUTIONNUMBER = None
|
|
68
|
+
if argv == "-PIPELINE_STREAM":
|
|
69
|
+
try:
|
|
70
|
+
PIPELINE_STREAM = int(sys.argv[idx + 1])
|
|
71
|
+
if len(lsstJobParams):
|
|
72
|
+
lsstJobParams += "|"
|
|
73
|
+
lsstJobParams += f"PIPELINE_STREAM={str(PIPELINE_STREAM)}"
|
|
74
|
+
except Exception:
|
|
75
|
+
PIPELINE_STREAM = None
|
|
76
|
+
|
|
77
|
+
# DP_USER and PIPELINE_USER preference
|
|
78
|
+
if prodUserNameDP is not None:
|
|
79
|
+
prodUserName = prodUserNameDP
|
|
80
|
+
elif prodUserNamePipeline is not None:
|
|
81
|
+
prodUserName = prodUserNamePipeline
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# site = sys.argv[1]
|
|
85
|
+
# site = 'ANALY_BNL-LSST' #orig
|
|
86
|
+
# site = 'BNL-LSST'
|
|
87
|
+
# site = 'SWT2_CPB-LSST'
|
|
88
|
+
# site = 'UTA_SWT2-LSST'
|
|
89
|
+
# site = 'ANALY_SWT2_CPB-LSST'
|
|
90
|
+
|
|
91
|
+
destName = None
|
|
92
|
+
|
|
93
|
+
if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_PROCESSINSTANCE is not None:
|
|
94
|
+
datasetName = f"panda.lsst.user.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(prodUserName)}"
|
|
95
|
+
else:
|
|
96
|
+
datasetName = f"panda.lsst.user.jschovan.{str(uuid.uuid4())}"
|
|
97
|
+
|
|
98
|
+
if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_EXECUTIONNUMBER is not None and PIPELINE_STREAM is not None:
|
|
99
|
+
jobName = f"job.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(PIPELINE_EXECUTIONNUMBER)}.{str(prodUserName)}.{str(PIPELINE_STREAM)}"
|
|
100
|
+
else:
|
|
101
|
+
jobName = f"{str(uuid.uuid4())}"
|
|
102
|
+
|
|
103
|
+
if PIPELINE_STREAM is not None:
|
|
104
|
+
jobDefinitionID = PIPELINE_STREAM
|
|
105
|
+
else:
|
|
106
|
+
jobDefinitionID = int(time.time()) % 10000
|
|
107
|
+
job = JobSpec()
|
|
108
|
+
job.jobDefinitionID = jobDefinitionID
|
|
109
|
+
job.jobName = jobName
|
|
110
|
+
job.transformation = "http://rpm-test.pandawms.org/pandawms-jobcache/lsst-trf.sh"
|
|
111
|
+
job.destinationDBlock = datasetName
|
|
112
|
+
job.destinationSE = "local"
|
|
113
|
+
job.currentPriority = 1000
|
|
114
|
+
job.prodSourceLabel = "panda"
|
|
115
|
+
job.jobParameters = f' --lsstJobParams="{lsstJobParams}" '
|
|
116
|
+
if prodUserName is not None:
|
|
117
|
+
job.prodUserName = prodUserName
|
|
118
|
+
else:
|
|
119
|
+
job.prodUserName = prodUserNameDefault
|
|
120
|
+
if PIPELINE_PROCESSINSTANCE is not None:
|
|
121
|
+
job.taskID = PIPELINE_PROCESSINSTANCE
|
|
122
|
+
if PIPELINE_EXECUTIONNUMBER is not None:
|
|
123
|
+
job.attemptNr = PIPELINE_EXECUTIONNUMBER
|
|
124
|
+
if PIPELINE_TASK is not None:
|
|
125
|
+
job.processingType = PIPELINE_TASK
|
|
126
|
+
job.computingSite = site
|
|
127
|
+
job.VO = "lsst"
|
|
128
|
+
|
|
129
|
+
fileOL = FileSpec()
|
|
130
|
+
fileOL.lfn = f"{job.jobName}.job.log.tgz"
|
|
131
|
+
fileOL.destinationDBlock = job.destinationDBlock
|
|
132
|
+
fileOL.destinationSE = job.destinationSE
|
|
133
|
+
fileOL.dataset = job.destinationDBlock
|
|
134
|
+
fileOL.type = "log"
|
|
135
|
+
job.addFile(fileOL)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
s, o = Client.submit_jobs([job])
|
|
139
|
+
print(s)
|
|
140
|
+
for x in o:
|
|
141
|
+
print(f"PandaID={x[0]}")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import time
|
|
3
|
+
import uuid
|
|
4
|
+
|
|
5
|
+
from pandaserver.taskbuffer.FileSpec import FileSpec
|
|
6
|
+
from pandaserver.taskbuffer.JobSpec import JobSpec
|
|
7
|
+
from pandaserver.userinterface import Client
|
|
8
|
+
|
|
9
|
+
# site = sys.argv[1]
|
|
10
|
+
site = "ANALY_BNL-LSST" # orig
|
|
11
|
+
# site = 'BNL-LSST'
|
|
12
|
+
# site = 'SWT2_CPB-LSST'
|
|
13
|
+
# site = 'UTA_SWT2-LSST'
|
|
14
|
+
# site = 'ANALY_SWT2_CPB-LSST'
|
|
15
|
+
|
|
16
|
+
datasetName = f"panda.user.jschovan.lsst.{str(uuid.uuid4())}"
|
|
17
|
+
destName = None
|
|
18
|
+
|
|
19
|
+
job = JobSpec()
|
|
20
|
+
job.jobDefinitionID = int(time.time()) % 10000
|
|
21
|
+
job.jobName = f"{str(uuid.uuid4())}"
|
|
22
|
+
# job.transformation = 'http://www.usatlas.bnl.gov/~wenaus/lsst-trf/lsst-trf.sh'
|
|
23
|
+
# job.transformation = 'http://pandawms.org/pandawms-jobcache/lsst-trf.sh'
|
|
24
|
+
job.transformation = "http://pandawms.org/pandawms-jobcache/lsst-trf-phosim332.sh"
|
|
25
|
+
job.destinationDBlock = datasetName
|
|
26
|
+
# job.destinationSE = destName
|
|
27
|
+
job.destinationSE = "local"
|
|
28
|
+
job.currentPriority = 1000
|
|
29
|
+
# job.prodSourceLabel = 'ptest'
|
|
30
|
+
# job.prodSourceLabel = 'panda'
|
|
31
|
+
# job.prodSourceLabel = 'ptest'
|
|
32
|
+
# job.prodSourceLabel = 'test'
|
|
33
|
+
# job.prodSourceLabel = 'ptest'
|
|
34
|
+
# 2014-01-27
|
|
35
|
+
# job.prodSourceLabel = 'user'
|
|
36
|
+
job.prodSourceLabel = "panda"
|
|
37
|
+
job.computingSite = site
|
|
38
|
+
job.jobParameters = ""
|
|
39
|
+
job.VO = "lsst"
|
|
40
|
+
|
|
41
|
+
fileOL = FileSpec()
|
|
42
|
+
fileOL.lfn = f"{job.jobName}.job.log.tgz"
|
|
43
|
+
fileOL.destinationDBlock = job.destinationDBlock
|
|
44
|
+
fileOL.destinationSE = job.destinationSE
|
|
45
|
+
fileOL.dataset = job.destinationDBlock
|
|
46
|
+
fileOL.type = "log"
|
|
47
|
+
job.addFile(fileOL)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
s, o = Client.submit_jobs([job])
|
|
51
|
+
print(s)
|
|
52
|
+
for x in o:
|
|
53
|
+
print(f"PandaID={x[0]}")
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
REPOSITORY="https://github.com/PanDAWMS/panda-server.git"
|
|
3
|
+
BRANCH="mysql"
|
|
4
|
+
WRKDIR="/tmp/${USER}/bigpandaclient"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
mkdir -p ${WRKDIR}
|
|
8
|
+
if [ ! -d ${WRKDIR} ]; then
|
|
9
|
+
WRKDIR="/tmp"
|
|
10
|
+
fi
|
|
11
|
+
|
|
12
|
+
cd ${WRKDIR}
|
|
13
|
+
TARBALLNAMEBASE=bigpanda-client-mysql-0.0.1
|
|
14
|
+
TARBALLNAME=${TARBALLNAMEBASE}.tar.gz
|
|
15
|
+
TARBALLDIR=${WRKDIR}/${TARBALLNAMEBASE}
|
|
16
|
+
TARBALL=${WRKDIR}/${TARBALLNAME}
|
|
17
|
+
mkdir -p ${TARBALLDIR}
|
|
18
|
+
GITREPODIRNAME=panda-server
|
|
19
|
+
GITREPODIR=${WRKDIR}/${GITREPODIRNAME}
|
|
20
|
+
git clone -b ${BRANCH} ${REPOSITORY} ${GITREPODIRNAME}
|
|
21
|
+
|
|
22
|
+
cd ${GITREPODIR}/pandaserver
|
|
23
|
+
mkdir -p ${TARBALLDIR}/{taskbuffer,userinterface,test}
|
|
24
|
+
|
|
25
|
+
cp __init__.py ${TARBALLDIR}
|
|
26
|
+
cp taskbuffer/{__init__,JobSpec,FileSpec}.py ${TARBALLDIR}/taskbuffer/
|
|
27
|
+
cp userinterface/*.py ${TARBALLDIR}/userinterface/
|
|
28
|
+
cp test/lsst/*.py ${TARBALLDIR}/test/
|
|
29
|
+
cp ${GITREPODIR}/bigpanda-client/README.txt ${TARBALLDIR}/
|
|
30
|
+
|
|
31
|
+
cd ${WRKDIR}/
|
|
32
|
+
tar zcf ${TARBALL} ${TARBALLNAMEBASE}
|
|
33
|
+
|
|
34
|
+
if [ $? == 0 ]; then
|
|
35
|
+
echo "BigPanDA:MySQL client is prepared in ${TARBALL}."
|
|
36
|
+
echo "Please upload it to the web repository."
|
|
37
|
+
echo "Please clean up ${WRKDIR} afterwards."
|
|
38
|
+
fi
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from pandaserver.userinterface import Client
|
|
4
|
+
|
|
5
|
+
if len(sys.argv) == 2:
|
|
6
|
+
job_id = int(sys.argv[1])
|
|
7
|
+
ret = Client.reassign_jobs([job_id])
|
|
8
|
+
else:
|
|
9
|
+
job_id_start = int(sys.argv[1])
|
|
10
|
+
job_id_end = int(sys.argv[2])
|
|
11
|
+
if job_id_start > job_id_end:
|
|
12
|
+
print(f"{job_id_end} is less than {job_id_start}")
|
|
13
|
+
sys.exit(1)
|
|
14
|
+
|
|
15
|
+
ret = Client.reassign_jobs(range(job_id_start, job_id_end + 1))
|
|
16
|
+
|
|
17
|
+
print(ret)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import optparse
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from pandacommon.pandautils.PandaUtils import naive_utcnow
|
|
6
|
+
|
|
7
|
+
from pandaserver.config import panda_config
|
|
8
|
+
from pandaserver.taskbuffer.OraDBProxy import DBProxy
|
|
9
|
+
from pandaserver.userinterface import Client
|
|
10
|
+
|
|
11
|
+
usage = "%prog [options] siteName"
|
|
12
|
+
option_parser = optparse.OptionParser(usage=usage, conflict_handler="resolve")
|
|
13
|
+
option_parser.add_option(
|
|
14
|
+
"--assigned",
|
|
15
|
+
action="store_const",
|
|
16
|
+
const=True,
|
|
17
|
+
dest="assigned",
|
|
18
|
+
default=False,
|
|
19
|
+
help="reassign jobs in assigned state. Jobs in activated state are reassigned by default",
|
|
20
|
+
)
|
|
21
|
+
option_parser.add_option(
|
|
22
|
+
"--olderThan",
|
|
23
|
+
action="store",
|
|
24
|
+
dest="olderThan",
|
|
25
|
+
default=1,
|
|
26
|
+
help="reassign jobs with modificationTime older than N hours (1 by default)",
|
|
27
|
+
)
|
|
28
|
+
options, args = option_parser.parse_args()
|
|
29
|
+
|
|
30
|
+
# password
|
|
31
|
+
|
|
32
|
+
proxyS = DBProxy()
|
|
33
|
+
proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
|
|
34
|
+
|
|
35
|
+
site = args[0]
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
options.olderThan = int(options.olderThan)
|
|
39
|
+
except Exception:
|
|
40
|
+
pass
|
|
41
|
+
timeLimit = naive_utcnow() - datetime.timedelta(hours=options.olderThan)
|
|
42
|
+
varMap = {}
|
|
43
|
+
if options.assigned:
|
|
44
|
+
varMap[":jobStatus"] = "assigned"
|
|
45
|
+
else:
|
|
46
|
+
varMap[":jobStatus"] = "activated"
|
|
47
|
+
varMap[":modificationTime"] = timeLimit
|
|
48
|
+
varMap[":prodSourceLabel"] = "managed"
|
|
49
|
+
varMap[":computingSite"] = site
|
|
50
|
+
if options.assigned:
|
|
51
|
+
sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsDefined4 "
|
|
52
|
+
else:
|
|
53
|
+
sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsActive4 "
|
|
54
|
+
sql += (
|
|
55
|
+
"WHERE jobStatus=:jobStatus AND computingSite=:computingSite AND modificationTime<:modificationTime AND prodSourceLabel=:prodSourceLabel ORDER BY PandaID"
|
|
56
|
+
)
|
|
57
|
+
status, res = proxyS.querySQLS(sql, varMap)
|
|
58
|
+
|
|
59
|
+
print(f"got {len(res)} jobs")
|
|
60
|
+
|
|
61
|
+
jobs = []
|
|
62
|
+
jediJobs = []
|
|
63
|
+
if res is not None:
|
|
64
|
+
for id, lockedby in res:
|
|
65
|
+
if lockedby == "jedi":
|
|
66
|
+
jediJobs.append(id)
|
|
67
|
+
else:
|
|
68
|
+
jobs.append(id)
|
|
69
|
+
if len(jobs):
|
|
70
|
+
nJob = 100
|
|
71
|
+
iJob = 0
|
|
72
|
+
while iJob < len(jobs):
|
|
73
|
+
print(f"reassign {str(jobs[iJob:iJob + nJob])}")
|
|
74
|
+
Client.reassign_jobs(jobs[iJob : iJob + nJob])
|
|
75
|
+
iJob += nJob
|
|
76
|
+
if len(jediJobs) != 0:
|
|
77
|
+
nJob = 100
|
|
78
|
+
iJob = 0
|
|
79
|
+
while iJob < len(jediJobs):
|
|
80
|
+
print(f"kill JEDI jobs {str(jediJobs[iJob:iJob + nJob])}")
|
|
81
|
+
Client.kill_jobs(jediJobs[iJob : iJob + nJob], 51)
|
|
82
|
+
iJob += nJob
|