panda-server-source 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panda_server_source-0.8.0.dist-info/METADATA +55 -0
- panda_server_source-0.8.0.dist-info/RECORD +375 -0
- panda_server_source-0.8.0.dist-info/WHEEL +4 -0
- panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
- pandajedi/__init__.py +0 -0
- pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
- pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
- pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
- pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
- pandajedi/jedibrokerage/GenJobBroker.py +381 -0
- pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
- pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
- pandajedi/jedibrokerage/__init__.py +0 -0
- pandajedi/jediconfig/__init__.py +0 -0
- pandajedi/jediconfig/jedi_config.py +40 -0
- pandajedi/jedicore/FactoryBase.py +182 -0
- pandajedi/jedicore/Interaction.py +433 -0
- pandajedi/jedicore/JediDBProxy.py +56 -0
- pandajedi/jedicore/JediDBProxyPool.py +37 -0
- pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
- pandajedi/jedicore/JediException.py +13 -0
- pandajedi/jedicore/JediTaskBuffer.py +984 -0
- pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
- pandajedi/jedicore/MsgWrapper.py +124 -0
- pandajedi/jedicore/ProcessUtils.py +34 -0
- pandajedi/jedicore/SiteCandidate.py +90 -0
- pandajedi/jedicore/ThreadUtils.py +205 -0
- pandajedi/jedicore/__init__.py +0 -0
- pandajedi/jediddm/AtlasDDMClient.py +1708 -0
- pandajedi/jediddm/DDMClientBase.py +16 -0
- pandajedi/jediddm/DDMInterface.py +83 -0
- pandajedi/jediddm/GenDDMClient.py +17 -0
- pandajedi/jediddm/__init__.py +0 -0
- pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
- pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
- pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
- pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
- pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
- pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
- pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
- pandajedi/jedidog/GenWatchDog.py +21 -0
- pandajedi/jedidog/JumboWatchDog.py +147 -0
- pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
- pandajedi/jedidog/WatchDogBase.py +53 -0
- pandajedi/jedidog/__init__.py +0 -0
- pandajedi/jediexec/reniceJEDI.py +25 -0
- pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
- pandajedi/jedigen/TaskGeneratorBase.py +15 -0
- pandajedi/jedigen/__init__.py +0 -0
- pandajedi/jedimsgprocessor/__init__.py +0 -0
- pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
- pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
- pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
- pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
- pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
- pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
- pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
- pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
- pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
- pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
- pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
- pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
- pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
- pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
- pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
- pandajedi/jediorder/ContentsFeeder.py +785 -0
- pandajedi/jediorder/JediDaemon.py +27 -0
- pandajedi/jediorder/JediKnight.py +55 -0
- pandajedi/jediorder/JediMaster.py +238 -0
- pandajedi/jediorder/JediMsgProcessor.py +32 -0
- pandajedi/jediorder/JobBroker.py +33 -0
- pandajedi/jediorder/JobGenerator.py +2763 -0
- pandajedi/jediorder/JobSplitter.py +308 -0
- pandajedi/jediorder/JobThrottler.py +28 -0
- pandajedi/jediorder/PostProcessor.py +166 -0
- pandajedi/jediorder/TaskBroker.py +268 -0
- pandajedi/jediorder/TaskCommando.py +334 -0
- pandajedi/jediorder/TaskGenerator.py +17 -0
- pandajedi/jediorder/TaskRefiner.py +611 -0
- pandajedi/jediorder/TaskSetupper.py +17 -0
- pandajedi/jediorder/WatchDog.py +79 -0
- pandajedi/jediorder/__init__.py +0 -0
- pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
- pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
- pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
- pandajedi/jedipprocess/GenPostProcessor.py +35 -0
- pandajedi/jedipprocess/MailTemplates.py +185 -0
- pandajedi/jedipprocess/PostProcessorBase.py +269 -0
- pandajedi/jedipprocess/__init__.py +0 -0
- pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
- pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
- pandajedi/jedirefine/GenTaskRefiner.py +61 -0
- pandajedi/jedirefine/RefinerUtils.py +154 -0
- pandajedi/jedirefine/TaskRefinerBase.py +996 -0
- pandajedi/jedirefine/__init__.py +0 -0
- pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
- pandajedi/jedisetup/GenTaskSetupper.py +16 -0
- pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
- pandajedi/jedisetup/TaskSetupperBase.py +15 -0
- pandajedi/jedisetup/__init__.py +0 -0
- pandajedi/jeditest/SchemaChecker.py +41 -0
- pandajedi/jeditest/addContWoSoTask.py +54 -0
- pandajedi/jeditest/addDataCarouselRequest.py +47 -0
- pandajedi/jeditest/addEventServiceTask.py +101 -0
- pandajedi/jeditest/addHpoTask.py +81 -0
- pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
- pandajedi/jeditest/addNonAtlasTask.py +71 -0
- pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
- pandajedi/jeditest/addTestTask.py +124 -0
- pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
- pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
- pandajedi/jeditest/brokerTest.py +78 -0
- pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
- pandajedi/jeditest/contentsFeederTest.py +28 -0
- pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
- pandajedi/jeditest/ddmTest.py +4 -0
- pandajedi/jeditest/jobGenForOneTask.py +38 -0
- pandajedi/jeditest/jobGeneratorTest.py +43 -0
- pandajedi/jeditest/jobSplitterTest.py +104 -0
- pandajedi/jeditest/jobThrottlerTest.py +22 -0
- pandajedi/jeditest/msgProcessorTest.py +9 -0
- pandajedi/jeditest/postProcessorTest.py +18 -0
- pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
- pandajedi/jeditest/resubmitTaskParam.py +20 -0
- pandajedi/jeditest/taskBrokerTest.py +22 -0
- pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
- pandajedi/jeditest/taskBufferTest.py +5 -0
- pandajedi/jeditest/taskCommandoTest.py +28 -0
- pandajedi/jeditest/taskRefinerTest.py +30 -0
- pandajedi/jeditest/testOneTaskToRefine.py +49 -0
- pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
- pandajedi/jeditest/testPandaDDMMap.py +62 -0
- pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
- pandajedi/jeditest/watchDogTest.py +25 -0
- pandajedi/jeditest/workQueueTest.py +22 -0
- pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
- pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
- pandajedi/jedithrottle/GenJobThrottler.py +26 -0
- pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
- pandajedi/jedithrottle/__init__.py +0 -0
- pandaserver/__init__.py +0 -0
- pandaserver/api/__init__.py +0 -0
- pandaserver/api/v1/__init__.py +0 -0
- pandaserver/api/v1/common.py +339 -0
- pandaserver/api/v1/credential_management_api.py +372 -0
- pandaserver/api/v1/data_carousel_api.py +264 -0
- pandaserver/api/v1/event_api.py +269 -0
- pandaserver/api/v1/file_server_api.py +942 -0
- pandaserver/api/v1/harvester_api.py +445 -0
- pandaserver/api/v1/http_client.py +166 -0
- pandaserver/api/v1/idds_api.py +180 -0
- pandaserver/api/v1/job_api.py +463 -0
- pandaserver/api/v1/metaconfig_api.py +122 -0
- pandaserver/api/v1/pilot_api.py +895 -0
- pandaserver/api/v1/statistics_api.py +207 -0
- pandaserver/api/v1/system_api.py +186 -0
- pandaserver/api/v1/task_api.py +1520 -0
- pandaserver/api/v1/tests/__init__.py +0 -0
- pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
- pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
- pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
- pandaserver/api/v1/tests/idds_api_tests.py +45 -0
- pandaserver/api/v1/tests/job_api_tests.py +112 -0
- pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
- pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
- pandaserver/api/v1/tests/system_api_tests.py +58 -0
- pandaserver/api/v1/tests/task_api_tests.py +590 -0
- pandaserver/api/v1/version.py +2 -0
- pandaserver/api/v1/workflow_api.py +136 -0
- pandaserver/brokerage/SiteMapper.py +406 -0
- pandaserver/brokerage/__init__.py +0 -0
- pandaserver/brokerage/broker.py +272 -0
- pandaserver/config/__init__.py +0 -0
- pandaserver/config/config_utils.py +15 -0
- pandaserver/config/daemon_config.py +29 -0
- pandaserver/config/mcp_config.py +38 -0
- pandaserver/config/panda_config.py +233 -0
- pandaserver/configurator/Carbon.py +65 -0
- pandaserver/configurator/Configurator.py +908 -0
- pandaserver/configurator/__init__.py +0 -0
- pandaserver/configurator/aux.py +73 -0
- pandaserver/daemons/__init__.py +0 -0
- pandaserver/daemons/master.py +110 -0
- pandaserver/daemons/master_systemd.py +79 -0
- pandaserver/daemons/scripts/__init__.py +0 -0
- pandaserver/daemons/scripts/add_main.py +264 -0
- pandaserver/daemons/scripts/add_sub.py +221 -0
- pandaserver/daemons/scripts/cache_pilots.py +47 -0
- pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
- pandaserver/daemons/scripts/carbon.py +50 -0
- pandaserver/daemons/scripts/configurator.py +94 -0
- pandaserver/daemons/scripts/copyArchive.py +1387 -0
- pandaserver/daemons/scripts/datasetManager.py +1154 -0
- pandaserver/daemons/scripts/dummy_test.py +25 -0
- pandaserver/daemons/scripts/evpPD2P.py +144 -0
- pandaserver/daemons/scripts/hs_scrapers.py +264 -0
- pandaserver/daemons/scripts/metric_collector.py +927 -0
- pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
- pandaserver/daemons/scripts/pilotStreaming.py +109 -0
- pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
- pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
- pandaserver/daemons/scripts/task_evaluator.py +405 -0
- pandaserver/daemons/scripts/tmpwatch.py +70 -0
- pandaserver/daemons/scripts/worker_synchronization.py +97 -0
- pandaserver/daemons/utils.py +753 -0
- pandaserver/dataservice/DataServiceUtils.py +323 -0
- pandaserver/dataservice/ErrorCode.py +25 -0
- pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
- pandaserver/dataservice/__init__.py +0 -0
- pandaserver/dataservice/activator.py +79 -0
- pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
- pandaserver/dataservice/adder_dummy_plugin.py +31 -0
- pandaserver/dataservice/adder_gen.py +954 -0
- pandaserver/dataservice/adder_plugin_base.py +27 -0
- pandaserver/dataservice/adder_result.py +75 -0
- pandaserver/dataservice/adder_simple_plugin.py +160 -0
- pandaserver/dataservice/closer.py +247 -0
- pandaserver/dataservice/closer_atlas_plugin.py +90 -0
- pandaserver/dataservice/ddm.py +1381 -0
- pandaserver/dataservice/ddm_handler.py +95 -0
- pandaserver/dataservice/dyn_data_distributer.py +824 -0
- pandaserver/dataservice/event_lookup_client_ei.py +77 -0
- pandaserver/dataservice/event_picker.py +268 -0
- pandaserver/dataservice/finisher.py +194 -0
- pandaserver/dataservice/setupper.py +225 -0
- pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
- pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
- pandaserver/dataservice/setupper_plugin_base.py +67 -0
- pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
- pandaserver/jobdispatcher/ErrorCode.py +19 -0
- pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
- pandaserver/jobdispatcher/Protocol.py +443 -0
- pandaserver/jobdispatcher/Watcher.py +186 -0
- pandaserver/jobdispatcher/__init__.py +0 -0
- pandaserver/pandamcp/__init__.py +0 -0
- pandaserver/pandamcp/mcp_main.py +56 -0
- pandaserver/pandamcp/mcp_test_client.py +95 -0
- pandaserver/pandamcp/mcp_utils.py +74 -0
- pandaserver/proxycache/__init__.py +0 -0
- pandaserver/proxycache/panda_proxy_cache.py +240 -0
- pandaserver/proxycache/token_cache.py +141 -0
- pandaserver/server/.gacl +6 -0
- pandaserver/server/panda.py +539 -0
- pandaserver/srvcore/CoreUtils.py +572 -0
- pandaserver/srvcore/MailUtils.py +106 -0
- pandaserver/srvcore/__init__.py +0 -0
- pandaserver/srvcore/allowed_methods.py +100 -0
- pandaserver/srvcore/oidc_utils.py +152 -0
- pandaserver/srvcore/panda_request.py +196 -0
- pandaserver/srvcore/srv_msg_utils.py +42 -0
- pandaserver/taskbuffer/ConBridge.py +513 -0
- pandaserver/taskbuffer/DBProxyPool.py +95 -0
- pandaserver/taskbuffer/DataCarousel.py +3053 -0
- pandaserver/taskbuffer/DatasetSpec.py +137 -0
- pandaserver/taskbuffer/DdmSpec.py +201 -0
- pandaserver/taskbuffer/ErrorCode.py +87 -0
- pandaserver/taskbuffer/EventServiceUtils.py +396 -0
- pandaserver/taskbuffer/FileSpec.py +290 -0
- pandaserver/taskbuffer/GlobalShares.py +244 -0
- pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
- pandaserver/taskbuffer/Initializer.py +82 -0
- pandaserver/taskbuffer/InputChunk.py +1080 -0
- pandaserver/taskbuffer/JediCacheSpec.py +57 -0
- pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
- pandaserver/taskbuffer/JediFileSpec.py +242 -0
- pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
- pandaserver/taskbuffer/JobSpec.py +1006 -0
- pandaserver/taskbuffer/JobUtils.py +232 -0
- pandaserver/taskbuffer/NucleusSpec.py +116 -0
- pandaserver/taskbuffer/OraDBProxy.py +174 -0
- pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
- pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
- pandaserver/taskbuffer/ParseJobXML.py +350 -0
- pandaserver/taskbuffer/PickleFileSpec.py +18 -0
- pandaserver/taskbuffer/PickleJobSpec.py +25 -0
- pandaserver/taskbuffer/PrioUtil.py +40 -0
- pandaserver/taskbuffer/ProcessGroups.py +111 -0
- pandaserver/taskbuffer/ResourceSpec.py +175 -0
- pandaserver/taskbuffer/SQLDumper.py +37 -0
- pandaserver/taskbuffer/SQLManager.py +60 -0
- pandaserver/taskbuffer/SiteSpec.py +304 -0
- pandaserver/taskbuffer/SupErrors.py +10 -0
- pandaserver/taskbuffer/TaskBuffer.py +2882 -0
- pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
- pandaserver/taskbuffer/Utils.py +848 -0
- pandaserver/taskbuffer/WorkQueue.py +227 -0
- pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
- pandaserver/taskbuffer/WorkerSpec.py +143 -0
- pandaserver/taskbuffer/WrappedCursor.py +487 -0
- pandaserver/taskbuffer/WrappedPickle.py +88 -0
- pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
- pandaserver/taskbuffer/__init__.py +0 -0
- pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
- pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
- pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
- pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
- pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
- pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
- pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
- pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
- pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
- pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
- pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
- pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
- pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
- pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
- pandaserver/taskbuffer/retryModule.py +552 -0
- pandaserver/taskbuffer/task_split_rules.py +308 -0
- pandaserver/taskbuffer/workflow_processor.py +249 -0
- pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
- pandaserver/test/SchemaChecker.py +40 -0
- pandaserver/test/__init__.py +0 -0
- pandaserver/test/banUser.py +43 -0
- pandaserver/test/boostPrio.py +19 -0
- pandaserver/test/boostUser.py +35 -0
- pandaserver/test/callbackDDM.py +11 -0
- pandaserver/test/daod_on_demand.py +95 -0
- pandaserver/test/finishJob.py +184 -0
- pandaserver/test/finishTaskJEDI.py +11 -0
- pandaserver/test/getJobs.py +53 -0
- pandaserver/test/killJob.py +79 -0
- pandaserver/test/killJobLowPrio.py +99 -0
- pandaserver/test/killJobsInTask.py +56 -0
- pandaserver/test/killTask.py +80 -0
- pandaserver/test/killTaskJEDI.py +11 -0
- pandaserver/test/killUser.py +83 -0
- pandaserver/test/lsst/README.txt +7 -0
- pandaserver/test/lsst/lsstSubmit.py +139 -0
- pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
- pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
- pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
- pandaserver/test/reassignJobs.py +17 -0
- pandaserver/test/reassignSite.py +82 -0
- pandaserver/test/reassignTask.py +117 -0
- pandaserver/test/reassignWaiting.py +39 -0
- pandaserver/test/recoverLostFiles.py +7 -0
- pandaserver/test/reloadInputDS.py +51 -0
- pandaserver/test/sendCommandToJob.py +26 -0
- pandaserver/test/setDebugMode.py +38 -0
- pandaserver/test/setPriority.py +28 -0
- pandaserver/test/testEvgen.py +60 -0
- pandaserver/test/testEvgen17.py +58 -0
- pandaserver/test/testG4sim.py +81 -0
- pandaserver/test/testG4sim17.py +90 -0
- pandaserver/test/testGlobalShares.py +131 -0
- pandaserver/test/testReco.py +103 -0
- pandaserver/test/testSimulReco14.py +98 -0
- pandaserver/test/testSiteMap.py +26 -0
- pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
- pandaserver/userinterface/Client.py +1686 -0
- pandaserver/userinterface/UserIF.py +1799 -0
- pandaserver/userinterface/__init__.py +0 -0
- pandaserver/workflow/__init__.py +0 -0
- pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
- pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
- pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
- pandaserver/workflow/pcwl_test.py +122 -0
- pandaserver/workflow/pcwl_utils.py +384 -0
- pandaserver/workflow/psnakemake_container.json +68 -0
- pandaserver/workflow/psnakemake_task.json +62 -0
- pandaserver/workflow/psnakemake_test.py +60 -0
- pandaserver/workflow/snakeparser/__init__.py +3 -0
- pandaserver/workflow/snakeparser/extensions.py +74 -0
- pandaserver/workflow/snakeparser/log.py +53 -0
- pandaserver/workflow/snakeparser/names.py +3 -0
- pandaserver/workflow/snakeparser/parser.py +332 -0
- pandaserver/workflow/snakeparser/utils.py +29 -0
- pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
- pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
- pandaserver/workflow/workflow_base.py +531 -0
- pandaserver/workflow/workflow_core.py +1974 -0
- pandaserver/workflow/workflow_core_smoketest.py +388 -0
- pandaserver/workflow/workflow_parser.py +222 -0
- pandaserver/workflow/workflow_utils.py +1047 -0
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# Description: Unit tests for the Pilot API methods
|
|
2
|
+
import os
|
|
3
|
+
import unittest
|
|
4
|
+
from datetime import datetime, timedelta
|
|
5
|
+
|
|
6
|
+
from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TestPilotAPI(unittest.TestCase):
|
|
10
|
+
def setUp(self):
|
|
11
|
+
self.http_client = HttpClient()
|
|
12
|
+
|
|
13
|
+
def test_acquire_jobs(self):
|
|
14
|
+
url = f"{api_url_ssl}/pilot/acquire_jobs"
|
|
15
|
+
print(f"Testing URL: {url}")
|
|
16
|
+
data = {
|
|
17
|
+
"site_name": "CERN",
|
|
18
|
+
"timeout": 60,
|
|
19
|
+
"memory": 999999999,
|
|
20
|
+
"disk_space": 999999999,
|
|
21
|
+
"prod_source_label": "managed",
|
|
22
|
+
"node": "aipanda120.cern.ch",
|
|
23
|
+
"computing_element": "CERN",
|
|
24
|
+
"prod_user_id": None,
|
|
25
|
+
"get_proxy_key": None,
|
|
26
|
+
"task_id": None,
|
|
27
|
+
"n_jobs": 1,
|
|
28
|
+
"background": False,
|
|
29
|
+
"resource_type": "SCORE",
|
|
30
|
+
"harvester_id": "imaginary_harvester",
|
|
31
|
+
"worker_id": 12345,
|
|
32
|
+
"scheduler_id": "imaginary_scheduler",
|
|
33
|
+
"job_type": "user",
|
|
34
|
+
"via_topic": False,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
status, output = self.http_client.post(url, data)
|
|
38
|
+
print(output)
|
|
39
|
+
output["status"] = status
|
|
40
|
+
|
|
41
|
+
expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
|
|
42
|
+
self.assertEqual(output, expected_response)
|
|
43
|
+
|
|
44
|
+
def test_acquire_jobs(self):
|
|
45
|
+
url = f"{api_url_ssl}/pilot/acquire_jobs"
|
|
46
|
+
print(f"Testing URL: {url}")
|
|
47
|
+
data = {
|
|
48
|
+
"site_name": "CERN",
|
|
49
|
+
"timeout": 60,
|
|
50
|
+
"memory": 999999999,
|
|
51
|
+
"disk_space": 999999999,
|
|
52
|
+
"prod_source_label": "managed",
|
|
53
|
+
"node": "aipanda120.cern.ch",
|
|
54
|
+
"computing_element": "CERN",
|
|
55
|
+
"prod_user_id": None,
|
|
56
|
+
"get_proxy_key": None,
|
|
57
|
+
"task_id": None,
|
|
58
|
+
"n_jobs": 1,
|
|
59
|
+
"background": False,
|
|
60
|
+
"resource_type": "SCORE",
|
|
61
|
+
"harvester_id": "imaginary_harvester",
|
|
62
|
+
"worker_id": 12345,
|
|
63
|
+
"scheduler_id": "imaginary_scheduler",
|
|
64
|
+
"job_type": "user",
|
|
65
|
+
"via_topic": False,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
status, output = self.http_client.post(url, data)
|
|
69
|
+
print(output)
|
|
70
|
+
output["status"] = status
|
|
71
|
+
|
|
72
|
+
expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
|
|
73
|
+
self.assertEqual(output, expected_response)
|
|
74
|
+
|
|
75
|
+
def test_get_job_status(self):
|
|
76
|
+
url = f"{api_url_ssl}/pilot/get_job_status"
|
|
77
|
+
print(f"Testing URL: {url}")
|
|
78
|
+
data = {
|
|
79
|
+
"job_ids": "4674379299",
|
|
80
|
+
"timeout": 60,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
status, output = self.http_client.get(url, data)
|
|
84
|
+
print(output)
|
|
85
|
+
output["status"] = status
|
|
86
|
+
|
|
87
|
+
expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
|
|
88
|
+
self.assertEqual(output, expected_response)
|
|
89
|
+
|
|
90
|
+
def test_update_job(self):
|
|
91
|
+
url = f"{api_url_ssl}/pilot/update_job"
|
|
92
|
+
print(f"Testing URL: {url}")
|
|
93
|
+
data = {"job_id": 4674379299, "job_status": "starting"}
|
|
94
|
+
|
|
95
|
+
status, output = self.http_client.post(url, data)
|
|
96
|
+
print(output)
|
|
97
|
+
output["status"] = status
|
|
98
|
+
|
|
99
|
+
expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
|
|
100
|
+
self.assertEqual(output, expected_response)
|
|
101
|
+
|
|
102
|
+
def test_update_worker_node(self):
|
|
103
|
+
url = f"{api_url_ssl}/pilot/update_worker_node"
|
|
104
|
+
print(f"Testing URL: {url}")
|
|
105
|
+
data = {
|
|
106
|
+
"site": "CERN",
|
|
107
|
+
"panda_queue": "CERN",
|
|
108
|
+
"host_name": "slot1@wn1.cern.ch",
|
|
109
|
+
"cpu_model": "AMD EPYC 7B12",
|
|
110
|
+
"n_logical_cpus": 64,
|
|
111
|
+
"n_sockets": 2,
|
|
112
|
+
"cores_per_socket": 8,
|
|
113
|
+
"threads_per_core": 2,
|
|
114
|
+
"cpu_architecture": "x86_64",
|
|
115
|
+
"cpu_architecture_level": "x86_64-v3",
|
|
116
|
+
"clock_speed": 2.7,
|
|
117
|
+
"total_memory": 3350,
|
|
118
|
+
"total_local_disk": 75,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
status, output = self.http_client.post(url, data)
|
|
122
|
+
print(output)
|
|
123
|
+
output["status"] = status
|
|
124
|
+
|
|
125
|
+
expected_response = {"status": 0, "success": True, "data": None, "message": "Inserted new worker node."}
|
|
126
|
+
self.assertEqual(output, expected_response)
|
|
127
|
+
|
|
128
|
+
def test_update_worker_node_gpu(self):
|
|
129
|
+
url = f"{api_url_ssl}/pilot/update_worker_node_gpu"
|
|
130
|
+
print(f"Testing URL: {url}")
|
|
131
|
+
data = {
|
|
132
|
+
"site": "CERN",
|
|
133
|
+
"host_name": "slot1@lxplus940.cern.ch",
|
|
134
|
+
"vendor": "NVIDIA",
|
|
135
|
+
"model": "Tesla T4",
|
|
136
|
+
"architecture": "Turing",
|
|
137
|
+
"vram": 15360,
|
|
138
|
+
"framework": "CUDA",
|
|
139
|
+
"framework_version": "12.9",
|
|
140
|
+
"driver_version": "575.51.03",
|
|
141
|
+
"count": 1,
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
status, output = self.http_client.post(url, data)
|
|
145
|
+
print(output)
|
|
146
|
+
output["status"] = status
|
|
147
|
+
|
|
148
|
+
expected_response = {"status": 0, "success": True, "data": None, "message": "Inserted new worker node GPU."}
|
|
149
|
+
self.assertEqual(output, expected_response)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Run tests
|
|
153
|
+
if __name__ == "__main__":
|
|
154
|
+
unittest.main()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Description: Unit tests for the Statistics API methods
|
|
2
|
+
import unittest
|
|
3
|
+
|
|
4
|
+
from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestTaskAPI(unittest.TestCase):
|
|
8
|
+
def setUp(self):
|
|
9
|
+
self.http_client = HttpClient()
|
|
10
|
+
|
|
11
|
+
def test_job_stats_by_cloud(self):
|
|
12
|
+
url = f"{api_url_ssl}/statistics/job_stats_by_cloud"
|
|
13
|
+
print(f"Testing URL: {url}")
|
|
14
|
+
data = {"type": "production"}
|
|
15
|
+
status, output = self.http_client.get(url, data)
|
|
16
|
+
print(output)
|
|
17
|
+
output["status"] = status
|
|
18
|
+
|
|
19
|
+
# Remove the data field from the response for comparison
|
|
20
|
+
del output["data"]
|
|
21
|
+
|
|
22
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
23
|
+
|
|
24
|
+
self.assertEqual(output, expected_response)
|
|
25
|
+
|
|
26
|
+
def test_production_job_stats_by_cloud_and_processing_type(self):
|
|
27
|
+
url = f"{api_url_ssl}/statistics/production_job_stats_by_cloud_and_processing_type"
|
|
28
|
+
print(f"Testing URL: {url}")
|
|
29
|
+
data = {}
|
|
30
|
+
status, output = self.http_client.get(url, data)
|
|
31
|
+
print(output)
|
|
32
|
+
output["status"] = status
|
|
33
|
+
|
|
34
|
+
# Remove the data field from the response for comparison
|
|
35
|
+
del output["data"]
|
|
36
|
+
|
|
37
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
38
|
+
|
|
39
|
+
self.assertEqual(output, expected_response)
|
|
40
|
+
|
|
41
|
+
def test_active_job_stats_by_site(self):
|
|
42
|
+
url = f"{api_url_ssl}/statistics/active_job_stats_by_site"
|
|
43
|
+
print(f"Testing URL: {url}")
|
|
44
|
+
data = {}
|
|
45
|
+
status, output = self.http_client.get(url, data)
|
|
46
|
+
print(output)
|
|
47
|
+
output["status"] = status
|
|
48
|
+
|
|
49
|
+
# Remove the data field from the response for comparison
|
|
50
|
+
del output["data"]
|
|
51
|
+
|
|
52
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
53
|
+
|
|
54
|
+
self.assertEqual(output, expected_response)
|
|
55
|
+
|
|
56
|
+
def test_active_job_detailed_stats_by_site(self):
|
|
57
|
+
url = f"{api_url_ssl}/statistics/active_job_detailed_stats_by_site"
|
|
58
|
+
print(f"Testing URL: {url}")
|
|
59
|
+
data = {}
|
|
60
|
+
status, output = self.http_client.get(url, data)
|
|
61
|
+
print(output)
|
|
62
|
+
output["status"] = status
|
|
63
|
+
|
|
64
|
+
# Remove the data field from the response for comparison
|
|
65
|
+
del output["data"]
|
|
66
|
+
|
|
67
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
68
|
+
|
|
69
|
+
self.assertEqual(output, expected_response)
|
|
70
|
+
|
|
71
|
+
def test_job_stats_by_site_and_resource_type(self):
|
|
72
|
+
url = f"{api_url_ssl}/statistics/job_stats_by_site_and_resource_type"
|
|
73
|
+
print(f"Testing URL: {url}")
|
|
74
|
+
data = {"time_window": 12 * 60}
|
|
75
|
+
status, output = self.http_client.get(url, data)
|
|
76
|
+
print(output)
|
|
77
|
+
output["status"] = status
|
|
78
|
+
|
|
79
|
+
# Remove the data field from the response for comparison
|
|
80
|
+
del output["data"]
|
|
81
|
+
|
|
82
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
83
|
+
|
|
84
|
+
self.assertEqual(output, expected_response)
|
|
85
|
+
|
|
86
|
+
def test_job_stats_by_site_share_and_resource_type(self):
|
|
87
|
+
url = f"{api_url_ssl}/statistics/job_stats_by_site_share_and_resource_type"
|
|
88
|
+
print(f"Testing URL: {url}")
|
|
89
|
+
data = {"time_window": 12 * 60}
|
|
90
|
+
status, output = self.http_client.get(url, data)
|
|
91
|
+
print(output)
|
|
92
|
+
output["status"] = status
|
|
93
|
+
|
|
94
|
+
# Remove the data field from the response for comparison
|
|
95
|
+
del output["data"]
|
|
96
|
+
|
|
97
|
+
expected_response = {"success": True, "message": "", "status": 0}
|
|
98
|
+
|
|
99
|
+
self.assertEqual(output, expected_response)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# Run tests
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
unittest.main()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Description: Unit tests for the System API methods
|
|
2
|
+
import unittest
|
|
3
|
+
|
|
4
|
+
from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestSystemAPI(unittest.TestCase):
|
|
8
|
+
def setUp(self):
|
|
9
|
+
self.http_client = HttpClient()
|
|
10
|
+
|
|
11
|
+
def test_get_attributes(self):
|
|
12
|
+
url = f"{api_url_ssl}/system/get_attributes"
|
|
13
|
+
print(f"Testing URL: {url}")
|
|
14
|
+
data = {"test_key_string": "test_value", "test_key_int": 12345}
|
|
15
|
+
status, output = self.http_client.get(url, data)
|
|
16
|
+
print(output)
|
|
17
|
+
output["status"] = status
|
|
18
|
+
|
|
19
|
+
expected_response = {"status": 0, "success": True}
|
|
20
|
+
self.assertEqual(output, expected_response)
|
|
21
|
+
|
|
22
|
+
def test_get_user_attributes(self):
|
|
23
|
+
url = f"{api_url_ssl}/system/get_user_attributes"
|
|
24
|
+
print(f"Testing URL: {url}")
|
|
25
|
+
data = {}
|
|
26
|
+
status, output = self.http_client.get(url, data)
|
|
27
|
+
print(output)
|
|
28
|
+
output["status"] = status
|
|
29
|
+
|
|
30
|
+
expected_response = {"status": 0, "success": True}
|
|
31
|
+
self.assertEqual(output, expected_response)
|
|
32
|
+
|
|
33
|
+
def test_get_voms_attributes(self):
|
|
34
|
+
url = f"{api_url_ssl}/system/get_voms_attributes"
|
|
35
|
+
print(f"Testing URL: {url}")
|
|
36
|
+
data = {}
|
|
37
|
+
status, output = self.http_client.get(url, data)
|
|
38
|
+
print(output)
|
|
39
|
+
output["status"] = status
|
|
40
|
+
|
|
41
|
+
expected_response = {"status": 0, "success": True}
|
|
42
|
+
self.assertEqual(output, expected_response)
|
|
43
|
+
|
|
44
|
+
def test_is_alive(self):
|
|
45
|
+
url = f"{api_url_ssl}/system/is_alive"
|
|
46
|
+
print(f"Testing URL: {url}")
|
|
47
|
+
data = {}
|
|
48
|
+
status, output = self.http_client.get(url, data)
|
|
49
|
+
print(output)
|
|
50
|
+
output["status"] = status
|
|
51
|
+
|
|
52
|
+
expected_response = {"status": 0, "success": True}
|
|
53
|
+
self.assertEqual(output, expected_response)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Run tests
|
|
57
|
+
if __name__ == "__main__":
|
|
58
|
+
unittest.main()
|