panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,154 @@
1
+ # Description: Unit tests for the Pilot API methods
2
+ import os
3
+ import unittest
4
+ from datetime import datetime, timedelta
5
+
6
+ from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
7
+
8
+
9
+ class TestPilotAPI(unittest.TestCase):
10
+ def setUp(self):
11
+ self.http_client = HttpClient()
12
+
13
+ def test_acquire_jobs(self):
14
+ url = f"{api_url_ssl}/pilot/acquire_jobs"
15
+ print(f"Testing URL: {url}")
16
+ data = {
17
+ "site_name": "CERN",
18
+ "timeout": 60,
19
+ "memory": 999999999,
20
+ "disk_space": 999999999,
21
+ "prod_source_label": "managed",
22
+ "node": "aipanda120.cern.ch",
23
+ "computing_element": "CERN",
24
+ "prod_user_id": None,
25
+ "get_proxy_key": None,
26
+ "task_id": None,
27
+ "n_jobs": 1,
28
+ "background": False,
29
+ "resource_type": "SCORE",
30
+ "harvester_id": "imaginary_harvester",
31
+ "worker_id": 12345,
32
+ "scheduler_id": "imaginary_scheduler",
33
+ "job_type": "user",
34
+ "via_topic": False,
35
+ }
36
+
37
+ status, output = self.http_client.post(url, data)
38
+ print(output)
39
+ output["status"] = status
40
+
41
+ expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
42
+ self.assertEqual(output, expected_response)
43
+
44
+ def test_acquire_jobs(self):
45
+ url = f"{api_url_ssl}/pilot/acquire_jobs"
46
+ print(f"Testing URL: {url}")
47
+ data = {
48
+ "site_name": "CERN",
49
+ "timeout": 60,
50
+ "memory": 999999999,
51
+ "disk_space": 999999999,
52
+ "prod_source_label": "managed",
53
+ "node": "aipanda120.cern.ch",
54
+ "computing_element": "CERN",
55
+ "prod_user_id": None,
56
+ "get_proxy_key": None,
57
+ "task_id": None,
58
+ "n_jobs": 1,
59
+ "background": False,
60
+ "resource_type": "SCORE",
61
+ "harvester_id": "imaginary_harvester",
62
+ "worker_id": 12345,
63
+ "scheduler_id": "imaginary_scheduler",
64
+ "job_type": "user",
65
+ "via_topic": False,
66
+ }
67
+
68
+ status, output = self.http_client.post(url, data)
69
+ print(output)
70
+ output["status"] = status
71
+
72
+ expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
73
+ self.assertEqual(output, expected_response)
74
+
75
+ def test_get_job_status(self):
76
+ url = f"{api_url_ssl}/pilot/get_job_status"
77
+ print(f"Testing URL: {url}")
78
+ data = {
79
+ "job_ids": "4674379299",
80
+ "timeout": 60,
81
+ }
82
+
83
+ status, output = self.http_client.get(url, data)
84
+ print(output)
85
+ output["status"] = status
86
+
87
+ expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
88
+ self.assertEqual(output, expected_response)
89
+
90
+ def test_update_job(self):
91
+ url = f"{api_url_ssl}/pilot/update_job"
92
+ print(f"Testing URL: {url}")
93
+ data = {"job_id": 4674379299, "job_status": "starting"}
94
+
95
+ status, output = self.http_client.post(url, data)
96
+ print(output)
97
+ output["status"] = status
98
+
99
+ expected_response = {"status": 0, "success": False, "data": 2, "message": ""}
100
+ self.assertEqual(output, expected_response)
101
+
102
+ def test_update_worker_node(self):
103
+ url = f"{api_url_ssl}/pilot/update_worker_node"
104
+ print(f"Testing URL: {url}")
105
+ data = {
106
+ "site": "CERN",
107
+ "panda_queue": "CERN",
108
+ "host_name": "slot1@wn1.cern.ch",
109
+ "cpu_model": "AMD EPYC 7B12",
110
+ "n_logical_cpus": 64,
111
+ "n_sockets": 2,
112
+ "cores_per_socket": 8,
113
+ "threads_per_core": 2,
114
+ "cpu_architecture": "x86_64",
115
+ "cpu_architecture_level": "x86_64-v3",
116
+ "clock_speed": 2.7,
117
+ "total_memory": 3350,
118
+ "total_local_disk": 75,
119
+ }
120
+
121
+ status, output = self.http_client.post(url, data)
122
+ print(output)
123
+ output["status"] = status
124
+
125
+ expected_response = {"status": 0, "success": True, "data": None, "message": "Inserted new worker node."}
126
+ self.assertEqual(output, expected_response)
127
+
128
+ def test_update_worker_node_gpu(self):
129
+ url = f"{api_url_ssl}/pilot/update_worker_node_gpu"
130
+ print(f"Testing URL: {url}")
131
+ data = {
132
+ "site": "CERN",
133
+ "host_name": "slot1@lxplus940.cern.ch",
134
+ "vendor": "NVIDIA",
135
+ "model": "Tesla T4",
136
+ "architecture": "Turing",
137
+ "vram": 15360,
138
+ "framework": "CUDA",
139
+ "framework_version": "12.9",
140
+ "driver_version": "575.51.03",
141
+ "count": 1,
142
+ }
143
+
144
+ status, output = self.http_client.post(url, data)
145
+ print(output)
146
+ output["status"] = status
147
+
148
+ expected_response = {"status": 0, "success": True, "data": None, "message": "Inserted new worker node GPU."}
149
+ self.assertEqual(output, expected_response)
150
+
151
+
152
+ # Run tests
153
+ if __name__ == "__main__":
154
+ unittest.main()
@@ -0,0 +1,104 @@
1
+ # Description: Unit tests for the Statistics API methods
2
+ import unittest
3
+
4
+ from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
5
+
6
+
7
+ class TestTaskAPI(unittest.TestCase):
8
+ def setUp(self):
9
+ self.http_client = HttpClient()
10
+
11
+ def test_job_stats_by_cloud(self):
12
+ url = f"{api_url_ssl}/statistics/job_stats_by_cloud"
13
+ print(f"Testing URL: {url}")
14
+ data = {"type": "production"}
15
+ status, output = self.http_client.get(url, data)
16
+ print(output)
17
+ output["status"] = status
18
+
19
+ # Remove the data field from the response for comparison
20
+ del output["data"]
21
+
22
+ expected_response = {"success": True, "message": "", "status": 0}
23
+
24
+ self.assertEqual(output, expected_response)
25
+
26
+ def test_production_job_stats_by_cloud_and_processing_type(self):
27
+ url = f"{api_url_ssl}/statistics/production_job_stats_by_cloud_and_processing_type"
28
+ print(f"Testing URL: {url}")
29
+ data = {}
30
+ status, output = self.http_client.get(url, data)
31
+ print(output)
32
+ output["status"] = status
33
+
34
+ # Remove the data field from the response for comparison
35
+ del output["data"]
36
+
37
+ expected_response = {"success": True, "message": "", "status": 0}
38
+
39
+ self.assertEqual(output, expected_response)
40
+
41
+ def test_active_job_stats_by_site(self):
42
+ url = f"{api_url_ssl}/statistics/active_job_stats_by_site"
43
+ print(f"Testing URL: {url}")
44
+ data = {}
45
+ status, output = self.http_client.get(url, data)
46
+ print(output)
47
+ output["status"] = status
48
+
49
+ # Remove the data field from the response for comparison
50
+ del output["data"]
51
+
52
+ expected_response = {"success": True, "message": "", "status": 0}
53
+
54
+ self.assertEqual(output, expected_response)
55
+
56
+ def test_active_job_detailed_stats_by_site(self):
57
+ url = f"{api_url_ssl}/statistics/active_job_detailed_stats_by_site"
58
+ print(f"Testing URL: {url}")
59
+ data = {}
60
+ status, output = self.http_client.get(url, data)
61
+ print(output)
62
+ output["status"] = status
63
+
64
+ # Remove the data field from the response for comparison
65
+ del output["data"]
66
+
67
+ expected_response = {"success": True, "message": "", "status": 0}
68
+
69
+ self.assertEqual(output, expected_response)
70
+
71
+ def test_job_stats_by_site_and_resource_type(self):
72
+ url = f"{api_url_ssl}/statistics/job_stats_by_site_and_resource_type"
73
+ print(f"Testing URL: {url}")
74
+ data = {"time_window": 12 * 60}
75
+ status, output = self.http_client.get(url, data)
76
+ print(output)
77
+ output["status"] = status
78
+
79
+ # Remove the data field from the response for comparison
80
+ del output["data"]
81
+
82
+ expected_response = {"success": True, "message": "", "status": 0}
83
+
84
+ self.assertEqual(output, expected_response)
85
+
86
+ def test_job_stats_by_site_share_and_resource_type(self):
87
+ url = f"{api_url_ssl}/statistics/job_stats_by_site_share_and_resource_type"
88
+ print(f"Testing URL: {url}")
89
+ data = {"time_window": 12 * 60}
90
+ status, output = self.http_client.get(url, data)
91
+ print(output)
92
+ output["status"] = status
93
+
94
+ # Remove the data field from the response for comparison
95
+ del output["data"]
96
+
97
+ expected_response = {"success": True, "message": "", "status": 0}
98
+
99
+ self.assertEqual(output, expected_response)
100
+
101
+
102
+ # Run tests
103
+ if __name__ == "__main__":
104
+ unittest.main()
@@ -0,0 +1,58 @@
1
+ # Description: Unit tests for the System API methods
2
+ import unittest
3
+
4
+ from pandaserver.api.v1.http_client import HttpClient, api_url_ssl
5
+
6
+
7
+ class TestSystemAPI(unittest.TestCase):
8
+ def setUp(self):
9
+ self.http_client = HttpClient()
10
+
11
+ def test_get_attributes(self):
12
+ url = f"{api_url_ssl}/system/get_attributes"
13
+ print(f"Testing URL: {url}")
14
+ data = {"test_key_string": "test_value", "test_key_int": 12345}
15
+ status, output = self.http_client.get(url, data)
16
+ print(output)
17
+ output["status"] = status
18
+
19
+ expected_response = {"status": 0, "success": True}
20
+ self.assertEqual(output, expected_response)
21
+
22
+ def test_get_user_attributes(self):
23
+ url = f"{api_url_ssl}/system/get_user_attributes"
24
+ print(f"Testing URL: {url}")
25
+ data = {}
26
+ status, output = self.http_client.get(url, data)
27
+ print(output)
28
+ output["status"] = status
29
+
30
+ expected_response = {"status": 0, "success": True}
31
+ self.assertEqual(output, expected_response)
32
+
33
+ def test_get_voms_attributes(self):
34
+ url = f"{api_url_ssl}/system/get_voms_attributes"
35
+ print(f"Testing URL: {url}")
36
+ data = {}
37
+ status, output = self.http_client.get(url, data)
38
+ print(output)
39
+ output["status"] = status
40
+
41
+ expected_response = {"status": 0, "success": True}
42
+ self.assertEqual(output, expected_response)
43
+
44
+ def test_is_alive(self):
45
+ url = f"{api_url_ssl}/system/is_alive"
46
+ print(f"Testing URL: {url}")
47
+ data = {}
48
+ status, output = self.http_client.get(url, data)
49
+ print(output)
50
+ output["status"] = status
51
+
52
+ expected_response = {"status": 0, "success": True}
53
+ self.assertEqual(output, expected_response)
54
+
55
+
56
+ # Run tests
57
+ if __name__ == "__main__":
58
+ unittest.main()