panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,372 @@
1
+ import json
2
+ import os
3
+ import socket
4
+ from threading import Lock
5
+ from typing import List
6
+
7
+ from pandacommon.pandalogger.LogWrapper import LogWrapper
8
+ from pandacommon.pandalogger.PandaLogger import PandaLogger
9
+ from pandacommon.pandautils.PandaUtils import naive_utcnow
10
+
11
+ from pandaserver.api.v1.common import generate_response, get_dn, request_validation
12
+ from pandaserver.config import panda_config
13
+ from pandaserver.proxycache import panda_proxy_cache, token_cache
14
+ from pandaserver.srvcore import CoreUtils
15
+ from pandaserver.srvcore.CoreUtils import clean_user_id, get_bare_dn
16
+ from pandaserver.srvcore.panda_request import PandaRequest
17
+ from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
18
+
19
+ _logger = PandaLogger().getLogger("api_cred_management")
20
+
21
+ # These global variables are initialized in the init_task_buffer method
22
+ global_task_buffer = None
23
+ global_dispatch_parameter_cache = None
24
+
25
+ # These global variables don't depend on DB access and can be initialized here
26
+ global_proxy_cache = panda_proxy_cache.MyProxyInterface()
27
+ global_token_cache = token_cache.TokenCache()
28
+
29
+ global_lock = Lock()
30
+
31
+
32
+ def init_task_buffer(task_buffer: TaskBuffer) -> None:
33
+ """
34
+ Initialize the task buffer. This method needs to be called before any other method in this module.
35
+ """
36
+
37
+ with global_lock:
38
+ global global_task_buffer
39
+ global_task_buffer = task_buffer
40
+
41
+ # This variable depends on having an initialized task buffer
42
+ global global_dispatch_parameter_cache
43
+ global_dispatch_parameter_cache = CoreUtils.CachedObject("dispatcher_params", 60 * 10, _get_dispatch_parameters, _logger)
44
+
45
+ global global_token_cache_config
46
+ global_token_cache_config = _read_token_cache_configuration()
47
+
48
+
49
+ def _read_token_cache_configuration():
50
+ # config of token cacher
51
+ try:
52
+ with open(panda_config.token_cache_config) as f:
53
+ return json.load(f)
54
+ except Exception:
55
+ return {}
56
+
57
+
58
+ def _get_dispatch_parameters():
59
+ """
60
+ Wrapper function around taskBuffer.get_special_dispatch_params to convert list to set since task buffer cannot return set
61
+ """
62
+ parameters = global_task_buffer.get_special_dispatch_params()
63
+ for client_name, client_data in parameters["tokenKeys"].items():
64
+ client_data["fullList"] = set(client_data["fullList"])
65
+
66
+ return True, parameters
67
+
68
+
69
+ def _validate_user_permissions(compact_name, tokenized=False) -> dict:
70
+ allowed_names = global_dispatch_parameter_cache.get("allowProxy", [])
71
+
72
+ # The user is allowed to get a proxy or token
73
+ if compact_name in allowed_names:
74
+ return True, ""
75
+
76
+ # Permission denied for user
77
+ secret = "proxy" if not tokenized else "access token"
78
+ tmp_msg = f"failed: '{compact_name}' not in authorized users with 'p' in {panda_config.schemaMETA}.USERS.GRIDPREF to get {secret}"
79
+
80
+ return False, tmp_msg
81
+
82
+
83
+ @request_validation(_logger, secure=True, request_method="POST")
84
+ def set_user_secrets(req: PandaRequest, key: str = None, value: str = None) -> dict:
85
+ """
86
+ Set user secrets
87
+
88
+ Set a key-value pair to store in PanDA. Requires a secure connection.
89
+
90
+ API details:
91
+ HTTP Method: POST
92
+ Path: /v1/creds/set_user_secrets
93
+
94
+ Args:
95
+ req(PandaRequest): internally generated request object
96
+ key(str): Key to reference the secret
97
+ value(str): Value of the secret
98
+
99
+ Returns:
100
+ dict: The system response. True for success, False for failure, and an error message.
101
+ """
102
+
103
+ tmp_logger = LogWrapper(_logger, f"set_user_secret-{naive_utcnow().isoformat('/')}")
104
+
105
+ # Get client DN and clean it. The request validation has already verified that the DN is present.
106
+ dn = get_dn(req)
107
+ owner = clean_user_id(dn)
108
+ tmp_logger.debug(f"Setting secret for client={owner}")
109
+ success, message = global_task_buffer.set_user_secret(owner, key, value)
110
+ return generate_response(success, message)
111
+
112
+
113
+ @request_validation(_logger, secure=True, request_method="GET")
114
+ def get_user_secrets(req: PandaRequest, keys: List[str] = None) -> dict:
115
+ """
116
+ Get user secrets
117
+
118
+ Get the secrets for a user identified by a list of keys. Requires a secure connection.
119
+
120
+ API details:
121
+ HTTP Method: POST
122
+ Path: /v1/creds/get_user_secrets
123
+
124
+ Args:
125
+ req(PandaRequest): internally generated request object
126
+ keys(list of str, optional): List of keys to reference the secrets to retrieve
127
+
128
+ Returns:
129
+ dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the user secrets. When unsuccessful, the message field contains the error message.
130
+ """
131
+ tmp_logger = LogWrapper(_logger, f"get_user_secrets-{naive_utcnow().isoformat('/')}")
132
+
133
+ # Get client DN and clean it. The request validation has already verified that the DN is present.
134
+ dn = get_dn(req)
135
+ owner = clean_user_id(dn)
136
+ tmp_logger.debug(f"Start for client={owner}")
137
+
138
+ # The task buffer method expects a comma-separated string of keys
139
+ if keys is None:
140
+ keys_str = ""
141
+ else:
142
+ keys_str = ",".join(keys)
143
+
144
+ success, data_or_message = global_task_buffer.get_user_secrets(owner, keys_str)
145
+ message, data = "", {}
146
+ if success:
147
+ data = data_or_message
148
+ else:
149
+ message = data_or_message
150
+ tmp_logger.debug(f"Done for client={owner}")
151
+ return generate_response(success, message, data)
152
+
153
+
154
+ @request_validation(_logger, secure=True, request_method="GET")
155
+ def get_key_pair(req: PandaRequest, public_key_name: str, private_key_name: str) -> dict:
156
+ """
157
+ Get key pair
158
+
159
+ This function retrieves the distinguished name (DN) from the request and uses it to get a key pair. Requires a secure connection.
160
+
161
+ API details:
162
+ HTTP Method: GET
163
+ Path: /v1/creds/get_key_pair
164
+
165
+ Args:
166
+ req(PandaRequest): internally generated request object
167
+ public_key_name(str): The name of the public key.
168
+ private_key_name(str): The name of the private key.
169
+
170
+ Returns:
171
+ dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the user secrets. When unsuccessful, the message field contains the error message.
172
+ """
173
+ tmp_logger = LogWrapper(_logger, f"get_key_pair {public_key_name}/{private_key_name}")
174
+ tmp_logger.debug("Start")
175
+
176
+ real_dn = get_dn(req)
177
+
178
+ # get compact DN
179
+ compact_dn = clean_user_id(real_dn)
180
+
181
+ # check permission
182
+ global_dispatch_parameter_cache.update()
183
+ allowed_keys = global_dispatch_parameter_cache.get("allowKeyPair", [])
184
+ if compact_dn not in allowed_keys:
185
+ # permission denied
186
+ tmp_msg = f"Failed since '{compact_dn}' not authorized with 'k' in {panda_config.schemaMETA}.USERS.GRIDPREF"
187
+ tmp_logger.debug(tmp_msg)
188
+ return generate_response(False, tmp_msg)
189
+
190
+ # retrieve the key pairs
191
+ key_pair_dict = global_dispatch_parameter_cache.get("keyPair", {})
192
+
193
+ if public_key_name not in key_pair_dict: # public key is missing
194
+ tmp_msg = f"Failed for '{compact_dn}' since {public_key_name} is missing on {socket.getfqdn()}"
195
+ tmp_logger.debug(tmp_msg)
196
+ return generate_response(False, tmp_msg)
197
+
198
+ if private_key_name not in key_pair_dict: # private key is missing
199
+ tmp_msg = f"Failed for '{compact_dn}' since {private_key_name} is missing on {socket.getfqdn()}"
200
+ tmp_logger.debug(tmp_msg)
201
+ return generate_response(False, tmp_msg)
202
+
203
+ # key pair is available
204
+ data = {"public_key": key_pair_dict[public_key_name], "private_key": key_pair_dict[private_key_name]}
205
+ tmp_logger.debug(f"Done with key-pair for '{compact_dn}'")
206
+
207
+ return generate_response(True, data=data)
208
+
209
+
210
+ @request_validation(_logger, secure=True, request_method="GET")
211
+ def get_proxy(req: PandaRequest, role: str = None, dn: str = None) -> dict:
212
+ """
213
+ Get proxy
214
+
215
+ Get the x509 proxy certificate for a user with a role. Requires a secure connection.
216
+
217
+ API details:
218
+ HTTP Method: GET
219
+ Path: /v1/creds/get_proxy
220
+
221
+ Args:
222
+ req(PandaRequest): internally generated request object
223
+ role(str, optional): The role of the user. Defaults to None.
224
+ dn(str, optional): The distinguished name of the user. Defaults to None.
225
+
226
+ Returns:
227
+ dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the x509 proxy. When unsuccessful, the message field contains the error message.
228
+ """
229
+ tmp_logger = LogWrapper(_logger, f"get_proxy PID={os.getpid()}")
230
+
231
+ # Set the requested DN or, if not set, the requestor DN
232
+ real_dn = get_dn(req)
233
+ target_dn = dn if dn else real_dn
234
+
235
+ # Default roles
236
+ if role == "":
237
+ role = None
238
+
239
+ tmp_logger.debug(f"Start for DN='{real_dn}' role={role}")
240
+
241
+ # check permission
242
+ global_dispatch_parameter_cache.update()
243
+ real_dn_compact = clean_user_id(real_dn)
244
+ allowed, tmp_msg = _validate_user_permissions(real_dn_compact, tokenized=False)
245
+ if not allowed:
246
+ tmp_logger.debug(tmp_msg)
247
+ return generate_response(False, tmp_msg)
248
+
249
+ # remove redundant extensions and retrieve the proxy
250
+ target_dn_bare = get_bare_dn(target_dn, keep_digits=False)
251
+ output = global_proxy_cache.retrieve(target_dn_bare, role=role)
252
+
253
+ # proxy not found
254
+ if output is None:
255
+ tmp_msg = f"'proxy' not found for {target_dn_bare}"
256
+ tmp_logger.debug(tmp_msg)
257
+ return generate_response(False, tmp_msg)
258
+
259
+ data = {"user_proxy": output}
260
+ tmp_logger.debug(f"Done")
261
+ return generate_response(True, data=data)
262
+
263
+
264
+ @request_validation(_logger, secure=True, request_method="GET")
265
+ def get_access_token(req: PandaRequest, client_name: str, token_key: str = None) -> dict:
266
+ """
267
+ Get access token
268
+
269
+ Get the OAuth access token for the specified client. Requires a secure connection.
270
+
271
+ API details:
272
+ HTTP Method: GET
273
+ Path: /v1/creds/get_access_token
274
+
275
+ Args:
276
+ req(PandaRequest): internally generated request object
277
+ client_name(str): client_name for the token as defined in token_cache_config
278
+ token_key(str, optional): key to get the token from the token cache. Defaults to None.
279
+
280
+ Returns:
281
+ dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the access token. When unsuccessful, the message field contains the error message.
282
+ """
283
+
284
+ tmp_logger = LogWrapper(_logger, f"get_proxy PID={os.getpid()}")
285
+
286
+ # Set the requested DN or, if not set, the requestor DN
287
+ real_dn = get_dn(req)
288
+ target_dn = client_name if client_name else real_dn
289
+
290
+ tmp_logger.debug(f"Start for DN='{real_dn}' target_dn='{target_dn}' token_key={token_key}")
291
+
292
+ # check permission
293
+ global_dispatch_parameter_cache.update()
294
+ real_dn_compact = clean_user_id(real_dn)
295
+ allowed, tmp_msg = _validate_user_permissions(real_dn_compact, tokenized=True)
296
+ if not allowed:
297
+ tmp_logger.debug(tmp_msg)
298
+ return generate_response(False, tmp_msg)
299
+
300
+ # check if the token key is valid
301
+ if target_dn in global_token_cache_config and global_token_cache_config[target_dn].get("use_token_key") is True:
302
+ if (
303
+ target_dn not in global_dispatch_parameter_cache["tokenKeys"]
304
+ or token_key not in global_dispatch_parameter_cache["tokenKeys"][target_dn]["fullList"]
305
+ ):
306
+ tmp_msg = f"failed since token key is invalid for {target_dn}"
307
+ tmp_logger.debug(tmp_msg)
308
+ return generate_response(False, tmp_msg)
309
+
310
+ # remove redundant extensions
311
+ target_dn_bare = get_bare_dn(target_dn, keep_digits=False)
312
+
313
+ # get token
314
+ output = global_token_cache.get_access_token(target_dn_bare)
315
+
316
+ # access token not found
317
+ if output is None:
318
+ tmp_msg = f"'token' not found for {target_dn_bare}"
319
+ tmp_logger.debug(tmp_msg)
320
+ return generate_response(False, tmp_msg)
321
+
322
+ data = {"user_proxy": output}
323
+ tmp_logger.debug("Done")
324
+ return generate_response(True, data=data)
325
+
326
+
327
+ @request_validation(_logger, secure=True, request_method="GET")
328
+ def get_token_key(req: PandaRequest, client_name: str) -> dict:
329
+ """
330
+ Get token key
331
+
332
+ This function retrieves the distinguished name (DN) from the request and uses it to get a token key for the specified client. Requires a secure connection.
333
+
334
+ API details:
335
+ HTTP Method: GET
336
+ Path: /v1/creds/get_token_key
337
+
338
+ Args:
339
+ req(PandaRequest): internally generated request object
340
+ client_name (str): The name of the client requesting the token key
341
+
342
+ Returns:
343
+ dict: The system response `{"success": success, "message": message, "data": data}`. When successful, the data field contains the token key. When unsuccessful, the message field contains the error message.
344
+ """
345
+ tmp_logger = LogWrapper(_logger, f"get_token_key client={client_name} PID={os.getpid()}")
346
+ tmp_logger.debug("Start")
347
+
348
+ # get DN and clean it
349
+ real_dn = get_dn(req)
350
+ compact_name = clean_user_id(real_dn)
351
+
352
+ global_dispatch_parameter_cache.update()
353
+
354
+ # check if user is allowed to retrieved token keys
355
+ allowed_users = global_dispatch_parameter_cache.get("allowTokenKey", [])
356
+ if compact_name not in allowed_users:
357
+ # permission denied
358
+ tmp_msg = f"Denied since '{compact_name}' not authorized with 't' in {panda_config.schemaMETA}.USERS.GRIDPREF"
359
+ tmp_logger.debug(tmp_msg)
360
+ return generate_response(False, tmp_msg)
361
+
362
+ # token key is missing
363
+ if client_name not in global_dispatch_parameter_cache["tokenKeys"]:
364
+ tmp_msg = f"Token key is missing for '{client_name}'"
365
+ tmp_logger.debug(tmp_msg)
366
+ return generate_response(False, tmp_msg)
367
+
368
+ # send token key
369
+ data = {"tokenKey": global_dispatch_parameter_cache["tokenKeys"][client_name]["latest"]}
370
+ tmp_logger.debug(f"Done getting token key for '{compact_name}'")
371
+
372
+ return generate_response(True, data=data)
@@ -0,0 +1,264 @@
1
+ import datetime
2
+ from concurrent.futures import ThreadPoolExecutor
3
+ from threading import Lock
4
+ from typing import Any, Dict, List
5
+
6
+ from pandacommon.pandalogger.LogWrapper import LogWrapper
7
+ from pandacommon.pandalogger.PandaLogger import PandaLogger
8
+ from pandacommon.pandautils.PandaUtils import naive_utcnow
9
+
10
+ from pandaserver.api.v1.common import (
11
+ MESSAGE_DATABASE,
12
+ TIME_OUT,
13
+ TimedMethod,
14
+ generate_response,
15
+ get_dn,
16
+ request_validation,
17
+ )
18
+ from pandaserver.srvcore.panda_request import PandaRequest
19
+ from pandaserver.taskbuffer.DataCarousel import (
20
+ DataCarouselInterface,
21
+ DataCarouselRequestStatus,
22
+ )
23
+ from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
24
+
25
+ _logger = PandaLogger().getLogger("api_data_carousel")
26
+
27
+ # These global variables are initialized in the init_task_buffer method
28
+ global_task_buffer = None
29
+ global_dcif = None
30
+
31
+ # These global variables don't depend on DB access and can be initialized here
32
+ # global_proxy_cache = panda_proxy_cache.MyProxyInterface()
33
+ # global_token_cache = token_cache.TokenCache()
34
+
35
+
36
+ def init_task_buffer(task_buffer: TaskBuffer) -> None:
37
+ """
38
+ Initialize the task buffer and other interfaces. This method needs to be called before any other method in this module.
39
+ """
40
+ global global_task_buffer
41
+ global_task_buffer = task_buffer
42
+
43
+ global global_dcif
44
+ global_dcif = DataCarouselInterface(global_task_buffer)
45
+
46
+
47
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
48
+ def change_staging_destination(req: PandaRequest, request_id: int | None = None, dataset: str | None = None) -> dict:
49
+ """
50
+ Change destination of staging
51
+
52
+ The current active staging request will be cancelled, and a new request will be created with the newly selected destination RSE, excluding the original destination.
53
+ The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
54
+ Requires a secure connection production role.
55
+
56
+ API details:
57
+ HTTP Method: POST
58
+ Path: /v1/data_carousel/change_staging_destination
59
+
60
+ Args:
61
+ req(PandaRequest): internally generated request object
62
+ request_id (int|None): request_id of the staging request, e.g. `123`
63
+ dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
64
+
65
+ Returns:
66
+ dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
67
+ """
68
+ tmp_logger = LogWrapper(_logger, f"change_staging_destination request_id={request_id} dataset={dataset}")
69
+ tmp_logger.debug("Start")
70
+ success, message, data = False, "", None
71
+ dc_req_spec_resubmitted = None
72
+ to_submit_idds = False
73
+ time_start = naive_utcnow()
74
+
75
+ dc_req_spec = None
76
+ if request_id is not None:
77
+ # specified by request_id
78
+ dc_req_spec = global_dcif.get_request_by_id(request_id)
79
+ elif request_id is None and dataset is not None:
80
+ # specified by dataset
81
+ dc_req_spec = global_dcif.get_request_by_dataset(dataset)
82
+
83
+ if dc_req_spec is not None:
84
+ dc_req_spec_resubmitted, err_msg = global_dcif.resubmit_request(dc_req_spec, submit_idds_request=False, exclude_prev_dst=True)
85
+ if not dc_req_spec_resubmitted or err_msg:
86
+ err_msg = f"failed to resubmit request_id={dc_req_spec.request_id} : {err_msg}"
87
+ tmp_logger.error(err_msg)
88
+ success, message = False, err_msg
89
+ else:
90
+ to_submit_idds = True
91
+ else:
92
+ err_msg = f"failed to get corresponding request"
93
+ tmp_logger.error(err_msg)
94
+ success, message = False, err_msg
95
+
96
+ if dc_req_spec_resubmitted and dc_req_spec_resubmitted.status == DataCarouselRequestStatus.staging:
97
+ success = True
98
+ data = {"request_id": dc_req_spec.request_id, "new_request_id": dc_req_spec_resubmitted.request_id, "dataset": dc_req_spec_resubmitted.dataset}
99
+ message = "new request resubmitted, destination changed"
100
+ if to_submit_idds:
101
+ new_request_id = dc_req_spec_resubmitted.request_id
102
+ task_id_list = global_dcif._get_related_tasks(new_request_id)
103
+ if task_id_list:
104
+ tmp_logger.debug(f"related tasks: {task_id_list}")
105
+ with ThreadPoolExecutor() as thread_pool:
106
+ thread_pool.map((lambda task_id: global_dcif._submit_idds_stagein_request(task_id, dc_req_spec_resubmitted)), task_id_list)
107
+ tmp_logger.debug(f"submitted corresponding iDDS requests for related tasks")
108
+ message += "; submitted iDDS requests"
109
+
110
+ else:
111
+ err_msg = f"failed to get related tasks; skipped to submit iDDS requests"
112
+ tmp_logger.warning(err_msg)
113
+ message += f"; {err_msg}"
114
+
115
+ time_delta = naive_utcnow() - time_start
116
+ tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
117
+
118
+ return generate_response(success, message, data)
119
+
120
+
121
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
122
+ def change_staging_source(
123
+ req: PandaRequest,
124
+ request_id: int | None = None,
125
+ dataset: str | None = None,
126
+ cancel_fts: bool = False,
127
+ change_src_expr: bool = False,
128
+ source_rse: str | None = None,
129
+ ) -> dict:
130
+ """
131
+ Change source of staging
132
+
133
+ If the request is queued, its source_rse will be rechosen, excluding the original source.
134
+ If the request is staging, the source_replica_expression of its DDM rule is unset so new source can be tried.
135
+ Only effective on queued or staging requests.
136
+ The requests can be specified by request_id or dataset (if both exist, request_id is taken).
137
+ Requires a secure connection production role.
138
+
139
+ API details:
140
+ HTTP Method: POST
141
+ Path: /v1/data_carousel/change_staging_source
142
+
143
+ Args:
144
+ req(PandaRequest): internally generated request object
145
+ request_id (int|None): request_id of the staging request, e.g. `123`
146
+ dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
147
+ cancel_fts (bool): whether to cancel current FTS requests on DDM, False by default
148
+ change_src_expr (bool): whether to change source_replica_expression of the DDM rule by replacing old source with new one, instead of just dropping old source
149
+ source_rse (str|None): if set, use this source RSE instead of choosing one randomly, also force change_src_expr to be True; default is None
150
+
151
+ Returns:
152
+ dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
153
+ """
154
+ tmp_logger = LogWrapper(
155
+ _logger,
156
+ f"change_staging_source request_id={request_id} dataset={dataset} cancel_fts={cancel_fts} change_src_expr={change_src_expr} source_rse={source_rse}",
157
+ )
158
+ tmp_logger.debug("Start")
159
+ success, message, data = False, "", None
160
+ time_start = naive_utcnow()
161
+
162
+ dc_req_spec = None
163
+ if request_id is not None:
164
+ # specified by request_id
165
+ dc_req_spec = global_dcif.get_request_by_id(request_id)
166
+ elif request_id is None and dataset is not None:
167
+ # specified by dataset
168
+ dc_req_spec = global_dcif.get_request_by_dataset(dataset)
169
+
170
+ if dc_req_spec is not None:
171
+ status = dc_req_spec.status
172
+ orig_source_rse = dc_req_spec.source_rse
173
+ if status not in [DataCarouselRequestStatus.queued, DataCarouselRequestStatus.staging]:
174
+ err_msg = f"request_id={dc_req_spec.request_id} status={status} not queued or staging; skipped"
175
+ tmp_logger.warning(err_msg)
176
+ success, message = False, err_msg
177
+ else:
178
+ ret, dc_req_spec, err_msg = global_dcif.change_request_source_rse(dc_req_spec, cancel_fts, change_src_expr, source_rse)
179
+ if not ret:
180
+ err_msg = f"failed to change source request_id={dc_req_spec.request_id} : {err_msg}"
181
+ tmp_logger.error(err_msg)
182
+ success, message = False, err_msg
183
+ else:
184
+ success = True
185
+ if dc_req_spec.status == DataCarouselRequestStatus.queued or change_src_expr:
186
+ message = f"status={status} changed source_rse from {orig_source_rse} to {dc_req_spec.source_rse}"
187
+ else:
188
+ message = f"status={status} source replica expression is dropped"
189
+ data = {
190
+ "request_id": dc_req_spec.request_id,
191
+ "dataset": dc_req_spec.dataset,
192
+ "source_rse": dc_req_spec.source_rse,
193
+ "ddm_rule_id": dc_req_spec.ddm_rule_id,
194
+ }
195
+ else:
196
+ err_msg = f"failed to get corresponding request"
197
+ tmp_logger.error(err_msg)
198
+ success, message = False, err_msg
199
+
200
+ time_delta = naive_utcnow() - time_start
201
+ tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
202
+
203
+ return generate_response(success, message, data)
204
+
205
+
206
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
207
+ def force_to_staging(req: PandaRequest, request_id: int | None = None, dataset: str | None = None) -> dict:
208
+ """
209
+ Force to staging
210
+
211
+ The request will skip the queue and go to staging immediately (will submit DDM rules).
212
+ Only effective on queued requests.
213
+ The reqeusts can be specified by request_id or dataset (if both exist, request_id is taken).
214
+ Requires a secure connection production role.
215
+
216
+ API details:
217
+ HTTP Method: POST
218
+ Path: /v1/data_carousel/force_to_staging
219
+
220
+ Args:
221
+ req(PandaRequest): internally generated request object
222
+ request_id (int|None): request_id of the staging request, e.g. `123`
223
+ dataset (str|None): dataset name of the staging request in the format of Rucio DID, e.g. `"mc20_13TeV:mc20_13TeV.700449.Sh_2211_Wtaunu_mW_120_ECMS_BFilter.merge.AOD.e8351_s3681_r13144_r13146_tid36179107_00"`
224
+
225
+ Returns:
226
+ dict: dictionary `{'success': True/False, 'message': 'Description of error', 'data': <requested data>}`
227
+ """
228
+ tmp_logger = LogWrapper(_logger, f"force_to_staging request_id={request_id} dataset={dataset}")
229
+ tmp_logger.debug("Start")
230
+ success, message, data = False, "", None
231
+ time_start = naive_utcnow()
232
+
233
+ dc_req_spec = None
234
+ if request_id is not None:
235
+ # specified by request_id
236
+ dc_req_spec = global_dcif.get_request_by_id(request_id)
237
+ elif request_id is None and dataset is not None:
238
+ # specified by dataset
239
+ dc_req_spec = global_dcif.get_request_by_dataset(dataset)
240
+
241
+ if dc_req_spec is not None:
242
+ is_ok, err_msg, dc_req_spec = global_dcif.stage_request(dc_req_spec)
243
+ if not is_ok:
244
+ err_msg = f"failed to stage request_id={dc_req_spec.request_id} : {err_msg}"
245
+ tmp_logger.error(err_msg)
246
+ success, message = False, err_msg
247
+ else:
248
+ success = True
249
+ message = f"status has become {dc_req_spec.status}"
250
+ data = {
251
+ "request_id": dc_req_spec.request_id,
252
+ "dataset": dc_req_spec.dataset,
253
+ "status": dc_req_spec.status,
254
+ "ddm_rule_id": dc_req_spec.ddm_rule_id,
255
+ }
256
+ else:
257
+ err_msg = f"failed to get corresponding request"
258
+ tmp_logger.error(err_msg)
259
+ success, message = False, err_msg
260
+
261
+ time_delta = naive_utcnow() - time_start
262
+ tmp_logger.debug(f"Done. Took {time_delta.seconds}.{time_delta.microseconds // 1000:03d} sec")
263
+
264
+ return generate_response(success, message, data)