panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,1520 @@
1
+ # See the task state diagram for a better understanding of some of the actions https://panda-wms.readthedocs.io/en/latest/terminology/terminology.html#task
2
+
3
+ import datetime
4
+ import json
5
+ import re
6
+ from typing import Any, Dict, List
7
+
8
+ from pandacommon.pandalogger.LogWrapper import LogWrapper
9
+ from pandacommon.pandalogger.PandaLogger import PandaLogger
10
+ from pandacommon.pandautils.PandaUtils import naive_utcnow
11
+
12
+ from pandaserver.api.v1.common import (
13
+ MESSAGE_TASK_ID,
14
+ extract_production_working_groups,
15
+ generate_response,
16
+ get_dn,
17
+ get_email_address,
18
+ get_fqan,
19
+ has_production_role,
20
+ request_validation,
21
+ )
22
+ from pandaserver.srvcore.CoreUtils import clean_user_id, make_reassign_comment
23
+ from pandaserver.srvcore.panda_request import PandaRequest
24
+ from pandaserver.taskbuffer import task_split_rules
25
+ from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
26
+ from pandaserver.taskbuffer.TaskBuffer import TaskBuffer
27
+
28
+ _logger = PandaLogger().getLogger("api_task")
29
+
30
+ global_task_buffer = None
31
+
32
+
33
+ def init_task_buffer(task_buffer: TaskBuffer) -> None:
34
+ """
35
+ Initialize the task buffer. This method needs to be called before any other method in this module.
36
+ """
37
+ global global_task_buffer
38
+ global_task_buffer = task_buffer
39
+
40
+
41
+ @request_validation(_logger, secure=True, request_method="POST")
42
+ def retry(
43
+ req: PandaRequest,
44
+ task_id: int,
45
+ new_parameters: Dict | str = None,
46
+ no_child_retry: bool = False,
47
+ discard_events: bool = False,
48
+ disable_staging_mode: bool = False,
49
+ keep_gshare_priority: bool = False,
50
+ ignore_hard_exhausted: bool = False,
51
+ ) -> Dict[str, Any]:
52
+ """
53
+ Task retry
54
+
55
+ Retry a given task e.g. in exhausted state. Requires a secure connection without a production role to retry own tasks and with a production role to retry others' tasks.
56
+
57
+ API details:
58
+ HTTP Method: POST
59
+ Path: /v1/task/retry
60
+
61
+ Args:
62
+ req(PandaRequest): internally generated request object
63
+ task_id(int): JEDI Task ID
64
+ new_parameters(Dict, optional): a dictionary with the new parameters for rerunning the task. The new parameters are merged with the existing ones.
65
+ The parameters are the attributes in the JediTaskSpec object (https://github.com/PanDAWMS/panda-jedi/blob/master/pandajedi/jedicore/JediTaskSpec.py).
66
+ no_child_retry(bool, optional): if True, the child tasks are not retried. Defaults to False
67
+ discard_events(bool, optional): if True, events will be discarded. Defaults to False
68
+ disable_staging_mode(bool, optional): if True, the task skips staging state and directly goes to subsequent state. Defaults to False
69
+ keep_gshare_priority(bool, optional): if True, the task keeps current gshare and priority. Defaults to False
70
+ ignore_hard_exhausted(bool, optional): if True, the task ignores the limits for hard exhausted state and can be retried even if it is very faulty. Defaults to False
71
+
72
+ Returns:
73
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
74
+ """
75
+ tmp_logger = LogWrapper(_logger, f"retry < task_id={task_id} >")
76
+ tmp_logger.debug("Start")
77
+
78
+ user = get_dn(req)
79
+ production_role = has_production_role(req)
80
+
81
+ # retry with new parameters
82
+ if new_parameters:
83
+ if isinstance(new_parameters, str):
84
+ new_parameters = json.loads(new_parameters)
85
+
86
+ try:
87
+ # get original parameters
88
+ old_parameters_json = global_task_buffer.getTaskParamsPanda(task_id)
89
+ old_parameters = json.loads(old_parameters_json)
90
+
91
+ # update with new values
92
+ old_parameters.update(new_parameters)
93
+ final_task_parameters_json = json.dumps(old_parameters)
94
+
95
+ # retry with new parameters
96
+ ret = global_task_buffer.insertTaskParamsPanda(
97
+ final_task_parameters_json,
98
+ user,
99
+ production_role,
100
+ [],
101
+ properErrorCode=True,
102
+ allowActiveTask=True,
103
+ )
104
+ except Exception as e:
105
+ ret = 1, f"new parameter conversion failed with {str(e)}"
106
+ else:
107
+ # disable ignore_hard_exhausted if not production_role
108
+ if not production_role and ignore_hard_exhausted:
109
+ ignore_hard_exhausted = False
110
+ # get command qualifier
111
+ qualifier = JediTaskSpec.get_retry_command_qualifiers(no_child_retry, discard_events, disable_staging_mode, keep_gshare_priority, ignore_hard_exhausted)
112
+ qualifier = " ".join(qualifier)
113
+ # normal retry
114
+ ret = global_task_buffer.sendCommandTaskPanda(
115
+ task_id,
116
+ user,
117
+ production_role,
118
+ "retry",
119
+ properErrorCode=True,
120
+ comQualifier=qualifier,
121
+ )
122
+
123
+ if ret[0] == 5:
124
+ # retry failed analysis jobs
125
+ job_ids = global_task_buffer.getJobdefIDsForFailedJob(task_id)
126
+ clean_id = clean_user_id(user)
127
+ for job_id in job_ids:
128
+ global_task_buffer.finalizePendingJobs(clean_id, job_id)
129
+ global_task_buffer.increaseAttemptNrPanda(task_id, 5)
130
+ return_str = f"retry has been triggered for failed jobs while the task is still {ret[1]}"
131
+ if not new_parameters:
132
+ ret = 0, return_str
133
+ else:
134
+ ret = 3, return_str
135
+
136
+ tmp_logger.debug("Done")
137
+
138
+ data, message = ret
139
+ success = True
140
+ return generate_response(success, message, data)
141
+
142
+
143
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
144
+ def resume(req: PandaRequest, task_id: int) -> Dict[str, Any]:
145
+ """
146
+ Task resume
147
+
148
+ Resume a given task. This transitions a paused or throttled task back to its previous active state. Resume can also be used to kick a task in staging state to the next state.
149
+ Requires a secure connection and production role.
150
+
151
+ API details:
152
+ HTTP Method: POST
153
+ Path: /v1/task/resume
154
+
155
+ Args:
156
+ req(PandaRequest): internally generated request object
157
+ task_id(int): JEDI Task ID
158
+
159
+ Returns:
160
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
161
+ """
162
+ tmp_logger = LogWrapper(_logger, f"resume < jediTaskID={task_id} >")
163
+ tmp_logger.debug("Start")
164
+
165
+ user = get_dn(req)
166
+ is_production_role = has_production_role(req)
167
+
168
+ # check task_id
169
+ try:
170
+ task_id = int(task_id)
171
+ except ValueError:
172
+ tmp_logger.error("Failed due to invalid task_id")
173
+ return generate_response(False, message=MESSAGE_TASK_ID)
174
+
175
+ ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "resume", properErrorCode=True)
176
+ data, message = ret
177
+ success = data == 0
178
+
179
+ tmp_logger.debug("Done")
180
+
181
+ return generate_response(success, message, data)
182
+
183
+
184
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
185
+ def release(req: PandaRequest, task_id: int) -> Dict[str, Any]:
186
+ """
187
+ Task release
188
+
189
+ Release a given task by skipping iDDS for staging. Requires a secure connection and production role.
190
+
191
+ API details:
192
+ HTTP Method: POST
193
+ Path: /v1/task/release
194
+
195
+ Args:
196
+ req(PandaRequest): internally generated request object
197
+ task_id(int): JEDI Task ID
198
+
199
+ Returns:
200
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
201
+ """
202
+
203
+ tmp_logger = LogWrapper(_logger, f"release < task_id={task_id} >")
204
+ tmp_logger.debug("Start")
205
+
206
+ user = get_dn(req)
207
+ is_production_role = has_production_role(req)
208
+
209
+ # check task_id
210
+ try:
211
+ task_id = int(task_id)
212
+ except ValueError:
213
+ tmp_logger.error("Failed due to invalid task_id")
214
+ return generate_response(False, message=MESSAGE_TASK_ID)
215
+
216
+ ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "release", properErrorCode=True)
217
+ data, message = ret
218
+ success = data == 0
219
+
220
+ tmp_logger.debug("Done")
221
+
222
+ return generate_response(success, message, data)
223
+
224
+
225
+ # reassign task to site/cloud
226
+ @request_validation(_logger, secure=True, request_method="POST")
227
+ def reassign(req: PandaRequest, task_id: int, site: str = None, cloud: str = None, nucleus: str = None, mode: str = None):
228
+ """
229
+ Task reassign
230
+
231
+ Reassign a given task to a site, nucleus or cloud - depending on the parameters. Requires a secure connection.
232
+
233
+ API details:
234
+ HTTP Method: POST
235
+ Path: /v1/task/reassign
236
+
237
+ Args:
238
+ req(PandaRequest): internally generated request object
239
+ task_id(int): JEDI Task ID
240
+ site(str, optional): site name
241
+ cloud(str, optional): cloud name
242
+ nucleus(str, optional): nucleus name
243
+ mode(str, optional): `kill` (kills all jobs, default), `soft` (kills queued jobs) or `nokill` (doesn't kill jobs)
244
+
245
+ Returns:
246
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
247
+ """
248
+
249
+ tmp_logger = LogWrapper(_logger, f"reassign < task_id={task_id} >")
250
+ tmp_logger.debug("Start")
251
+
252
+ # check task_id
253
+ try:
254
+ task_id = int(task_id)
255
+ except ValueError:
256
+ tmp_logger.error("Failed due to invalid task_id")
257
+ return generate_response(False, message=MESSAGE_TASK_ID)
258
+
259
+ user = get_dn(req)
260
+ is_production_role = has_production_role(req)
261
+
262
+ # reassign to site, nucleus or cloud
263
+ comment = make_reassign_comment(site, cloud, nucleus, mode)
264
+
265
+ ret = global_task_buffer.sendCommandTaskPanda(
266
+ task_id,
267
+ user,
268
+ is_production_role,
269
+ "reassign",
270
+ comComment=comment,
271
+ properErrorCode=True,
272
+ )
273
+ data, message = ret
274
+ success = data == 0
275
+
276
+ tmp_logger.debug("Done")
277
+
278
+ return generate_response(success, message, data)
279
+
280
+
281
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
282
+ def pause(req: PandaRequest, task_id: int) -> Dict[str, Any]:
283
+ """
284
+ Task pause
285
+
286
+ Pause a given task. Requires a secure connection and production role.
287
+
288
+ API details:
289
+ HTTP Method: POST
290
+ Path: /v1/task/pause
291
+
292
+ Args:
293
+ req(PandaRequest): internally generated request object
294
+ task_id(int): JEDI Task ID
295
+
296
+ Returns:
297
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
298
+ """
299
+
300
+ tmp_logger = LogWrapper(_logger, f"pause < task_id={task_id} >")
301
+ tmp_logger.debug("Start")
302
+
303
+ try:
304
+ task_id = int(task_id)
305
+ except ValueError:
306
+ tmp_logger.error("Failed due to invalid task_id")
307
+ return generate_response(False, message=MESSAGE_TASK_ID)
308
+
309
+ user = get_dn(req)
310
+ is_production_role = has_production_role(req)
311
+
312
+ ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "pause", properErrorCode=True)
313
+ data, message = ret
314
+ success = data == 0
315
+
316
+ tmp_logger.debug("Done")
317
+
318
+ return generate_response(success, message, data)
319
+
320
+
321
+ @request_validation(_logger, secure=True, request_method="POST")
322
+ def kill(req: PandaRequest, task_id: int = None, broadcast: bool = False) -> Dict[str, Any]:
323
+ """
324
+ Task kill
325
+
326
+ Kill a given task. Requires a secure connection.
327
+
328
+ API details:
329
+ HTTP Method: POST
330
+ Path: /v1/task/kill
331
+
332
+ Args:
333
+ req(PandaRequest): internally generated request object
334
+ task_id(int): JEDI Task ID
335
+ broadcast(bool, optional): broadcast kill command to pilots to kill the jobs
336
+
337
+ Returns:
338
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
339
+ """
340
+ tmp_logger = LogWrapper(_logger, f"kill < task_id={task_id} >")
341
+ tmp_logger.debug("Start")
342
+
343
+ # check task_id
344
+ try:
345
+ task_id = int(task_id)
346
+ except ValueError:
347
+ tmp_logger.error("Failed due to invalid task_id")
348
+ return generate_response(False, message=MESSAGE_TASK_ID)
349
+
350
+ user = get_dn(req)
351
+ is_production_role = has_production_role(req)
352
+
353
+ ret = global_task_buffer.sendCommandTaskPanda(
354
+ task_id,
355
+ user,
356
+ is_production_role,
357
+ "kill",
358
+ properErrorCode=True,
359
+ broadcast=broadcast,
360
+ )
361
+ data, message = ret
362
+ success = data == 0
363
+
364
+ tmp_logger.debug("Done")
365
+
366
+ return generate_response(success, message, data)
367
+
368
+
369
+ @request_validation(_logger, secure=True, request_method="POST")
370
+ def kill_unfinished_jobs(req: PandaRequest, task_id: int, code: int = None, use_email_as_id: bool = False):
371
+ """
372
+ Kill all unfinished jobs in a task
373
+
374
+ Kills all unfinished jobs in a task. Requires a secure connection.
375
+
376
+ API details:
377
+ HTTP Method: POST
378
+ Path: /v1/task/kill_unfinished_jobs
379
+
380
+ Args:
381
+ req(PandaRequest): internally generated request object containing the env variables
382
+ task_id(int): JEDI task ID
383
+ code(int, optional): The kill code. Defaults to None.
384
+ ```
385
+ code
386
+ 2: expire
387
+ 3: aborted
388
+ 4: expire in waiting
389
+ 7: retry by server
390
+ 8: rebrokerage
391
+ 9: force kill
392
+ 10: fast rebrokerage in overloaded PQ
393
+ 50: kill by JEDI
394
+ 51: reassigned by JEDI
395
+ 52: force kill by JEDI
396
+ 55: killed since task is (almost) done
397
+ 60: workload was terminated by the pilot without actual work
398
+ 91: kill user jobs with prod role
399
+ 99: force kill user jobs with prod role
400
+ ```
401
+ use_email_as_id(bool, optional): Use the email as ID. Defaults to False.
402
+
403
+ Returns:
404
+ dict: The system response `{"success": success, "message": message, "data": data}`. The data field contains a list of bools indicating the success of the kill operations.
405
+ """
406
+
407
+ tmp_logger = LogWrapper(_logger, f"kill_unfinished_jobs")
408
+
409
+ # retrieve the user information
410
+ user = get_dn(req)
411
+ fqans = get_fqan(req)
412
+ is_production_manager = has_production_role(req)
413
+
414
+ if use_email_as_id:
415
+ email = get_email_address(user, tmp_logger)
416
+ if email:
417
+ user = email
418
+
419
+ tmp_logger.debug(f"Start user: {user} code: {code} is_production_manager: {is_production_manager} fqans: {fqans} task_id: {task_id}")
420
+
421
+ # Extract working groups with production role from FQANs
422
+ wg_prod_roles = extract_production_working_groups(fqans)
423
+
424
+ # get PandaIDs
425
+ job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id)
426
+
427
+ # kill jobs
428
+ ret = global_task_buffer.killJobs(job_ids, user, code, is_production_manager, wg_prod_roles, [])
429
+ tmp_logger.debug(f"Done with ret: {ret}")
430
+ return generate_response(True, data=ret)
431
+
432
+
433
+ @request_validation(_logger, secure=True, request_method="POST")
434
+ def finish(req: PandaRequest, task_id: int, soft: bool = False, broadcast: bool = False) -> Dict[str, Any]:
435
+ """
436
+ Task finish
437
+
438
+ Finish a given task. Requires a secure connection.
439
+
440
+ API details:
441
+ HTTP Method: POST
442
+ Path: /v1/task/finish
443
+
444
+ Args:
445
+ req(PandaRequest): internally generated request object
446
+ task_id(int): JEDI Task ID
447
+ soft(bool, optional): soft finish
448
+ broadcast(bool, optional): broadcast finish command to pilots
449
+
450
+ Returns:
451
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
452
+ """
453
+ tmp_logger = LogWrapper(_logger, f"finish < task_id={task_id} soft={soft} broadcast={broadcast} >")
454
+ tmp_logger.debug("Start")
455
+
456
+ qualifier = None
457
+ if soft:
458
+ qualifier = "soft"
459
+
460
+ user = get_dn(req)
461
+ is_production_role = has_production_role(req)
462
+
463
+ # check task_id
464
+ try:
465
+ task_id = int(task_id)
466
+ except ValueError:
467
+ tmp_logger.error("Failed due to invalid task_id")
468
+ return generate_response(False, message=MESSAGE_TASK_ID)
469
+
470
+ ret = global_task_buffer.sendCommandTaskPanda(
471
+ task_id,
472
+ user,
473
+ is_production_role,
474
+ "finish",
475
+ properErrorCode=True,
476
+ comQualifier=qualifier,
477
+ broadcast=broadcast,
478
+ )
479
+ data, message = ret
480
+ success = data == 0
481
+
482
+ tmp_logger.debug("Done")
483
+
484
+ return generate_response(success, message, data)
485
+
486
+
487
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
488
+ def reactivate(req: PandaRequest, task_id: int, keep_attempt_nr: bool = False, trigger_job_generation: bool = False) -> Dict[str, Any]:
489
+ """
490
+ Reactivate task
491
+
492
+ Reactivate a given task, i.e. recycle a finished/done task. A reactivated task will generate new jobs and then go to done/finished.
493
+ Requires a secure connection and production role.
494
+
495
+ API details:
496
+ HTTP Method: POST
497
+ Path: /v1/task/reactivate
498
+
499
+ Args:
500
+ req(PandaRequest): internally generated request object
501
+ task_id(int): JEDI Task ID
502
+ keep_attempt_nr(bool, optional): keep the original attempt number
503
+ trigger_job_generation(bool, optional): trigger the job generation
504
+
505
+ Returns:
506
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
507
+ """
508
+ tmp_logger = LogWrapper(_logger, f"reactivate < task_id={task_id} >")
509
+ tmp_logger.debug("Start")
510
+
511
+ try:
512
+ task_id = int(task_id)
513
+ except ValueError:
514
+ tmp_logger.error("Failed due to invalid task_id")
515
+ return generate_response(False, message=MESSAGE_TASK_ID)
516
+
517
+ ret = global_task_buffer.reactivateTask(task_id, keep_attempt_nr, trigger_job_generation)
518
+ code, message = ret
519
+ success = code == 0
520
+
521
+ tmp_logger.debug("Done")
522
+
523
+ return generate_response(success, message, code)
524
+
525
+
526
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
527
+ def avalanche(req: PandaRequest, task_id: int) -> Dict[str, Any]:
528
+ """
529
+ Task avalanche
530
+
531
+ Avalanche a given task. This triggers the avalanche for tasks in scouting state or dynamically reconfigures the task to skip over the scouting state. Requires a secure connection and production role.
532
+
533
+ API details:
534
+ HTTP Method: POST
535
+ Path: /v1/task/avalanche
536
+
537
+ Args:
538
+ req(PandaRequest): internally generated request object
539
+ task_id(int): JEDI Task ID
540
+
541
+ Returns:
542
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
543
+ """
544
+ tmp_logger = LogWrapper(_logger, f"avalanche < task_id={task_id} >")
545
+ tmp_logger.debug("Start")
546
+
547
+ user = get_dn(req)
548
+ is_production_role = has_production_role(req)
549
+
550
+ # check task_id
551
+ try:
552
+ task_id = int(task_id)
553
+ except ValueError:
554
+ tmp_logger.error("Failed due to invalid task_id")
555
+ return generate_response(False, message=MESSAGE_TASK_ID)
556
+
557
+ ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "avalanche", properErrorCode=True)
558
+ data, message = ret
559
+ success = data == 0
560
+
561
+ tmp_logger.debug("Done")
562
+
563
+ return generate_response(success, message, data)
564
+
565
+
566
+ @request_validation(_logger, secure=True, request_method="POST")
567
+ def reload_input(req: PandaRequest, task_id: int, ignore_hard_exhausted: bool = False) -> Dict[str, Any]:
568
+ """
569
+ Reload input
570
+
571
+ Request to reload the input for a given task. Requires a secure connection and production role.
572
+
573
+ API details:
574
+ HTTP Method: POST
575
+ Path: /v1/task/reload_input
576
+
577
+ Args:
578
+ req(PandaRequest): internally generated request object
579
+ task_id(int): JEDI Task ID
580
+ ignore_hard_exhausted(bool, optional): ignore the limits for hard exhausted
581
+
582
+ Returns:
583
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
584
+ """
585
+ tmp_logger = LogWrapper(_logger, f"reload_input < task_id={task_id} >")
586
+ tmp_logger.debug("Start")
587
+
588
+ user = get_dn(req)
589
+ is_production_role = has_production_role(req)
590
+
591
+ # check task_id
592
+ try:
593
+ task_id = int(task_id)
594
+ except ValueError:
595
+ tmp_logger.error("Failed due to invalid task_id")
596
+ return generate_response(False, message=MESSAGE_TASK_ID)
597
+
598
+ # allow ignore_hard_exhausted only for production role
599
+ if not is_production_role and ignore_hard_exhausted:
600
+ ignore_hard_exhausted = False
601
+
602
+ # specify ignore_hard_exhausted as a command qualifier in command's comment
603
+ com_qualifier = JediTaskSpec.get_retry_command_qualifiers(ignore_hard_exhausted=ignore_hard_exhausted)
604
+ com_comment = json.dumps([{}, com_qualifier])
605
+
606
+ ret = global_task_buffer.sendCommandTaskPanda(task_id, user, is_production_role, "incexec", comComment=com_comment, properErrorCode=True)
607
+ data, message = ret
608
+ success = data == 0
609
+
610
+ tmp_logger.debug("Done")
611
+
612
+ return generate_response(success, message, data)
613
+
614
+
615
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
616
+ def reassign_global_share(req: PandaRequest, task_id_list: List[int], share: str, reassign_running_jobs: bool) -> Dict[str, Any]:
617
+ """
618
+ Reassign the global share of a task
619
+
620
+ Reassign the global share of a task. Requires a secure connection and production role.
621
+
622
+ API details:
623
+ HTTP Method: POST
624
+ Path: /v1/task/reassign_global_share
625
+
626
+ Args:
627
+ req(PandaRequest): internally generated request object
628
+ task_id_list(list): List of JEDI task IDs to reassign
629
+ share(str): destination share
630
+ reassign_running_jobs(bool): whether you want to reassign existing running jobs
631
+
632
+ Returns:
633
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
634
+ """
635
+
636
+ tmp_logger = LogWrapper(_logger, f"reassign_global_share < task_id_list={task_id_list} share={share} reassign_running_jobs={reassign_running_jobs} >")
637
+ tmp_logger.debug("Start")
638
+
639
+ if not isinstance(task_id_list, list) or not isinstance(share, str):
640
+ tmp_logger.error("Failed due to invalid task list")
641
+ return generate_response(False, message="wrong parameters: task_ids must be list and share must be string")
642
+
643
+ code, message = global_task_buffer.reassignShare(task_id_list, share, reassign_running_jobs)
644
+ success = code == 0
645
+
646
+ tmp_logger.debug("Done")
647
+
648
+ return generate_response(success, message, code)
649
+
650
+
651
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
652
+ def enable_jumbo_jobs(req: PandaRequest, task_id: int, jumbo_jobs_total: int, jumbo_jobs_per_site: int = None):
653
+ """
654
+ Enable Jumbo jobs
655
+
656
+ Enables the Jumbo jobs for a given task ID. Requires a secure connection and production role.
657
+
658
+ API details:
659
+ HTTP Method: POST
660
+ Path: /v1/task/enable_jumbo_jobs
661
+
662
+ Args:
663
+ req(PandaRequest): internally generated request object
664
+ task_id(int): JEDI task ID
665
+ jumbo_jobs_total(int): Total number of jumbo jobs
666
+ jumbo_jobs_per_site(int): Number of jumbo jobs per site. Defaults to `jumbo_jobs_total`.
667
+
668
+ Returns:
669
+ dict: The system response. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
670
+ """
671
+
672
+ tmp_logger = LogWrapper(_logger, f"enable_jumbo_jobs < task_id={task_id} jumbo_jobs_total={jumbo_jobs_total} n_jumbo_jobs_per_site={jumbo_jobs_per_site} >")
673
+ tmp_logger.debug("Start")
674
+
675
+ if not jumbo_jobs_per_site:
676
+ jumbo_jobs_per_site = jumbo_jobs_total
677
+
678
+ code, message = global_task_buffer.enableJumboJobs(task_id, jumbo_jobs_total, jumbo_jobs_per_site)
679
+ if jumbo_jobs_total > 0 and code == 0:
680
+ tmp_logger.debug("Calling task avalanche")
681
+ avalanche(task_id)
682
+
683
+ success = code == 0
684
+
685
+ tmp_logger.debug("Done")
686
+ return generate_response(success, message, code)
687
+
688
+
689
+ @request_validation(_logger, secure=True, request_method="GET")
690
+ def get_jumbo_job_datasets(req: PandaRequest, from_offset: int, to_offset: int = 0) -> Dict:
691
+ """
692
+ Get jumbo job datasets
693
+
694
+ Gets a map of the jumbo-job-enabled tasks to their datasets, filtering by the last modification time (now - from_offset to now - to_offset).
695
+ Requires a secure connection.
696
+
697
+ API details:
698
+ HTTP Method: GET
699
+ Path: /v1/task/get_jumbo_job_datasets
700
+
701
+ Args:
702
+ req(PandaRequest): internally generated request object
703
+ from_offset(int): `now - from_offset` in days will serve as the floor for modification time (Previously called n_days)
704
+ to_offset(int, optional): `now - to_offset` in days will serve as the ceiling for modification time. Defaults to 0. (Previously called grace_period)
705
+
706
+ Returns:
707
+ dict: The system response `{"success": success, "message": message, "data": data}`.
708
+ When successful, the data field contains the dictionary of JEDI task IDs to datasets.
709
+ """
710
+ tmp_logger = LogWrapper(_logger, f"get_jumbo_job_datasets")
711
+
712
+ tmp_logger.debug("Start")
713
+ jumbo_datasets = global_task_buffer.getJumboJobDatasets(from_offset, to_offset)
714
+ tmp_logger.debug("Done")
715
+
716
+ return generate_response(True, data=jumbo_datasets)
717
+
718
+
719
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
720
+ def enable_job_cloning(
721
+ req: PandaRequest,
722
+ jedi_task_id: int,
723
+ mode: str = None,
724
+ multiplicity: int = None,
725
+ num_sites: int = None,
726
+ ) -> Dict[str, Any]:
727
+ """
728
+ Enable job cloning
729
+
730
+ Enable job cloning for a given task. Requires secure connection and production role.
731
+
732
+ API details:
733
+ HTTP Method: POST
734
+ Path: /v1/task/enable_job_cloning
735
+
736
+ Args:
737
+ req(PandaRequest): internally generated request object
738
+ jedi_task_id(int): JEDI Task ID
739
+ mode(str, optional): mode of operation, runonce or storeonce
740
+ multiplicity(int, optional): number of clones to be created for each target
741
+ num_sites(int, optional): number of sites to be used for each target
742
+
743
+ Returns:
744
+ dict: The system response `{"success": success, "message": message, "data": data}`.
745
+ When there was an error, the message field contains the description.
746
+ """
747
+ tmp_logger = LogWrapper(_logger, f"enable_job_cloning < jedi_task_id={jedi_task_id} >")
748
+ tmp_logger.debug("Start")
749
+ success, message = global_task_buffer.enable_job_cloning(jedi_task_id, mode, multiplicity, num_sites)
750
+ tmp_logger.debug("Done")
751
+ return generate_response(success, message)
752
+
753
+
754
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
755
+ def disable_job_cloning(req: PandaRequest, jedi_task_id: int) -> Dict[str, Any]:
756
+ """
757
+ Disable job cloning
758
+
759
+ Disable job cloning for a given task. Requires secure connection and production role.
760
+
761
+ API details:
762
+ HTTP Method: POST
763
+ Path: /v1/task/disable_job_cloning
764
+
765
+ Args:
766
+ req(PandaRequest): internally generated request object
767
+ jedi_task_id(int): JEDI Task ID
768
+
769
+ Returns:
770
+ dict: The system response `{"success": success, "message": message, "data": data}`.
771
+ When there was an error, the message field contains the description.
772
+ """
773
+ tmp_logger = LogWrapper(_logger, f"disable_job_cloning < jedi_task_id={jedi_task_id} >")
774
+ tmp_logger.debug("Start")
775
+ success, message = global_task_buffer.disable_job_cloning(jedi_task_id)
776
+ tmp_logger.debug("Done")
777
+ return generate_response(success, message)
778
+
779
+
780
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
781
+ def increase_attempts(req: PandaRequest, task_id: int, increase: int) -> Dict[str, Any]:
782
+ """
783
+ Increase possible task attempts
784
+
785
+ Increase possible task attempts. Requires secure connection and production role.
786
+
787
+ API details:
788
+ HTTP Method: POST
789
+ Path: /v1/task/increase_attempts
790
+
791
+ Args:
792
+ req(PandaRequest): internally generated request object
793
+ task_id(int): JEDI Task ID
794
+ increase(int): number of attempts to increase
795
+
796
+ Returns:
797
+ dict: The system response `{"success": success, "message": message, "data": data}`.
798
+ When there was an error, the message field contains the description and the data field contains the code.
799
+ """
800
+ tmp_logger = LogWrapper(_logger, f"increase_attempt_number task_id={task_id}")
801
+ tmp_logger.debug("Start")
802
+
803
+ try:
804
+ task_id = int(task_id)
805
+ except Exception:
806
+ tmp_logger.error("Failed due to invalid task_id")
807
+ return generate_response(False, message=MESSAGE_TASK_ID)
808
+
809
+ # check value for increase
810
+ try:
811
+ increase = int(increase)
812
+ if increase < 0:
813
+ raise ValueError
814
+ except Exception:
815
+ message = f"increase must be a positive integer, got {increase}"
816
+ tmp_logger.error(message)
817
+ return generate_response(False, message=message)
818
+
819
+ code, message = global_task_buffer.increaseAttemptNrPanda(task_id, increase)
820
+ success = code == 0
821
+
822
+ tmp_logger.debug("Done")
823
+
824
+ return generate_response(success, message, code)
825
+
826
+
827
+ @request_validation(_logger, secure=True, request_method="GET")
828
+ def get_status(req, task_id):
829
+ """
830
+ Get task status
831
+
832
+ Get the status of a given task. Requires secure connection.
833
+
834
+ API details:
835
+ HTTP Method: GET
836
+ Path: /v1/task/get_status
837
+
838
+ Args:
839
+ req(PandaRequest): internally generated request object
840
+ task_id(int): JEDI Task ID
841
+
842
+ Returns:
843
+ dict: The system response `{"success": success, "message": message, "data": data}`.
844
+ When successful, the data field contains the status of the task.
845
+ When there was an error, the message field contains the description.
846
+ """
847
+ tmp_logger = LogWrapper(_logger, f"get_status < task_id={task_id} >")
848
+ tmp_logger.debug("Start")
849
+
850
+ try:
851
+ task_id = int(task_id)
852
+ except ValueError:
853
+ tmp_logger.error("Failed due to invalid task_id")
854
+ return generate_response(False, message=MESSAGE_TASK_ID)
855
+
856
+ ret = global_task_buffer.getTaskStatus(task_id)
857
+ if not ret:
858
+ return generate_response(False, message="Task not found")
859
+ status = ret[0]
860
+
861
+ tmp_logger.debug("Done")
862
+
863
+ return generate_response(True, data=status)
864
+
865
+
866
+ @request_validation(_logger, request_method="GET", secure=True)
867
+ def get_details(req: PandaRequest, task_id: int, include_parameters: bool = False, include_status: bool = False):
868
+ """
869
+ Get task details
870
+
871
+ Get the details of a given task. Requires secure connection.
872
+
873
+ API details:
874
+ HTTP Method: GET
875
+ Path: /v1/task/get_details
876
+
877
+ Args:
878
+ req(PandaRequest): internally generated request object
879
+ task_id(int): JEDI Task ID
880
+ include_parameters(bool, optional): flag to include task parameter information (Previously fullFlag)
881
+ include_status(bool, optional): flag to include status information (Previously withTaskInfo)
882
+
883
+ Returns:
884
+ dict: The system response `{"success": success, "message": message, "data": data}`.
885
+ When successful, the data field contains the task details.
886
+ When there was an error, the message field contains the description.
887
+ """
888
+ tmp_logger = LogWrapper(_logger, f"get_details < task_id={task_id} include_parameters={include_parameters} include_status={include_status} >")
889
+ tmp_logger.debug("Start")
890
+
891
+ details = global_task_buffer.getJediTaskDetails(task_id, include_parameters, include_status)
892
+ if not details:
893
+ tmp_logger.error("Task not found or error retrieving the details")
894
+ return generate_response(False, message="Task not found or error retrieving the details")
895
+
896
+ tmp_logger.debug("Done")
897
+
898
+ return generate_response(True, data=details)
899
+
900
+
901
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
902
+ def change_attribute(req: PandaRequest, task_id: int, attribute_name: str, value: int) -> Dict[str, Any]:
903
+ """
904
+ Change a task attribute
905
+
906
+ Change a task attribute within the list of valid attributes ("ramCount", "wallTime", "cpuTime", "coreCount").
907
+ Requires a secure connection and production role.
908
+
909
+ API details:
910
+ HTTP Method: POST
911
+ Path: /v1/task/change_attribute
912
+
913
+ Args:
914
+ req(PandaRequest): internally generated request object
915
+ task_id(int): JEDI task ID
916
+ attribute_name(str): attribute to change
917
+ value(int): value to set to the attribute
918
+
919
+ Returns:
920
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
921
+ Return code in the data field, 0 for success, others for failure.
922
+ """
923
+ tmp_logger = LogWrapper(_logger, f"change_attribute < task_id={task_id} attribute_name={attribute_name} value={value} >")
924
+ tmp_logger.debug("Start")
925
+
926
+ # check task_id
927
+ try:
928
+ task_id = int(task_id)
929
+ except ValueError:
930
+ tmp_logger.error("Failed due to invalid task_id")
931
+ return generate_response(False, message=MESSAGE_TASK_ID)
932
+
933
+ # check if attribute_name is valid
934
+ valid_attributes = ["ramCount", "wallTime", "cpuTime", "coreCount"]
935
+ if attribute_name not in valid_attributes:
936
+ tmp_logger.error("Failed due to invalid attribute_name")
937
+ return generate_response(False, message=f"{attribute_name} is not a valid attribute. Valid attributes are {valid_attributes}")
938
+
939
+ n_tasks_changed = global_task_buffer.changeTaskAttributePanda(task_id, attribute_name, value)
940
+ if n_tasks_changed is None: # method excepted
941
+ tmp_logger.error("Failed due to exception while changing the attribute")
942
+ return generate_response(False, message="Exception while changing the attribute")
943
+
944
+ if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
945
+ tmp_logger.error("Failed due to task not found")
946
+ return generate_response(False, message="Task not found")
947
+
948
+ tmp_logger.debug("Done")
949
+ return generate_response(True, message=f"{n_tasks_changed} tasks changed")
950
+
951
+
952
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
953
+ def change_modification_time(req: PandaRequest, task_id: int, hour_offset: int) -> Dict[str, Any]:
954
+ """
955
+ Change task modification time
956
+
957
+ Change the modification time for a task to `now() + hour_offset`. Requires a secure connection and production role.
958
+
959
+ API details:
960
+ HTTP Method: POST
961
+ Path: /v1/task/change_modification_time
962
+
963
+ Args:
964
+ req(PandaRequest): internally generated request object
965
+ task_id(int): JEDI task ID
966
+ hour_offset(int): number of hours to add to the current time. Use a negative value (e.g. -12) to trigger task brokerage.
967
+
968
+ Returns:
969
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
970
+ """
971
+ tmp_logger = LogWrapper(_logger, f"change_modification_time < task_id={task_id} hour_offset={hour_offset} >")
972
+ tmp_logger.debug("Start")
973
+
974
+ # check offset
975
+ try:
976
+ new_modification_time = datetime.datetime.now() + datetime.timedelta(hours=hour_offset)
977
+ except ValueError:
978
+ tmp_logger.error("Failed due to invalid hour_offset")
979
+ return generate_response(False, message=f"failed to convert {hour_offset} to time")
980
+
981
+ n_tasks_changed = global_task_buffer.changeTaskAttributePanda(task_id, "modificationTime", new_modification_time)
982
+ if n_tasks_changed is None: # method excepted
983
+ tmp_logger.error("Failed due to exception while changing the attribute")
984
+ return generate_response(False, message="Exception while changing the attribute")
985
+
986
+ if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
987
+ tmp_logger.error("Failed due to task not found")
988
+ return generate_response(False, message="Task not found")
989
+
990
+ tmp_logger.debug("Done")
991
+ return generate_response(True, message=f"{n_tasks_changed} tasks changed")
992
+
993
+
994
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
995
+ def change_priority(req: PandaRequest, task_id: int, priority: int):
996
+ """
997
+ Change priority
998
+
999
+ Change the priority of a given task. Requires a secure connection and production role.
1000
+
1001
+ API details:
1002
+ HTTP Method: POST
1003
+ Path: /v1/task/change_priority
1004
+
1005
+ Args:
1006
+ req(PandaRequest): internally generated request object
1007
+ task_id(int): JEDI task ID
1008
+ priority(int): new priority for the task
1009
+
1010
+ Returns:
1011
+ dict: The system response `{"success": success, "message": message, "data": data}`.
1012
+ True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
1013
+ """
1014
+ tmp_logger = LogWrapper(_logger, f"change_priority < task_id={task_id} priority={priority} >")
1015
+ tmp_logger.debug("Start")
1016
+
1017
+ # check task_id
1018
+ try:
1019
+ task_id = int(task_id)
1020
+ except ValueError:
1021
+ tmp_logger.error("Failed due to invalid task_id")
1022
+ return generate_response(False, message=MESSAGE_TASK_ID)
1023
+
1024
+ # check priority
1025
+ try:
1026
+ priority = int(priority)
1027
+ except ValueError:
1028
+ tmp_logger.error("Failed due to invalid priority")
1029
+ return generate_response(False, message="priority must be an integer")
1030
+
1031
+ n_tasks_changed = global_task_buffer.changeTaskPriorityPanda(task_id, priority)
1032
+
1033
+ if n_tasks_changed is None: # method excepted
1034
+ tmp_logger.error("Failed due to exception while changing the priority")
1035
+ return generate_response(False, message="Exception while changing the priority")
1036
+
1037
+ if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
1038
+ tmp_logger.error("Failed due to task not found")
1039
+ return generate_response(False, message="Task not found")
1040
+
1041
+ tmp_logger.debug("Done")
1042
+
1043
+ return generate_response(True, message=f"{n_tasks_changed} tasks changed")
1044
+
1045
+
1046
+ @request_validation(_logger, secure=True, production=True, request_method="POST")
1047
+ def change_split_rule(req: PandaRequest, task_id: int, attribute_name: str, value: str) -> Dict[str, Any]:
1048
+ """
1049
+ Change the split rule
1050
+
1051
+ Change the split rule for a task by modifying or adding an `attribute_name=value pair`. Requires a secure connection and production role.
1052
+
1053
+ API details:
1054
+ HTTP Method: POST
1055
+ Path: /v1/task/change_split_rule
1056
+
1057
+ Args:
1058
+ req(PandaRequest): internally generated request object
1059
+ task_id(int): JEDI task ID
1060
+ attribute_name(str): split rule attribute to change. The allowed attributes are defined in `task_split_rules.changeable_split_rule_tags`
1061
+ value(str): value to set to the attribute
1062
+
1063
+ Returns:
1064
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
1065
+ Return code in the data field, 0 for success, others for failure.
1066
+ """
1067
+ tmp_logger = LogWrapper(_logger, f"change_split_rule < task_id={task_id} attribute_name={attribute_name} value={value} >")
1068
+ tmp_logger.debug("Start")
1069
+
1070
+ # check task_id
1071
+ try:
1072
+ task_id = int(task_id)
1073
+ except ValueError:
1074
+ tmp_logger.error("Failed due to invalid task_id")
1075
+ return generate_response(False, message=MESSAGE_TASK_ID)
1076
+
1077
+ # check attribute
1078
+ if attribute_name not in task_split_rules.changeable_split_rule_tags:
1079
+ tmp_logger.error("Failed due to invalid attribute_name")
1080
+ return generate_response(
1081
+ False, message=f"{attribute_name} is not a valid attribute. Valid attributes are {task_split_rules.changeable_split_rule_tags}", data=2
1082
+ )
1083
+
1084
+ n_tasks_changed = global_task_buffer.changeTaskSplitRulePanda(task_id, attribute_name, value)
1085
+ if n_tasks_changed is None: # method excepted
1086
+ tmp_logger.error("Failed due to exception while changing the split rule")
1087
+ return generate_response(False, message="Exception while changing the split rule")
1088
+
1089
+ if n_tasks_changed == 0: # no tasks were changed should mean that it doesn't exist
1090
+ tmp_logger.error("Failed due to task not found")
1091
+ return generate_response(False, message="Task not found")
1092
+
1093
+ tmp_logger.debug("Done")
1094
+
1095
+ return generate_response(True, message=f"{n_tasks_changed} tasks changed")
1096
+
1097
+
1098
+ @request_validation(_logger, secure=True, request_method="GET")
1099
+ def get_tasks_modified_since(req, since: str, dn: str = None, full: bool = False, min_task_id: int = None, prod_source_label: str = "user") -> Dict[str, Any]:
1100
+ """
1101
+ Get tasks modified since
1102
+
1103
+ Get the tasks with `modificationtime > since`. Requires a secure connection.
1104
+
1105
+ API details:
1106
+ HTTP Method: GET
1107
+ Path: /v1/task/get_tasks_modified_since
1108
+
1109
+ Args:
1110
+ req(PandaRequest): internally generated request object
1111
+ since(str): time in the format `%Y-%m-%d %H:%M:%S`, e.g. `2024-12-18 14:30:45`. The tasks with `modificationtime > since` will be returned
1112
+ dn(str, optional): user DN
1113
+ full(bool, optional): flag to include full task information. If `full=False` the basic fields are `jediTaskID, modificationTime, status, processingType, transUses, transHome, architecture, reqID, creationDate, site, cloud, taskName`
1114
+ min_task_id(int, optional): minimum task ID
1115
+ prod_source_label(str, optional): task type (e.g. `user`, `managed`, `test`, etc.)
1116
+
1117
+ Returns:
1118
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
1119
+ """
1120
+ tmp_logger = LogWrapper(_logger, "get_tasks_modified_since")
1121
+ tmp_logger.debug("Start")
1122
+
1123
+ if not dn:
1124
+ dn = get_dn(req)
1125
+
1126
+ tmp_logger.debug(f"parameters dn:{dn} since:{since} full:{full} min_task_id:{min_task_id} prod_source_label:{prod_source_label}")
1127
+
1128
+ tasks = global_task_buffer.getJediTasksInTimeRange(dn, since, full, min_task_id, prod_source_label)
1129
+
1130
+ tmp_logger.debug("Done")
1131
+
1132
+ return generate_response(True, data=tasks)
1133
+
1134
+
1135
+ @request_validation(_logger, secure=True, request_method="GET")
1136
+ def get_datasets_and_files(req, task_id, dataset_types: List[str] = ("input", "pseudo_input")) -> Dict[str, Any]:
1137
+ """
1138
+ Get datasets and files
1139
+
1140
+ Get the files in the datasets associated to a given task. You can filter passing a list of dataset types. The return format is:
1141
+ ```
1142
+ [
1143
+ {
1144
+ "dataset": {
1145
+ "name": dataset_name,
1146
+ "id": dataset_id
1147
+ },
1148
+ "files": [
1149
+ {
1150
+ "lfn": lfn,
1151
+ "scope": file_scope,
1152
+ "id": file_id,
1153
+ "status": status
1154
+ },
1155
+ ...
1156
+ ]
1157
+ },
1158
+ ...
1159
+ ]
1160
+ ```
1161
+ Requires a secure connection.
1162
+
1163
+ API details:
1164
+ HTTP Method: GET
1165
+ Path: /v1/task/get_datasets_and_files
1166
+
1167
+ Args:
1168
+ req(PandaRequest): internally generated request object
1169
+ task_id(int): JEDI task ID
1170
+ dataset_types(List, optional): list of dataset types, defaults to `["input", "pseudo_input"]`
1171
+
1172
+ Returns:
1173
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
1174
+ """
1175
+ tmp_logger = LogWrapper(_logger, f"get_datasets_and_files < task_id={task_id} dataset_types={dataset_types} >")
1176
+ tmp_logger.debug("Start")
1177
+
1178
+ data = global_task_buffer.get_files_in_datasets(task_id, dataset_types)
1179
+ if data is None:
1180
+ tmp_logger.error("Failed due to exception while gathering files")
1181
+ return generate_response(False, message="Database exception while gathering files")
1182
+
1183
+ if data == []:
1184
+ tmp_logger.error("Failed due to no data found for the task")
1185
+ return generate_response(False, message="No data found for the task")
1186
+
1187
+ tmp_logger.debug("Done")
1188
+
1189
+ return generate_response(True, data=data)
1190
+
1191
+
1192
+ @request_validation(_logger, secure=True, request_method="GET")
1193
+ def get_job_ids(req: PandaRequest, task_id: int) -> Dict[str, Any]:
1194
+ """
1195
+ Get job IDs
1196
+
1197
+ Get a list with the job IDs `[job_id, ...]` (in any status) associated to a given task. Requires a secure connection.
1198
+
1199
+ API details:
1200
+ HTTP Method: GET
1201
+ Path: /v1/task/get_job_ids
1202
+
1203
+ Args:
1204
+ req(PandaRequest): internally generated request object
1205
+ task_id(int): JEDI task ID
1206
+
1207
+ Returns:
1208
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
1209
+ """
1210
+ tmp_logger = LogWrapper(_logger, f"get_job_ids < task_id={task_id} >")
1211
+ tmp_logger.debug("Start")
1212
+
1213
+ try:
1214
+ task_id = int(task_id)
1215
+ except ValueError:
1216
+ tmp_logger.error("Failed due to invalid task_id")
1217
+ return generate_response(False, message=MESSAGE_TASK_ID)
1218
+
1219
+ job_id_list = global_task_buffer.getPandaIDsWithTaskID(task_id)
1220
+
1221
+ tmp_logger.debug("Done")
1222
+
1223
+ return generate_response(True, data=job_id_list)
1224
+
1225
+
1226
+ @request_validation(_logger, secure=True, request_method="POST")
1227
+ def submit(req: PandaRequest, task_parameters: Dict, parent_tid: int = None) -> Dict[str, Any]:
1228
+ """
1229
+ Register task
1230
+
1231
+ Insert the task parameters to register a task. Requires a secure connection.
1232
+
1233
+ API details:
1234
+ HTTP Method: POST
1235
+ Path: /v1/task/submit
1236
+
1237
+ Args:
1238
+ req(PandaRequest): internally generated request object
1239
+ task_parameters(dict): Dictionary with all the required task parameters. The parameters are the attributes in the JediTaskSpec object (https://github.com/PanDAWMS/panda-jedi/blob/master/pandajedi/jedicore/JediTaskSpec.py).
1240
+ parent_tid(int, optional): Parent task ID
1241
+
1242
+ Returns:
1243
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message.
1244
+ Return code in the data field, 0 for success, others for failure.
1245
+ """
1246
+ tmp_log = LogWrapper(_logger, f"submit {naive_utcnow().isoformat('/')}")
1247
+ tmp_log.debug("Start")
1248
+
1249
+ user = get_dn(req)
1250
+ is_production_role = has_production_role(req)
1251
+ fqans = get_fqan(req)
1252
+
1253
+ tmp_log.debug(f"user={user} is_production_role={is_production_role} FQAN:{str(fqans)} parent_tid={parent_tid}")
1254
+ ret = global_task_buffer.insertTaskParamsPanda(task_parameters, user, is_production_role, fqans, properErrorCode=True, parent_tid=parent_tid, decode=False)
1255
+
1256
+ code, message = ret
1257
+ success = code in (0, 3)
1258
+ if not success:
1259
+ return generate_response(False, message=message, data=code)
1260
+
1261
+ # Extract the task ID from the message
1262
+ task_id = None
1263
+ match = re.search(r"jediTaskID=(\d+)", message)
1264
+ if match:
1265
+ try:
1266
+ task_id = int(match.group(1)) # Convert to an integer
1267
+ tmp_log.debug(f"Created task with task_id: {task_id}")
1268
+ except ValueError:
1269
+ tmp_log.error("Failed to extract the task ID from the message")
1270
+ task_id = None
1271
+
1272
+ tmp_log.debug("Done")
1273
+
1274
+ return generate_response(True, message=message, data=task_id)
1275
+
1276
+
1277
+ @request_validation(_logger, request_method="GET")
1278
+ def get_task_parameters(req: PandaRequest, task_id: int) -> Dict[str, Any]:
1279
+ """
1280
+ Get task parameters
1281
+
1282
+ Get a dictionary with the task parameters used to create a task.
1283
+
1284
+ API details:
1285
+ HTTP Method: GET
1286
+ Path: /v1/task/get_task_parameters
1287
+
1288
+ Args:
1289
+ req(PandaRequest): internally generated request object
1290
+ task_id(int): JEDI task ID
1291
+
1292
+ Returns:
1293
+ dict: The system response `{"success": success, "message": message, "data": data}`. True for success, False for failure, and an error message. Return code in the data field, 0 for success, others for failure.
1294
+
1295
+ """
1296
+ tmp_logger = LogWrapper(_logger, f"get_task_parameters < task_id={task_id} >")
1297
+ tmp_logger.debug("Start")
1298
+
1299
+ # validate the task id
1300
+ try:
1301
+ task_id = int(task_id)
1302
+ except Exception:
1303
+ tmp_logger.error("Failed due to invalid task_id")
1304
+ return generate_response(False, message=MESSAGE_TASK_ID)
1305
+
1306
+ # get the parameters
1307
+ task_parameters_str = global_task_buffer.getTaskParamsMap(task_id)
1308
+ if not task_parameters_str:
1309
+ tmp_logger.error("Failed due to task not found")
1310
+ return generate_response(False, message="Task not found")
1311
+
1312
+ # decode the parameters
1313
+ try:
1314
+ task_parameters = json.loads(task_parameters_str)
1315
+ except json.JSONDecodeError as e:
1316
+ tmp_logger.error("Failed due to error decoding the task parameters")
1317
+ return generate_response(False, message=f"Error decoding the task parameters: {str(e)}")
1318
+
1319
+ tmp_logger.debug("Done")
1320
+
1321
+ return generate_response(True, data=task_parameters)
1322
+
1323
+
1324
+ @request_validation(_logger, secure=True, request_method="GET")
1325
+ def get_detailed_info(req: PandaRequest, task_id: int) -> Dict[str, Any]:
1326
+ """
1327
+ Get detailed task info
1328
+
1329
+ Get all task fields together with jobParamsTemplate and taskParams for a given task.
1330
+ The record is read without locking. Requires a secure connection.
1331
+
1332
+ API details:
1333
+ HTTP Method: GET
1334
+ Path: /v1/task/get_detailed_info
1335
+
1336
+ Args:
1337
+ req(PandaRequest): internally generated request object
1338
+ task_id(int): JEDI task ID
1339
+
1340
+ Returns:
1341
+ dict: The system response ``{"success": success, "message": message, "data": data}``.
1342
+ On success the ``data`` field contains a dictionary with all JediTaskSpec
1343
+ attributes plus additional information.
1344
+ On failure ``success`` is False and ``message`` contains the error description.
1345
+ """
1346
+ tmp_logger = LogWrapper(_logger, f"get_detailed_info < task_id={task_id} >")
1347
+ tmp_logger.debug("Start")
1348
+
1349
+ try:
1350
+ task_id = int(task_id)
1351
+ except Exception:
1352
+ tmp_logger.error("Failed due to invalid task_id")
1353
+ return generate_response(False, message=MESSAGE_TASK_ID)
1354
+
1355
+ task_info = global_task_buffer.get_task_details_json(task_id)
1356
+ if task_info is None:
1357
+ tmp_logger.error("Task not found or error retrieving task info")
1358
+ return generate_response(False, message="Task not found or error retrieving task info")
1359
+
1360
+ tmp_logger.debug("Done")
1361
+ return generate_response(True, data=task_info)
1362
+
1363
+
1364
+ @request_validation(_logger, secure=True, request_method="GET")
1365
+ def get_parent_detailed_info(req: PandaRequest, task_id: int) -> Dict[str, Any]:
1366
+ """
1367
+ Get detailed parent task info
1368
+
1369
+ Resolve the parent task from a given child task and return only the parent
1370
+ detailed information. Requires a secure connection.
1371
+
1372
+ API details:
1373
+ HTTP Method: GET
1374
+ Path: /v1/task/get_parent_detailed_info
1375
+
1376
+ Args:
1377
+ req(PandaRequest): internally generated request object
1378
+ task_id(int): child JEDI task ID
1379
+
1380
+ Returns:
1381
+ dict: The system response ``{"success": success, "message": message, "data": data}``.
1382
+ On success the ``data`` field contains the parent task details.
1383
+ On failure ``success`` is False with a message distinguishing:
1384
+ child-not-found, no-parent, parent-not-found, or retrieval error.
1385
+ """
1386
+ tmp_logger = LogWrapper(_logger, f"get_parent_detailed_info < child_task_id={task_id} >")
1387
+ tmp_logger.debug("Start")
1388
+
1389
+ try:
1390
+ task_id = int(task_id)
1391
+ except Exception:
1392
+ tmp_logger.error("Failed due to invalid task_id")
1393
+ return generate_response(False, message=MESSAGE_TASK_ID)
1394
+
1395
+ # Resolve parent and retrieve details in one DB-helper call.
1396
+ resolve_status, parent_task_id, task_info = global_task_buffer.get_task_details_json(
1397
+ task_id,
1398
+ resolve_parent=True,
1399
+ include_resolve_status=True,
1400
+ )
1401
+ tmp_logger.debug(f"resolve_parent status={resolve_status} child={task_id} parent={parent_task_id}")
1402
+
1403
+ if resolve_status == "child_not_found":
1404
+ tmp_logger.error(f"Child task not found child={task_id} parent={parent_task_id}")
1405
+ return generate_response(False, message="Child task not found")
1406
+ if resolve_status == "no_parent":
1407
+ tmp_logger.error(f"No valid parent task child={task_id} parent={parent_task_id}")
1408
+ return generate_response(False, message="No parent task found")
1409
+ if resolve_status == "parent_not_found":
1410
+ tmp_logger.error(f"Parent task not found child={task_id} parent={parent_task_id}")
1411
+ return generate_response(False, message="Parent task not found")
1412
+ if resolve_status in ["target_not_found", "error"] or task_info is None:
1413
+ tmp_logger.error(f"Parent task retrieval failed status={resolve_status} child={task_id} parent={parent_task_id}")
1414
+ return generate_response(False, message="Parent task details not found or retrieval error")
1415
+
1416
+ tmp_logger.debug(f"Done child={task_id} parent={parent_task_id}")
1417
+ return generate_response(True, data=task_info)
1418
+
1419
+
1420
+ @request_validation(_logger, secure=True, request_method="GET")
1421
+ def get_scout_job_descriptions(req: PandaRequest, task_id: int) -> Dict:
1422
+ """
1423
+ Get scout job descriptions for a task.
1424
+
1425
+ Resolves all PanDA job IDs associated with a JEDI task, retrieves their job
1426
+ descriptions including archive lookup, and returns only scout jobs. Scout
1427
+ jobs are identified by the ``sj`` token in the comma-separated
1428
+ ``specialHandling`` field.
1429
+ Requires a secure connection.
1430
+
1431
+ API details:
1432
+ HTTP Method: GET
1433
+ Path: /v1/task/get_scout_job_descriptions
1434
+
1435
+ Args:
1436
+ req(PandaRequest): internally generated request object containing the env variables.
1437
+ task_id (int): JEDI task ID.
1438
+
1439
+ Returns:
1440
+ dict: The system response `{"success": success, "message": message, "data": data}`.
1441
+ When successful, the data field contains a list of scout job descriptions.
1442
+ If no jobs are found or no scout jobs exist, returns success=True with empty list.
1443
+ """
1444
+ tmp_logger = LogWrapper(_logger, f"get_scout_job_descriptions task_id={task_id}")
1445
+ tmp_logger.debug("Start")
1446
+
1447
+ try:
1448
+ task_id = int(task_id)
1449
+ except Exception:
1450
+ tmp_logger.error("Failed due to invalid task_id")
1451
+ return generate_response(False, message=MESSAGE_TASK_ID)
1452
+
1453
+ job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id, scout_only=True)
1454
+ if not job_ids:
1455
+ tmp_logger.debug("No scout jobs found for task")
1456
+ return generate_response(True, data=[])
1457
+
1458
+ tmp_logger.debug(f"Found {len(job_ids)} scout job IDs for task")
1459
+
1460
+ max_ids = 5500
1461
+ if len(job_ids) > max_ids:
1462
+ tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
1463
+ job_ids = job_ids[:max_ids]
1464
+
1465
+ scout_jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
1466
+
1467
+ tmp_logger.debug(f"Done with {len(scout_jobs)} scout jobs")
1468
+ return generate_response(True, data=scout_jobs)
1469
+
1470
+
1471
+ @request_validation(_logger, secure=True, request_method="GET")
1472
+ def get_job_descriptions(req: PandaRequest, task_id: int, unsuccessful_only: bool = False) -> Dict:
1473
+ """
1474
+ Get job descriptions for a task.
1475
+
1476
+ Resolves all PanDA job IDs associated with a JEDI task, retrieves their job
1477
+ descriptions including archive lookup, and returns them. Optionally filters
1478
+ to only unsuccessful jobs (status ``failed``, ``cancelled``, or ``closed``).
1479
+ Requires a secure connection.
1480
+
1481
+ API details:
1482
+ HTTP Method: GET
1483
+ Path: /v1/task/get_job_descriptions
1484
+
1485
+ Args:
1486
+ req(PandaRequest): internally generated request object containing the env variables.
1487
+ task_id (int): JEDI task ID.
1488
+ unsuccessful_only (bool): When True, return only failed, cancelled, or closed jobs.
1489
+ Defaults to False.
1490
+
1491
+ Returns:
1492
+ dict: The system response `{"success": success, "message": message, "data": data}`.
1493
+ When successful, the data field contains a list of job descriptions.
1494
+ If no jobs are found, returns success=True with empty list.
1495
+ """
1496
+ tmp_logger = LogWrapper(_logger, f"get_job_descriptions task_id={task_id} unsuccessful_only={unsuccessful_only}")
1497
+ tmp_logger.debug("Start")
1498
+
1499
+ try:
1500
+ task_id = int(task_id)
1501
+ except Exception:
1502
+ tmp_logger.error("Failed due to invalid task_id")
1503
+ return generate_response(False, message=MESSAGE_TASK_ID)
1504
+
1505
+ job_ids = global_task_buffer.getPandaIDsWithTaskID(task_id, unsuccessful_only=unsuccessful_only)
1506
+ if not job_ids:
1507
+ tmp_logger.debug("No jobs found for task")
1508
+ return generate_response(True, data=[])
1509
+
1510
+ tmp_logger.debug(f"Found {len(job_ids)} job IDs for task")
1511
+
1512
+ max_ids = 5500
1513
+ if len(job_ids) > max_ids:
1514
+ tmp_logger.error(f"List of PanDA IDs is longer than {max_ids}. Truncating the list.")
1515
+ job_ids = job_ids[:max_ids]
1516
+
1517
+ jobs = [job.to_dict_advanced() for job in global_task_buffer.getFullJobStatus(job_ids) if job is not None]
1518
+
1519
+ tmp_logger.debug(f"Done with {len(jobs)} jobs")
1520
+ return generate_response(True, data=jobs)