panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,308 @@
1
+ # split rules
2
+ split_rule_dict = {
3
+ "allowEmptyInput": "AE",
4
+ "allowIncompleteInDS": "AI",
5
+ "addNthFieldToLFN": "AN",
6
+ "allowPartialFinish": "AP",
7
+ "altStageOut": "AT",
8
+ "avoidVP": "AV",
9
+ "maxCoreCount": "CC",
10
+ "cloudAsVO": "CV",
11
+ "ddmBackEnd": "DE",
12
+ "disableAutoFinish": "DF",
13
+ "disableReassign": "DI",
14
+ "debugMode": "DM",
15
+ "disableAutoRetry": "DR",
16
+ "dynamicNumEvents": "DY",
17
+ "nEsConsumers": "EC",
18
+ "nEventsPerInput": "EI",
19
+ "encJobParams": "EJ",
20
+ "nEventsPerWorker": "ES",
21
+ "firstContentsFeed": "FC",
22
+ "failGoalUnreached": "FG",
23
+ "fineGrainedProc": "FP",
24
+ "firstEvent": "FT",
25
+ "fullChain": "FU",
26
+ "groupBoundaryID": "GB",
27
+ "hpoWorkflow": "HO",
28
+ "instantiateTmplSite": "IA",
29
+ "inFilePosEvtNum": "IF",
30
+ "ipStack": "IK",
31
+ "allowInputLAN": "IL",
32
+ "ignoreMissingInDS": "IM",
33
+ "intermediateTask": "IN",
34
+ "ipConnectivity": "IP",
35
+ "inputPreStaging": "IS",
36
+ "instantiateTmpl": "IT",
37
+ "allowInputWAN": "IW",
38
+ "noLoopingCheck": "LC",
39
+ "useLocalIO": "LI",
40
+ "limitedSites": "LS",
41
+ "loadXML": "LX",
42
+ "minCpuEfficiency": "MC",
43
+ "messageDriven": "MD",
44
+ "mergeEsOnOS": "ME",
45
+ "nMaxFilesPerJob": "MF",
46
+ "maxJumboPerSite": "MJ",
47
+ "maxNumJobs": "MN",
48
+ "mergeOutput": "MO",
49
+ "multiStepExec": "MS",
50
+ "maxWalltime": "MW",
51
+ "maxEventsPerJob": "MX",
52
+ "noExecStrCnv": "NC",
53
+ "notDiscardEvents": "ND",
54
+ "nEventsPerJob": "NE",
55
+ "nFilesPerJob": "NF",
56
+ "nGBPerJob": "NG",
57
+ "noInputPooling": "NI",
58
+ "nJumboJobs": "NJ",
59
+ "noAutoPause": "NP",
60
+ "nSitesPerJob": "NS",
61
+ "nChunksToWait": "NT",
62
+ "noWaitParent": "NW",
63
+ "orderInputBy": "OI",
64
+ "orderByLB": "OL",
65
+ "onSiteMerging": "OM",
66
+ "osMatching": "OS",
67
+ "onlyTagsForFC": "OT",
68
+ "pushStatusChanges": "PC",
69
+ "pushJob": "PJ",
70
+ "pfnList": "PL",
71
+ "putLogToOS": "PO",
72
+ "runUntilClosed": "RC",
73
+ "registerDatasets": "RD",
74
+ "registerEsFiles": "RE",
75
+ "respectLB": "RL",
76
+ "retryModuleRules": "RM",
77
+ "reuseSecOnDemand": "RO",
78
+ "releasePerLB": "RP",
79
+ "respectSplitRule": "RR",
80
+ "randomSeed": "RS",
81
+ "retryRamOffset": "RX",
82
+ "retryRamStep": "RY",
83
+ "retryRamMax": "RZ",
84
+ "resurrectConsumers": "SC",
85
+ "switchEStoNormal": "SE",
86
+ "stayOutputOnSite": "SO",
87
+ "scoutSuccessRate": "SS",
88
+ "useSecrets": "ST",
89
+ "segmentedWork": "SW",
90
+ "totNumJobs": "TJ",
91
+ "tgtMaxOutputForNG": "TN",
92
+ "t1Weight": "TW",
93
+ "useBuild": "UB",
94
+ "useJobCloning": "UC",
95
+ "useRealNumEvents": "UE",
96
+ "useFileAsSourceLFN": "UF",
97
+ "usePrePro": "UP",
98
+ "useScout": "US",
99
+ "usePrefetcher": "UT",
100
+ "useExhausted": "UX",
101
+ "useZipToPin": "UZ",
102
+ "writeInputToFile": "WF",
103
+ "workflowHoldup": "WH",
104
+ "waitInput": "WI",
105
+ "maxAttemptES": "XA",
106
+ "decAttOnFailedES": "XF",
107
+ "maxAttemptEsJob": "XJ",
108
+ "nEventsPerMergeJob": "ZE",
109
+ "nFilesPerMergeJob": "ZF",
110
+ "nGBPerMergeJob": "ZG",
111
+ "nMaxFilesPerMergeJob": "ZM",
112
+ }
113
+
114
+ # changeable split rules
115
+ changeable_split_rule_names = [
116
+ "allowIncompleteInDS",
117
+ "t1Weight",
118
+ "nEsConsumers",
119
+ "nMaxFilesPerJob",
120
+ "nGBPerJob",
121
+ "noInputPooling",
122
+ "nFilesPerJob",
123
+ "nEventsPerWorker",
124
+ "nJumboJobs",
125
+ "avoidVP",
126
+ "allowInputLAN",
127
+ "useLocalIO",
128
+ "noLoopingCheck",
129
+ "maxCoreCount",
130
+ "onlyTagsForFC",
131
+ "useZipToPin",
132
+ "ignoreMissingInDS",
133
+ "noAutoPause",
134
+ ]
135
+
136
+ changeable_split_rule_tags = [split_rule_dict[tmp_name] for tmp_name in changeable_split_rule_names]
137
+
138
+
139
+ # extract rules
140
+ def extract_rule_values(split_rules: str, rule_names: list, is_sub_rule: bool = False) -> dict:
141
+ """
142
+ Extract rule values from split rule string
143
+
144
+ :param split_rules: a comma separated string
145
+ :param rule_names: a list of rule names
146
+ :param is_sub_rule: True to indicate the rule is a subrule
147
+ :return: dict of rule names and values
148
+ """
149
+ if split_rules is None:
150
+ split_rules = ""
151
+ ret = {}
152
+ if is_sub_rule:
153
+ rule_separator = "|"
154
+ key_value_separator = ":"
155
+ else:
156
+ rule_separator = ","
157
+ key_value_separator = "="
158
+ for tmp_rule in split_rules.split(rule_separator):
159
+ for tmp_name in rule_names:
160
+ if tmp_name in split_rule_dict and tmp_rule.startswith(split_rule_dict[tmp_name] + key_value_separator):
161
+ ret[tmp_name] = tmp_rule.split(key_value_separator)[-1]
162
+ break
163
+ for tmp_name in rule_names:
164
+ if tmp_name not in ret:
165
+ ret[tmp_name] = None
166
+ return ret
167
+
168
+
169
+ # replace a rule
170
+ def replace_rule(split_rules: str, rule_name: str, rule_value: int | str, is_sub_rule=False):
171
+ """
172
+ Replace a rule in the split rule string
173
+
174
+ :param split_rules: s comma separated string
175
+ :param rule_name: rule name
176
+ :param rule_value: rule value
177
+ :param is_sub_rule: True to indicate the rule is a subrule
178
+ :return: string of split rules
179
+ """
180
+ if rule_name not in split_rule_dict:
181
+ return split_rules
182
+ if split_rules is None:
183
+ split_rules = ""
184
+ if is_sub_rule:
185
+ rule_separator = "|"
186
+ key_value_separator = ":"
187
+ else:
188
+ rule_separator = ","
189
+ key_value_separator = "="
190
+ tmp_str = ""
191
+ for tmp_rule in split_rules.split(rule_separator):
192
+ if tmp_rule.startswith(split_rule_dict[rule_name] + key_value_separator):
193
+ continue
194
+ if tmp_str != "":
195
+ tmp_str += rule_separator
196
+ tmp_str += tmp_rule
197
+ if tmp_str != "":
198
+ tmp_str += rule_separator
199
+ tmp_str += split_rule_dict[rule_name] + key_value_separator + str(rule_value)
200
+ return tmp_str
201
+
202
+
203
+ # remove a rule
204
+ def remove_rule(split_rules: str, rule_token: str, is_sub_rule: bool = False):
205
+ """
206
+ Remove a rule from the split rule string
207
+
208
+ :param split_rules: s comma separated string
209
+ :param rule_token: rule token
210
+ :param is_sub_rule: TRue to indicate the rule is a subrule
211
+ :return: string of split rules
212
+ """
213
+ if split_rules is None:
214
+ split_rules = ""
215
+ if is_sub_rule:
216
+ rule_separator = "|"
217
+ key_value_separator = ":"
218
+ else:
219
+ rule_separator = ","
220
+ key_value_separator = "="
221
+ tmp_str = ""
222
+ for tmp_rule in split_rules.split(rule_separator):
223
+ if tmp_rule.startswith(rule_token + key_value_separator):
224
+ continue
225
+ if tmp_str != "":
226
+ tmp_str += rule_separator
227
+ tmp_str += tmp_rule
228
+ return tmp_str
229
+
230
+
231
+ # decode split rule string: replace two-letter tags with human-readable names
232
+ def decode_split_rule(split_rules: str) -> str:
233
+ """
234
+ Convert a compact split-rule string into a human-readable one by replacing
235
+ two-letter tags with their full names from split_rule_dict.
236
+
237
+ The input is a comma-separated string of ``TAG=value`` pairs, e.g.::
238
+
239
+ "AE=1,AN=1,2,3,AI=2"
240
+
241
+ Special case: ``AN`` (addNthFieldToLFN) may carry a comma-separated list
242
+ as its value (e.g. ``AN=1,2,3``). All other tags treat ``=`` as the sole
243
+ separator and never embed commas in their values.
244
+
245
+ :param split_rules: compact comma-separated split-rule string, or None/empty
246
+ :return: human-readable comma-separated string, e.g.
247
+ ``"allowEmptyInput=1,addNthFieldToLFN=1,2,3,allowIncompleteInDS=2"``
248
+ Unknown tags are kept as-is.
249
+ """
250
+ if not split_rules:
251
+ return split_rules or ""
252
+
253
+ # build reverse map: two-letter tag -> full name
254
+ tag_to_name = {tag: name for name, tag in split_rule_dict.items()}
255
+
256
+ # The AN tag is the only one whose value may contain commas.
257
+ an_tag = split_rule_dict["addNthFieldToLFN"] # "AN"
258
+
259
+ # Split on commas, then re-group tokens into (tag, value) pairs.
260
+ # A token starts a new rule when it matches "XX=..." where XX is a known
261
+ # two-letter tag. Otherwise, it is appended to the current rule's value.
262
+ tokens = split_rules.split(",")
263
+ rules = [] # list of [tag, value_parts...]
264
+ for token in tokens:
265
+ # Check whether this token begins a new TAG=value entry
266
+ eq_pos = token.find("=")
267
+ if eq_pos == 2:
268
+ candidate_tag = token[:2]
269
+ if candidate_tag in tag_to_name or candidate_tag == an_tag:
270
+ # Start a new rule
271
+ rules.append([candidate_tag, token[3:]]) # token[3:] = value after "XX="
272
+ continue
273
+ # Not a recognized new rule:
274
+ # - if there are existing rules and the last one is AN, treat as value continuation
275
+ # - otherwise keep the token verbatim as its own entry
276
+ if rules and rules[-1][0] == an_tag:
277
+ rules[-1].append(token)
278
+ else:
279
+ # unknown / non-standard token — keep as-is using a sentinel
280
+ rules.append([None, token])
281
+
282
+ # Reconstruct using full names; rejoin multipart AN values with commas
283
+ result_parts = []
284
+ for parts in rules:
285
+ tag = parts[0]
286
+ if tag is None:
287
+ # unknown / non-standard token kept verbatim
288
+ result_parts.append(parts[1])
289
+ else:
290
+ value = "_".join(parts[1:])
291
+ name = tag_to_name.get(tag, tag) # fall back to raw tag if unknown
292
+ result_parts.append(f"{name}={value}")
293
+
294
+ return ",".join(result_parts)
295
+
296
+
297
+ # remove a rule with name
298
+ def remove_rule_with_name(split_rules: str, rule_name: str, is_sub_rule: bool = False) -> str:
299
+ """
300
+ Remove a rule from the split rule string using the rule name
301
+
302
+ :param split_rules: a comma separated string
303
+ :param rule_name: rule name
304
+ :param is_sub_rule: True to indicate the rule is a subrule
305
+ :return: string of split rules
306
+ """
307
+ split_rules = remove_rule(split_rules, split_rule_dict[rule_name], is_sub_rule)
308
+ return split_rules
@@ -0,0 +1,249 @@
1
+ import json
2
+ import os.path
3
+ import sys
4
+ import tempfile
5
+ import traceback
6
+
7
+ import requests
8
+ from idds.client.clientmanager import ClientManager
9
+ from idds.common.utils import get_rest_host
10
+ from pandacommon.pandalogger.LogWrapper import LogWrapper
11
+
12
+ # import PandaLogger before idds modules not to change message levels of other modules
13
+ from pandacommon.pandalogger.PandaLogger import PandaLogger
14
+ from ruamel.yaml import YAML
15
+
16
+ from pandaserver.srvcore.CoreUtils import clean_user_id, commands_get_status_output
17
+ from pandaserver.srvcore.MailUtils import MailUtils
18
+ from pandaserver.workflow import pcwl_utils, workflow_utils
19
+ from pandaserver.workflow.snakeparser import Parser
20
+
21
+ _logger = PandaLogger().getLogger("workflow_processor")
22
+
23
+
24
+ SUPPORTED_WORKFLOW_LANGUAGES = ["cwl", "snakemake"]
25
+
26
+
27
+ # process workflow
28
+ class WorkflowProcessor(object):
29
+ # constructor
30
+ def __init__(self, task_buffer=None, log_stream=None):
31
+ self.taskBuffer = task_buffer
32
+ self.log = _logger
33
+
34
+ # process a file
35
+ def process(
36
+ self,
37
+ file_name,
38
+ to_delete=False,
39
+ test_mode=False,
40
+ get_log=False,
41
+ dump_workflow=False,
42
+ ):
43
+ is_fatal = False
44
+ request_id = None
45
+ dump_str = None
46
+ try:
47
+ with open(file_name) as f:
48
+ ops = json.load(f)
49
+ user_name = clean_user_id(ops["userName"])
50
+ base_platform = ops["data"].get("base_platform")
51
+ for task_type in ops["data"]["taskParams"]:
52
+ ops["data"]["taskParams"][task_type]["userName"] = user_name
53
+ if base_platform:
54
+ ops["data"]["taskParams"][task_type]["basePlatform"] = base_platform
55
+ log_token = f"< id=\"{user_name}\" test={test_mode} outDS={ops['data']['outDS']} >"
56
+ tmpLog = LogWrapper(self.log, log_token)
57
+ tmpLog.info(f"start {file_name}")
58
+ sandbox_url = os.path.join(ops["data"]["sourceURL"], "cache", ops["data"]["sandbox"])
59
+ # IO through json files
60
+ ops_file = tempfile.NamedTemporaryFile(delete=False, mode="w")
61
+ json.dump(ops, ops_file)
62
+ ops_file.close()
63
+ # execute main in another process to avoid chdir mess
64
+ tmp_stat, tmp_out = commands_get_status_output(
65
+ f"python {__file__} {sandbox_url} '{log_token}' {dump_workflow} {ops_file.name} '{user_name}' {test_mode}"
66
+ )
67
+ if tmp_stat:
68
+ is_OK = False
69
+ tmpLog.error(f"main execution failed with {tmp_stat}:{tmp_out}")
70
+ else:
71
+ with open(tmp_out.split("\n")[-1]) as tmp_out_file:
72
+ is_OK, is_fatal, request_id, dump_str = json.load(tmp_out_file)
73
+ try:
74
+ os.remove(tmp_out)
75
+ except Exception:
76
+ pass
77
+ if is_OK:
78
+ tmpLog.info(f"is_OK={is_OK} request_id={request_id}")
79
+ else:
80
+ tmpLog.info(f"is_OK={is_OK} is_fatal={is_fatal} request_id={request_id}")
81
+ if to_delete or (not test_mode and (is_OK or is_fatal)):
82
+ if dump_str:
83
+ dump_str = dump_str + tmpLog.dumpToString()
84
+ else:
85
+ dump_str = tmpLog.dumpToString()
86
+ tmpLog.debug(f"delete {file_name}")
87
+ try:
88
+ os.remove(file_name)
89
+ except Exception:
90
+ pass
91
+ # send notification
92
+ if not test_mode and self.taskBuffer is not None:
93
+ toAdder = self.taskBuffer.getEmailAddr(user_name)
94
+ if toAdder is None or toAdder.startswith("notsend"):
95
+ tmpLog.debug("skip to send notification since suppressed")
96
+ else:
97
+ # message
98
+ if is_OK:
99
+ mailSubject = f"PANDA Notification for Workflow {ops['data']['outDS']}"
100
+ mailBody = f"Hello,\n\nWorkflow:{ops['data']['outDS']} has been accepted with RequestID:{request_id}\n\n"
101
+ else:
102
+ mailSubject = f"PANDA WARNING for Workflow={ops['data']['outDS']}"
103
+ mailBody = "Hello,\n\nWorkflow {} was not accepted\n\n".format(ops["data"]["outDS"], request_id)
104
+ mailBody += f"Reason : {dump_str}\n"
105
+ # send
106
+ tmpSM = MailUtils().send(toAdder, mailSubject, mailBody)
107
+ tmpLog.debug(f"sent message with {tmpSM}")
108
+ except Exception as e:
109
+ is_OK = False
110
+ tmpLog.error(f"failed to run with {str(e)} {traceback.format_exc()}")
111
+ if get_log:
112
+ ret_val = {"status": is_OK, "request_id": request_id}
113
+ if is_OK:
114
+ ret_val["log"] = dump_str
115
+ else:
116
+ if dump_str is None:
117
+ ret_val["log"] = tmpLog.dumpToString()
118
+ else:
119
+ ret_val["log"] = dump_str
120
+ return ret_val
121
+
122
+
123
+ # execute chdir in another process
124
+ def core_exec(sandbox_url, log_token, dump_workflow, ops_file, user_name, test_mode):
125
+ tmpLog = LogWrapper(_logger, log_token)
126
+ is_OK = True
127
+ is_fatal = False
128
+ request_id = None
129
+ if dump_workflow == "True":
130
+ dump_workflow = True
131
+ else:
132
+ dump_workflow = False
133
+ if test_mode == "True":
134
+ test_mode = True
135
+ else:
136
+ test_mode = False
137
+ try:
138
+ with open(ops_file) as f:
139
+ ops = json.load(f)
140
+ try:
141
+ os.remove(ops_file)
142
+ except Exception:
143
+ pass
144
+ # go to temp dir
145
+ cur_dir = os.getcwd()
146
+ with tempfile.TemporaryDirectory() as tmp_dirname:
147
+ os.chdir(tmp_dirname)
148
+ # download sandbox
149
+ tmpLog.info(f"downloading sandbox from {sandbox_url}")
150
+ with requests.get(sandbox_url, allow_redirects=True, verify=False, stream=True) as r:
151
+ if r.status_code == 400:
152
+ tmpLog.error("not found")
153
+ is_fatal = True
154
+ is_OK = False
155
+ elif r.status_code != 200:
156
+ tmpLog.error(f"bad HTTP response {r.status_code}")
157
+ is_OK = False
158
+ # extract sandbox
159
+ if is_OK:
160
+ with open(ops["data"]["sandbox"], "wb") as fs:
161
+ for chunk in r.raw.stream(1024, decode_content=False):
162
+ if chunk:
163
+ fs.write(chunk)
164
+ fs.close()
165
+ tmp_stat, tmp_out = commands_get_status_output(f"tar xvfz {ops['data']['sandbox']}")
166
+ if tmp_stat != 0:
167
+ tmpLog.error(tmp_out)
168
+ dump_str = f"failed to extract {ops['data']['sandbox']}"
169
+ tmpLog.error(dump_str)
170
+ is_fatal = True
171
+ is_OK = False
172
+ # parse workflow files
173
+ if is_OK:
174
+ tmpLog.info("parse workflow")
175
+ workflow_name = None
176
+ if (wf_lang := ops["data"]["language"]) in SUPPORTED_WORKFLOW_LANGUAGES:
177
+ if wf_lang == "cwl":
178
+ workflow_name = ops["data"].get("workflow_name")
179
+ nodes, root_in = pcwl_utils.parse_workflow_file(ops["data"]["workflowSpecFile"], tmpLog)
180
+ with open(ops["data"]["workflowInputFile"]) as workflow_input:
181
+ yaml = YAML(typ="safe", pure=True)
182
+ data = yaml.load(workflow_input)
183
+ elif wf_lang == "snakemake":
184
+ parser = Parser(ops["data"]["workflowSpecFile"], logger=tmpLog)
185
+ nodes, root_in = parser.parse_nodes()
186
+ data = dict()
187
+ # resolve nodes
188
+ s_id, t_nodes, nodes = workflow_utils.resolve_nodes(nodes, root_in, data, 0, set(), ops["data"]["outDS"], tmpLog)
189
+ workflow_utils.set_workflow_outputs(nodes)
190
+ id_node_map = workflow_utils.get_node_id_map(nodes)
191
+ [node.resolve_params(ops["data"]["taskParams"], id_node_map) for node in nodes]
192
+ dump_str = "the description was internally converted as follows\n" + workflow_utils.dump_nodes(nodes)
193
+ tmpLog.info(dump_str)
194
+ for node in nodes:
195
+ s_check, o_check = node.verify()
196
+ tmp_str = f"Verification failure in ID:{node.id} {o_check}"
197
+ if not s_check:
198
+ tmpLog.error(tmp_str)
199
+ dump_str += tmp_str
200
+ dump_str += "\n"
201
+ is_fatal = True
202
+ is_OK = False
203
+ else:
204
+ dump_str = "{} is not supported to describe the workflow"
205
+ tmpLog.error(dump_str)
206
+ is_fatal = True
207
+ is_OK = False
208
+ # convert to workflow
209
+ if is_OK:
210
+ (
211
+ workflow_to_submit,
212
+ dump_str_list,
213
+ ) = workflow_utils.convert_nodes_to_workflow(nodes, workflow_name=workflow_name)
214
+ try:
215
+ if workflow_to_submit:
216
+ if not test_mode:
217
+ tmpLog.info("submit workflow")
218
+ wm = ClientManager(host=get_rest_host())
219
+ request_id = wm.submit(workflow_to_submit, username=user_name, use_dataset_name=False)
220
+ try:
221
+ request_id = int(request_id)
222
+ except Exception:
223
+ # wrong request_id
224
+ is_fatal = True
225
+ is_OK = False
226
+ else:
227
+ dump_str = "workflow is empty"
228
+ tmpLog.error(dump_str)
229
+ is_fatal = True
230
+ is_OK = False
231
+ except Exception as e:
232
+ dump_str = f"failed to submit the workflow with {str(e)}"
233
+ tmpLog.error(f"{dump_str} {traceback.format_exc()}")
234
+ if dump_workflow:
235
+ tmpLog.debug("\n" + "".join(dump_str_list))
236
+ os.chdir(cur_dir)
237
+ except Exception as e:
238
+ is_OK = False
239
+ is_fatal = True
240
+ tmpLog.error(f"failed to run with {str(e)} {traceback.format_exc()}")
241
+
242
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as tmp_json:
243
+ json.dump([is_OK, is_fatal, request_id, tmpLog.dumpToString()], tmp_json)
244
+ print(tmp_json.name)
245
+ sys.exit(0)
246
+
247
+
248
+ if __name__ == "__main__":
249
+ core_exec(*sys.argv[1:])
@@ -0,0 +1,18 @@
1
+ # wrapper for Oracle Connection
2
+ class WrappedOracleConn(object):
3
+ def __init__(self, conn):
4
+ self.orig_conn = conn
5
+
6
+ def __getattribute__(self, item):
7
+ try:
8
+ return object.__getattribute__(self.orig_conn, item)
9
+ except Exception:
10
+ pass
11
+ return object.__getattribute__(self, item)
12
+
13
+ # override context manager protocol not to close connection
14
+ def __enter__(self):
15
+ return self.orig_conn.__enter__()
16
+
17
+ def __exit__(self, exc_type, exc_val, exc_tb):
18
+ self.orig_conn.commit()
@@ -0,0 +1,40 @@
1
+ """
2
+ Checking DB schema version for PanDA Server.
3
+ If there is an issue and the pandaEmailNotification var is
4
+ defined in panda_config, it will send an email notification.
5
+ """
6
+
7
+ from packaging import version
8
+
9
+ from pandaserver.config import panda_config
10
+ from pandaserver.srvcore.MailUtils import MailUtils
11
+ from pandaserver.taskbuffer import PandaDBSchemaInfo
12
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
13
+
14
+ proxyS = DBProxy()
15
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
16
+
17
+ sql = "select major || '.' || minor || '.' || patch from ATLAS_PANDA.pandadb_version where component = 'PanDA'"
18
+
19
+ res = proxyS.querySQL(sql)
20
+ dbVersion = res[0][0]
21
+
22
+ serverDBVersion = PandaDBSchemaInfo.PandaDBSchemaInfo().method()
23
+
24
+ if version.parse(dbVersion) >= version.parse(serverDBVersion):
25
+ print("DB schema check: OK")
26
+ else:
27
+ message_body = (
28
+ f"There is an issue with {panda_config.pserveralias}. "
29
+ f"PanDA DB schema installed is {dbVersion} while PanDA Server requires "
30
+ f"version {serverDBVersion} to be installed. Please check the official docs "
31
+ f"for instructions on how to upgrade the schema."
32
+ )
33
+ print(message_body)
34
+
35
+ if "pandaEmailNotification" in panda_config.__dict__:
36
+ MailUtils().send(
37
+ panda_config.pandaEmailNotification,
38
+ f"PanDA DB Version installed is not correct for {panda_config.pserveralias}",
39
+ message_body,
40
+ )
File without changes
@@ -0,0 +1,43 @@
1
+ import optparse
2
+ import sys
3
+
4
+ # password
5
+ from pandaserver.config import panda_config
6
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
7
+
8
+ option_parser = optparse.OptionParser(conflict_handler="resolve")
9
+ option_parser.add_option("--user", action="store", dest="user", default=None, help="prodUserName")
10
+ option_parser.add_option(
11
+ "--unban",
12
+ action="store_const",
13
+ const=True,
14
+ dest="unban",
15
+ default=False,
16
+ help="unban the user",
17
+ )
18
+
19
+ options, args = option_parser.parse_args()
20
+
21
+ if options.user is None:
22
+ print("--user=<prodUserName> is required")
23
+ sys.exit(1)
24
+
25
+ proxyS = DBProxy()
26
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
27
+
28
+ prodUserName = sys.argv[1]
29
+
30
+ varMap = {}
31
+ varMap[":name"] = options.user
32
+ if options.unban:
33
+ varMap[":status"] = None
34
+ else:
35
+ varMap[":status"] = "disabled"
36
+
37
+ sql = "UPDATE ATLAS_PANDAMETA.users SET status=:status WHERE name=:name"
38
+
39
+ status, res = proxyS.querySQLS(sql, varMap)
40
+ if res is None:
41
+ print("Failed with database error")
42
+ else:
43
+ print(f"{res} rows updated")
@@ -0,0 +1,19 @@
1
+ import sys
2
+
3
+ # password
4
+ from pandaserver.config import panda_config
5
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
6
+
7
+ proxyS = DBProxy()
8
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
9
+
10
+ varMap = {}
11
+ varMap[":prodSourceLabel"] = "managed"
12
+ varMap[":taskID"] = sys.argv[1]
13
+ varMap[":prio"] = sys.argv[2]
14
+ sql = "UPDATE %s SET currentPriority=currentPriority+:prio WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID"
15
+ for table in [
16
+ "ATLAS_PANDA.jobsActive4",
17
+ "ATLAS_PANDA.jobsDefined4",
18
+ ]:
19
+ status, res = proxyS.querySQLS(sql % table, varMap)