panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,80 @@
1
+ import optparse
2
+ import time
3
+
4
+ # password
5
+ from pandaserver.config import panda_config
6
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
7
+ from pandaserver.userinterface import Client
8
+
9
+ option_parser = optparse.OptionParser(conflict_handler="resolve")
10
+ option_parser.add_option(
11
+ "-9",
12
+ action="store_const",
13
+ const=True,
14
+ dest="forceKill",
15
+ default=False,
16
+ help="kill jobs even if they are still running",
17
+ )
18
+ option_parser.add_option(
19
+ "--noRunning",
20
+ action="store_const",
21
+ const=True,
22
+ dest="noRunning",
23
+ default=True,
24
+ help="kill jobs if they are not in running or transferring (ON by default)",
25
+ )
26
+ option_parser.add_option(
27
+ "--killAny",
28
+ action="store_const",
29
+ const=True,
30
+ dest="killAny",
31
+ default=False,
32
+ help="kill jobs in any status",
33
+ )
34
+ option_parser.add_option(
35
+ "--prodSourceLabel",
36
+ action="store",
37
+ dest="prodSourceLabel",
38
+ default="managed",
39
+ help="prodSourceLabel",
40
+ )
41
+
42
+ options, args = option_parser.parse_args()
43
+
44
+ proxyS = DBProxy()
45
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
46
+
47
+ jobs = []
48
+
49
+ varMap = {}
50
+ varMap[":prodSourceLabel"] = options.prodSourceLabel
51
+ varMap[":taskID"] = args[0]
52
+ if not options.noRunning or options.killAny:
53
+ sql = "SELECT PandaID FROM %s WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID ORDER BY PandaID"
54
+ else:
55
+ sql = "SELECT PandaID FROM %s WHERE prodSourceLabel=:prodSourceLabel AND taskID=:taskID AND NOT jobStatus IN (:js1,:js2,:js3) ORDER BY PandaID"
56
+ varMap[":js1"] = "running"
57
+ varMap[":js2"] = "transferring"
58
+ varMap[":js3"] = "holding"
59
+ for table in [
60
+ "ATLAS_PANDA.jobsActive4",
61
+ "ATLAS_PANDA.jobsDefined4",
62
+ ]:
63
+ status, res = proxyS.querySQLS(sql % table, varMap)
64
+ if res is not None:
65
+ for (id,) in res:
66
+ if id not in jobs:
67
+ jobs.append(id)
68
+
69
+ print(f"The number of jobs to be killed for prodSourceLabel={options.prodSourceLabel} taskID={args[0]}: {len(jobs)}")
70
+ if len(jobs):
71
+ nJob = 100
72
+ iJob = 0
73
+ while iJob < len(jobs):
74
+ print(f"kill {str(jobs[iJob:iJob + nJob])}")
75
+ if options.forceKill:
76
+ Client.kill_jobs(jobs[iJob : iJob + nJob], 9)
77
+ else:
78
+ Client.kill_jobs(jobs[iJob : iJob + nJob])
79
+ iJob += nJob
80
+ time.sleep(1)
@@ -0,0 +1,11 @@
1
+ import optparse
2
+
3
+ from pandaserver.userinterface import Client
4
+
5
+ option_parser = optparse.OptionParser(conflict_handler="resolve")
6
+ options, args = option_parser.parse_args()
7
+
8
+ task_id = int(args[0])
9
+
10
+ s, o = Client.kill_task(task_id)
11
+ print(o)
@@ -0,0 +1,83 @@
1
+ import optparse
2
+ import sys
3
+
4
+ # password
5
+ from pandaserver.config import panda_config
6
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
7
+ from pandaserver.userinterface import Client
8
+
9
+ option_parser = optparse.OptionParser(conflict_handler="resolve")
10
+ option_parser.add_option("--user", action="store", dest="user", default=None, help="prodUserName")
11
+ option_parser.add_option("--jobID", action="store", dest="jobID", default=None, help="jobDefinitionID")
12
+ option_parser.add_option(
13
+ "--jobsetID",
14
+ action="store",
15
+ dest="jobsetID",
16
+ default=None,
17
+ help="jobsetID, or 'all' to kill all jobs",
18
+ )
19
+ option_parser.add_option(
20
+ "--prodSourceLabel",
21
+ action="store",
22
+ dest="prodSourceLabel",
23
+ default=None,
24
+ help="additional prodSourceLabel",
25
+ )
26
+
27
+
28
+ options, args = option_parser.parse_args()
29
+
30
+ if options.user is None:
31
+ print("--user=<prodUserName> is required")
32
+ sys.exit(1)
33
+ if options.jobID is None and options.jobsetID is None:
34
+ print("--jobID=<jobDefinitionID> or --jobsetID=<jobsetID or 'all'> is required")
35
+ sys.exit(1)
36
+
37
+
38
+ proxyS = DBProxy()
39
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
40
+
41
+ prodUserName = sys.argv[1]
42
+
43
+ varMap = {}
44
+ varMap[":src1"] = "user"
45
+ varMap[":src2"] = "panda"
46
+ varMap[":prodUserName"] = options.user
47
+ srcSQL = "(:src1,:src2"
48
+ if options.jobID is not None:
49
+ varMap[":jobDefinitionID"] = options.jobID
50
+ if options.jobsetID not in (None, "all"):
51
+ varMap[":jobsetID"] = options.jobsetID
52
+ if options.prodSourceLabel is not None:
53
+ varMap[":src3"] = options.prodSourceLabel
54
+ srcSQL += ",:src3"
55
+ srcSQL += ")"
56
+
57
+ jobs = []
58
+ tables = [
59
+ "ATLAS_PANDA.jobsActive4",
60
+ "ATLAS_PANDA.jobsDefined4",
61
+ ]
62
+ for table in tables:
63
+ sql = f"SELECT PandaID FROM {table} WHERE prodUserName=:prodUserName AND prodSourceLabel IN {srcSQL} "
64
+ if options.jobID is not None:
65
+ sql += "AND jobDefinitionID=:jobDefinitionID "
66
+ if options.jobsetID not in (None, "all"):
67
+ sql += "AND jobsetID=:jobsetID "
68
+ sql += "ORDER BY PandaID "
69
+ status, res = proxyS.querySQLS(sql, varMap)
70
+ if res is not None:
71
+ for (id,) in res:
72
+ if id not in jobs:
73
+ jobs.append(id)
74
+ if len(jobs):
75
+ iJob = 0
76
+ nJob = 1000
77
+ while iJob < len(jobs):
78
+ subJobs = jobs[iJob : iJob + nJob]
79
+ print(f"kill {str(subJobs)} {iJob}/{len(jobs)}")
80
+ Client.kill_jobs(subJobs, code=9)
81
+ iJob += nJob
82
+ else:
83
+ print("no job was killed")
@@ -0,0 +1,7 @@
1
+ 1. Configure your environment
2
+ export PANDA_URL="http://pandawms.org:25080/server/panda"
3
+ export PANDA_URL_SSL="https://pandawms.org:25443/server/panda"
4
+ voms-proxy-init -voms lsst:/lsst
5
+
6
+ 2. Submit a job
7
+ python lsstSubmit.py
@@ -0,0 +1,139 @@
1
+ import sys
2
+ import time
3
+ import uuid
4
+
5
+ from pandaserver.taskbuffer.FileSpec import FileSpec
6
+ from pandaserver.taskbuffer.JobSpec import JobSpec
7
+ from pandaserver.userinterface import Client
8
+
9
+ prodUserNameDefault = "unknown-user"
10
+ prodUserName = None
11
+ prodUserNameDP = None
12
+ prodUserNamePipeline = None
13
+ site = "ANALY_BNL-LSST"
14
+ PIPELINE_TASK = None
15
+ PIPELINE_PROCESSINSTANCE = None
16
+ PIPELINE_EXECUTIONNUMBER = None
17
+ PIPELINE_STREAM = None
18
+ lsstJobParams = ""
19
+
20
+ for idx, argv in enumerate(sys.argv):
21
+ if argv == "--site":
22
+ try:
23
+ site = sys.argv[idx + 1]
24
+ except Exception:
25
+ site = "ANALY_BNL-LSST"
26
+ if argv == "-DP_USER":
27
+ try:
28
+ prodUserNameDP = sys.argv[idx + 1]
29
+ if len(lsstJobParams):
30
+ lsstJobParams += "|"
31
+ lsstJobParams += f"DP_USER={str(prodUserNameDP)}"
32
+ except Exception:
33
+ prodUserNameDP = None
34
+ if argv == "-PIPELINE_USER":
35
+ try:
36
+ prodUserNamePipeline = sys.argv[idx + 1]
37
+ if len(lsstJobParams):
38
+ lsstJobParams += "|"
39
+ lsstJobParams += f"PIPELINE_USER={str(prodUserNamePipeline)}"
40
+ except Exception:
41
+ prodUserNamePipeline = None
42
+ if argv == "-PIPELINE_TASK":
43
+ try:
44
+ PIPELINE_TASK = sys.argv[idx + 1]
45
+ if len(lsstJobParams):
46
+ lsstJobParams += "|"
47
+ lsstJobParams += f"PIPELINE_TASK={str(PIPELINE_TASK)}"
48
+ except Exception:
49
+ PIPELINE_TASK = None
50
+ if argv == "-PIPELINE_PROCESSINSTANCE":
51
+ try:
52
+ PIPELINE_PROCESSINSTANCE = int(sys.argv[idx + 1])
53
+ if len(lsstJobParams):
54
+ lsstJobParams += "|"
55
+ lsstJobParams += f"PIPELINE_PROCESSINSTANCE={str(PIPELINE_PROCESSINSTANCE)}"
56
+ except Exception:
57
+ PIPELINE_PROCESSINSTANCE = None
58
+ if argv == "-PIPELINE_EXECUTIONNUMBER":
59
+ try:
60
+ PIPELINE_EXECUTIONNUMBER = int(sys.argv[idx + 1])
61
+ if len(lsstJobParams):
62
+ lsstJobParams += "|"
63
+ lsstJobParams += f"PIPELINE_EXECUTIONNUMBER={str(PIPELINE_EXECUTIONNUMBER)}"
64
+ except Exception:
65
+ PIPELINE_EXECUTIONNUMBER = None
66
+ if argv == "-PIPELINE_STREAM":
67
+ try:
68
+ PIPELINE_STREAM = int(sys.argv[idx + 1])
69
+ if len(lsstJobParams):
70
+ lsstJobParams += "|"
71
+ lsstJobParams += f"PIPELINE_STREAM={str(PIPELINE_STREAM)}"
72
+ except Exception:
73
+ PIPELINE_STREAM = None
74
+
75
+ # DP_USER and PIPELINE_USER preference
76
+ if prodUserNameDP is not None:
77
+ prodUserName = prodUserNameDP
78
+ elif prodUserNamePipeline is not None:
79
+ prodUserName = prodUserNamePipeline
80
+
81
+
82
+ # site = sys.argv[1]
83
+ # site = 'ANALY_BNL-LSST' #orig
84
+ # site = 'BNL-LSST'
85
+ # site = 'SWT2_CPB-LSST'
86
+ # site = 'UTA_SWT2-LSST'
87
+ # site = 'ANALY_SWT2_CPB-LSST'
88
+
89
+ destName = None
90
+
91
+ if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_PROCESSINSTANCE is not None:
92
+ datasetName = f"panda.lsst.user.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(prodUserName)}"
93
+ else:
94
+ datasetName = f"panda.lsst.user.jschovan.{str(uuid.uuid4())}"
95
+
96
+ if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_EXECUTIONNUMBER is not None and PIPELINE_STREAM is not None:
97
+ jobName = f"job.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(PIPELINE_EXECUTIONNUMBER)}.{str(prodUserName)}.{str(PIPELINE_STREAM)}"
98
+ else:
99
+ jobName = f"{str(uuid.uuid4())}"
100
+
101
+ if PIPELINE_STREAM is not None:
102
+ jobDefinitionID = PIPELINE_STREAM
103
+ else:
104
+ jobDefinitionID = int(time.time()) % 10000
105
+ job = JobSpec()
106
+ job.jobDefinitionID = jobDefinitionID
107
+ job.jobName = jobName
108
+ job.transformation = "http://pandawms.org/pandawms-jobcache/lsst-trf.sh"
109
+ job.destinationDBlock = datasetName
110
+ job.destinationSE = "local"
111
+ job.currentPriority = 1000
112
+ job.prodSourceLabel = "panda"
113
+ job.jobParameters = f' --lsstJobParams="{lsstJobParams}" '
114
+ if prodUserName is not None:
115
+ job.prodUserName = prodUserName
116
+ else:
117
+ job.prodUserName = prodUserNameDefault
118
+ if PIPELINE_PROCESSINSTANCE is not None:
119
+ job.taskID = PIPELINE_PROCESSINSTANCE
120
+ if PIPELINE_EXECUTIONNUMBER is not None:
121
+ job.attemptNr = PIPELINE_EXECUTIONNUMBER
122
+ if PIPELINE_TASK is not None:
123
+ job.processingType = PIPELINE_TASK
124
+ job.computingSite = site
125
+ job.VO = "lsst"
126
+
127
+ fileOL = FileSpec()
128
+ fileOL.lfn = f"{job.jobName}.job.log.tgz"
129
+ fileOL.destinationDBlock = job.destinationDBlock
130
+ fileOL.destinationSE = job.destinationSE
131
+ fileOL.dataset = job.destinationDBlock
132
+ fileOL.type = "log"
133
+ job.addFile(fileOL)
134
+
135
+
136
+ s, o = Client.submit_jobs([job])
137
+ print(s)
138
+ for x in o:
139
+ print(f"PandaID={x[0]}")
@@ -0,0 +1,141 @@
1
+ import sys
2
+ import time
3
+ import uuid
4
+
5
+ from pandaserver.taskbuffer.FileSpec import FileSpec
6
+ from pandaserver.taskbuffer.JobSpec import JobSpec
7
+ from pandaserver.userinterface import Client
8
+
9
+ prodUserNameDefault = "jschovan"
10
+ prodUserName = None
11
+ prodUserNameDP = None
12
+ prodUserNamePipeline = None
13
+ # site = 'ANALY_BNL-LSST-OSGRPMtest'
14
+ # site = 'ANALY_BNL-LSST-gridgk01-mergetest'
15
+ site = "ANALY_BNL-LSST-mergetest"
16
+ PIPELINE_TASK = None
17
+ PIPELINE_PROCESSINSTANCE = None
18
+ PIPELINE_EXECUTIONNUMBER = None
19
+ PIPELINE_STREAM = None
20
+ lsstJobParams = ""
21
+
22
+ for idx, argv in enumerate(sys.argv):
23
+ if argv == "--site":
24
+ try:
25
+ site = sys.argv[idx + 1]
26
+ except Exception:
27
+ site = "ANALY_BNL-LSST-mergetest"
28
+ if argv == "-DP_USER":
29
+ try:
30
+ prodUserNameDP = sys.argv[idx + 1]
31
+ if len(lsstJobParams):
32
+ lsstJobParams += "|"
33
+ lsstJobParams += f"DP_USER={str(prodUserNameDP)}"
34
+ except Exception:
35
+ prodUserNameDP = None
36
+ if argv == "-PIPELINE_USER":
37
+ try:
38
+ prodUserNamePipeline = sys.argv[idx + 1]
39
+ if len(lsstJobParams):
40
+ lsstJobParams += "|"
41
+ lsstJobParams += f"PIPELINE_USER={str(prodUserNamePipeline)}"
42
+ except Exception:
43
+ prodUserNamePipeline = None
44
+ if argv == "-PIPELINE_TASK":
45
+ try:
46
+ PIPELINE_TASK = sys.argv[idx + 1]
47
+ if len(lsstJobParams):
48
+ lsstJobParams += "|"
49
+ lsstJobParams += f"PIPELINE_TASK={str(PIPELINE_TASK)}"
50
+ except Exception:
51
+ PIPELINE_TASK = None
52
+ if argv == "-PIPELINE_PROCESSINSTANCE":
53
+ try:
54
+ PIPELINE_PROCESSINSTANCE = int(sys.argv[idx + 1])
55
+ if len(lsstJobParams):
56
+ lsstJobParams += "|"
57
+ lsstJobParams += f"PIPELINE_PROCESSINSTANCE={str(PIPELINE_PROCESSINSTANCE)}"
58
+ except Exception:
59
+ PIPELINE_PROCESSINSTANCE = None
60
+ if argv == "-PIPELINE_EXECUTIONNUMBER":
61
+ try:
62
+ PIPELINE_EXECUTIONNUMBER = int(sys.argv[idx + 1])
63
+ if len(lsstJobParams):
64
+ lsstJobParams += "|"
65
+ lsstJobParams += f"PIPELINE_EXECUTIONNUMBER={str(PIPELINE_EXECUTIONNUMBER)}"
66
+ except Exception:
67
+ PIPELINE_EXECUTIONNUMBER = None
68
+ if argv == "-PIPELINE_STREAM":
69
+ try:
70
+ PIPELINE_STREAM = int(sys.argv[idx + 1])
71
+ if len(lsstJobParams):
72
+ lsstJobParams += "|"
73
+ lsstJobParams += f"PIPELINE_STREAM={str(PIPELINE_STREAM)}"
74
+ except Exception:
75
+ PIPELINE_STREAM = None
76
+
77
+ # DP_USER and PIPELINE_USER preference
78
+ if prodUserNameDP is not None:
79
+ prodUserName = prodUserNameDP
80
+ elif prodUserNamePipeline is not None:
81
+ prodUserName = prodUserNamePipeline
82
+
83
+
84
+ # site = sys.argv[1]
85
+ # site = 'ANALY_BNL-LSST' #orig
86
+ # site = 'BNL-LSST'
87
+ # site = 'SWT2_CPB-LSST'
88
+ # site = 'UTA_SWT2-LSST'
89
+ # site = 'ANALY_SWT2_CPB-LSST'
90
+
91
+ destName = None
92
+
93
+ if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_PROCESSINSTANCE is not None:
94
+ datasetName = f"panda.lsst.user.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(prodUserName)}"
95
+ else:
96
+ datasetName = f"panda.lsst.user.jschovan.{str(uuid.uuid4())}"
97
+
98
+ if prodUserName is not None and PIPELINE_TASK is not None and PIPELINE_EXECUTIONNUMBER is not None and PIPELINE_STREAM is not None:
99
+ jobName = f"job.{str(PIPELINE_PROCESSINSTANCE)}.{str(PIPELINE_TASK)}.{str(PIPELINE_EXECUTIONNUMBER)}.{str(prodUserName)}.{str(PIPELINE_STREAM)}"
100
+ else:
101
+ jobName = f"{str(uuid.uuid4())}"
102
+
103
+ if PIPELINE_STREAM is not None:
104
+ jobDefinitionID = PIPELINE_STREAM
105
+ else:
106
+ jobDefinitionID = int(time.time()) % 10000
107
+ job = JobSpec()
108
+ job.jobDefinitionID = jobDefinitionID
109
+ job.jobName = jobName
110
+ job.transformation = "http://rpm-test.pandawms.org/pandawms-jobcache/lsst-trf.sh"
111
+ job.destinationDBlock = datasetName
112
+ job.destinationSE = "local"
113
+ job.currentPriority = 1000
114
+ job.prodSourceLabel = "panda"
115
+ job.jobParameters = f' --lsstJobParams="{lsstJobParams}" '
116
+ if prodUserName is not None:
117
+ job.prodUserName = prodUserName
118
+ else:
119
+ job.prodUserName = prodUserNameDefault
120
+ if PIPELINE_PROCESSINSTANCE is not None:
121
+ job.taskID = PIPELINE_PROCESSINSTANCE
122
+ if PIPELINE_EXECUTIONNUMBER is not None:
123
+ job.attemptNr = PIPELINE_EXECUTIONNUMBER
124
+ if PIPELINE_TASK is not None:
125
+ job.processingType = PIPELINE_TASK
126
+ job.computingSite = site
127
+ job.VO = "lsst"
128
+
129
+ fileOL = FileSpec()
130
+ fileOL.lfn = f"{job.jobName}.job.log.tgz"
131
+ fileOL.destinationDBlock = job.destinationDBlock
132
+ fileOL.destinationSE = job.destinationSE
133
+ fileOL.dataset = job.destinationDBlock
134
+ fileOL.type = "log"
135
+ job.addFile(fileOL)
136
+
137
+
138
+ s, o = Client.submit_jobs([job])
139
+ print(s)
140
+ for x in o:
141
+ print(f"PandaID={x[0]}")
@@ -0,0 +1,53 @@
1
+ import sys
2
+ import time
3
+ import uuid
4
+
5
+ from pandaserver.taskbuffer.FileSpec import FileSpec
6
+ from pandaserver.taskbuffer.JobSpec import JobSpec
7
+ from pandaserver.userinterface import Client
8
+
9
+ # site = sys.argv[1]
10
+ site = "ANALY_BNL-LSST" # orig
11
+ # site = 'BNL-LSST'
12
+ # site = 'SWT2_CPB-LSST'
13
+ # site = 'UTA_SWT2-LSST'
14
+ # site = 'ANALY_SWT2_CPB-LSST'
15
+
16
+ datasetName = f"panda.user.jschovan.lsst.{str(uuid.uuid4())}"
17
+ destName = None
18
+
19
+ job = JobSpec()
20
+ job.jobDefinitionID = int(time.time()) % 10000
21
+ job.jobName = f"{str(uuid.uuid4())}"
22
+ # job.transformation = 'http://www.usatlas.bnl.gov/~wenaus/lsst-trf/lsst-trf.sh'
23
+ # job.transformation = 'http://pandawms.org/pandawms-jobcache/lsst-trf.sh'
24
+ job.transformation = "http://pandawms.org/pandawms-jobcache/lsst-trf-phosim332.sh"
25
+ job.destinationDBlock = datasetName
26
+ # job.destinationSE = destName
27
+ job.destinationSE = "local"
28
+ job.currentPriority = 1000
29
+ # job.prodSourceLabel = 'ptest'
30
+ # job.prodSourceLabel = 'panda'
31
+ # job.prodSourceLabel = 'ptest'
32
+ # job.prodSourceLabel = 'test'
33
+ # job.prodSourceLabel = 'ptest'
34
+ # 2014-01-27
35
+ # job.prodSourceLabel = 'user'
36
+ job.prodSourceLabel = "panda"
37
+ job.computingSite = site
38
+ job.jobParameters = ""
39
+ job.VO = "lsst"
40
+
41
+ fileOL = FileSpec()
42
+ fileOL.lfn = f"{job.jobName}.job.log.tgz"
43
+ fileOL.destinationDBlock = job.destinationDBlock
44
+ fileOL.destinationSE = job.destinationSE
45
+ fileOL.dataset = job.destinationDBlock
46
+ fileOL.type = "log"
47
+ job.addFile(fileOL)
48
+
49
+
50
+ s, o = Client.submit_jobs([job])
51
+ print(s)
52
+ for x in o:
53
+ print(f"PandaID={x[0]}")
@@ -0,0 +1,41 @@
1
+ #!/bin/bash
2
+ REPOSITORY="https://github.com/PanDAWMS/panda-server.git"
3
+ BRANCH="mysql"
4
+ WRKDIR="/tmp/${USER}/bigpandaclient"
5
+
6
+
7
+ mkdir -p ${WRKDIR}
8
+ if [ ! -d ${WRKDIR} ]; then
9
+ WRKDIR="/tmp"
10
+ fi
11
+
12
+ cd ${WRKDIR}
13
+ TARBALLNAMEBASE=bigpanda-client-mysql-0.0.1
14
+ TARBALLNAME=${TARBALLNAMEBASE}.tar.gz
15
+ TARBALLDIR=${WRKDIR}/${TARBALLNAMEBASE}
16
+ TARBALL=${WRKDIR}/${TARBALLNAME}
17
+ mkdir -p ${TARBALLDIR}
18
+ GITREPODIRNAME=panda-server
19
+ GITREPODIR=${WRKDIR}/${GITREPODIRNAME}
20
+ git clone -b ${BRANCH} ${REPOSITORY} ${GITREPODIRNAME}
21
+
22
+ cd ${GITREPODIR}/pandaserver
23
+ mkdir -p ${TARBALLDIR}/{taskbuffer,userinterface,test}
24
+
25
+ cp __init__.py ${TARBALLDIR}
26
+ cp taskbuffer/{__init__,JobSpec,FileSpec}.py ${TARBALLDIR}/taskbuffer/
27
+ cp userinterface/*.py ${TARBALLDIR}/userinterface/
28
+ cp test/lsst/*.py ${TARBALLDIR}/test/
29
+ cp ${GITREPODIR}/bigpanda-client/README.txt ${TARBALLDIR}/
30
+
31
+ cd ${WRKDIR}/
32
+ tar zcf ${TARBALL} ${TARBALLNAMEBASE}
33
+
34
+ if [ $? == 0 ]; then
35
+ echo "BigPanDA:MySQL client is prepared in ${TARBALL}."
36
+ echo "Please upload it to the web repository."
37
+ echo "Please clean up ${WRKDIR} afterwards."
38
+ fi
39
+
40
+
41
+
@@ -0,0 +1,17 @@
1
+ import sys
2
+
3
+ from pandaserver.userinterface import Client
4
+
5
+ if len(sys.argv) == 2:
6
+ job_id = int(sys.argv[1])
7
+ ret = Client.reassign_jobs([job_id])
8
+ else:
9
+ job_id_start = int(sys.argv[1])
10
+ job_id_end = int(sys.argv[2])
11
+ if job_id_start > job_id_end:
12
+ print(f"{job_id_end} is less than {job_id_start}")
13
+ sys.exit(1)
14
+
15
+ ret = Client.reassign_jobs(range(job_id_start, job_id_end + 1))
16
+
17
+ print(ret)
@@ -0,0 +1,82 @@
1
+ import datetime
2
+ import optparse
3
+ import re
4
+
5
+ from pandacommon.pandautils.PandaUtils import naive_utcnow
6
+
7
+ from pandaserver.config import panda_config
8
+ from pandaserver.taskbuffer.OraDBProxy import DBProxy
9
+ from pandaserver.userinterface import Client
10
+
11
+ usage = "%prog [options] siteName"
12
+ option_parser = optparse.OptionParser(usage=usage, conflict_handler="resolve")
13
+ option_parser.add_option(
14
+ "--assigned",
15
+ action="store_const",
16
+ const=True,
17
+ dest="assigned",
18
+ default=False,
19
+ help="reassign jobs in assigned state. Jobs in activated state are reassigned by default",
20
+ )
21
+ option_parser.add_option(
22
+ "--olderThan",
23
+ action="store",
24
+ dest="olderThan",
25
+ default=1,
26
+ help="reassign jobs with modificationTime older than N hours (1 by default)",
27
+ )
28
+ options, args = option_parser.parse_args()
29
+
30
+ # password
31
+
32
+ proxyS = DBProxy()
33
+ proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname)
34
+
35
+ site = args[0]
36
+
37
+ try:
38
+ options.olderThan = int(options.olderThan)
39
+ except Exception:
40
+ pass
41
+ timeLimit = naive_utcnow() - datetime.timedelta(hours=options.olderThan)
42
+ varMap = {}
43
+ if options.assigned:
44
+ varMap[":jobStatus"] = "assigned"
45
+ else:
46
+ varMap[":jobStatus"] = "activated"
47
+ varMap[":modificationTime"] = timeLimit
48
+ varMap[":prodSourceLabel"] = "managed"
49
+ varMap[":computingSite"] = site
50
+ if options.assigned:
51
+ sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsDefined4 "
52
+ else:
53
+ sql = "SELECT PandaID,lockedby FROM ATLAS_PANDA.jobsActive4 "
54
+ sql += (
55
+ "WHERE jobStatus=:jobStatus AND computingSite=:computingSite AND modificationTime<:modificationTime AND prodSourceLabel=:prodSourceLabel ORDER BY PandaID"
56
+ )
57
+ status, res = proxyS.querySQLS(sql, varMap)
58
+
59
+ print(f"got {len(res)} jobs")
60
+
61
+ jobs = []
62
+ jediJobs = []
63
+ if res is not None:
64
+ for id, lockedby in res:
65
+ if lockedby == "jedi":
66
+ jediJobs.append(id)
67
+ else:
68
+ jobs.append(id)
69
+ if len(jobs):
70
+ nJob = 100
71
+ iJob = 0
72
+ while iJob < len(jobs):
73
+ print(f"reassign {str(jobs[iJob:iJob + nJob])}")
74
+ Client.reassign_jobs(jobs[iJob : iJob + nJob])
75
+ iJob += nJob
76
+ if len(jediJobs) != 0:
77
+ nJob = 100
78
+ iJob = 0
79
+ while iJob < len(jediJobs):
80
+ print(f"kill JEDI jobs {str(jediJobs[iJob:iJob + nJob])}")
81
+ Client.kill_jobs(jediJobs[iJob : iJob + nJob], 51)
82
+ iJob += nJob