panda-server-source 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. panda_server_source-0.8.0.dist-info/METADATA +55 -0
  2. panda_server_source-0.8.0.dist-info/RECORD +375 -0
  3. panda_server_source-0.8.0.dist-info/WHEEL +4 -0
  4. panda_server_source-0.8.0.dist-info/licenses/LICENSE.txt +202 -0
  5. pandajedi/__init__.py +0 -0
  6. pandajedi/jedibrokerage/AtlasAnalJobBroker.py +2343 -0
  7. pandajedi/jedibrokerage/AtlasBrokerUtils.py +1292 -0
  8. pandajedi/jedibrokerage/AtlasProdJobBroker.py +2064 -0
  9. pandajedi/jedibrokerage/AtlasProdTaskBroker.py +629 -0
  10. pandajedi/jedibrokerage/GenJobBroker.py +381 -0
  11. pandajedi/jedibrokerage/JobBrokerBase.py +133 -0
  12. pandajedi/jedibrokerage/TaskBrokerBase.py +15 -0
  13. pandajedi/jedibrokerage/__init__.py +0 -0
  14. pandajedi/jediconfig/__init__.py +0 -0
  15. pandajedi/jediconfig/jedi_config.py +40 -0
  16. pandajedi/jedicore/FactoryBase.py +182 -0
  17. pandajedi/jedicore/Interaction.py +433 -0
  18. pandajedi/jedicore/JediDBProxy.py +56 -0
  19. pandajedi/jedicore/JediDBProxyPool.py +37 -0
  20. pandajedi/jedicore/JediDBSchemaInfo.py +17 -0
  21. pandajedi/jedicore/JediException.py +13 -0
  22. pandajedi/jedicore/JediTaskBuffer.py +984 -0
  23. pandajedi/jedicore/JediTaskBufferInterface.py +50 -0
  24. pandajedi/jedicore/MsgWrapper.py +124 -0
  25. pandajedi/jedicore/ProcessUtils.py +34 -0
  26. pandajedi/jedicore/SiteCandidate.py +90 -0
  27. pandajedi/jedicore/ThreadUtils.py +205 -0
  28. pandajedi/jedicore/__init__.py +0 -0
  29. pandajedi/jediddm/AtlasDDMClient.py +1708 -0
  30. pandajedi/jediddm/DDMClientBase.py +16 -0
  31. pandajedi/jediddm/DDMInterface.py +83 -0
  32. pandajedi/jediddm/GenDDMClient.py +17 -0
  33. pandajedi/jediddm/__init__.py +0 -0
  34. pandajedi/jedidog/AtlasAnalWatchDog.py +648 -0
  35. pandajedi/jedidog/AtlasDataCarouselWatchDog.py +189 -0
  36. pandajedi/jedidog/AtlasDataLocalityUpdaterWatchDog.py +215 -0
  37. pandajedi/jedidog/AtlasProdWatchDog.py +443 -0
  38. pandajedi/jedidog/AtlasQueueFillerWatchDog.py +800 -0
  39. pandajedi/jedidog/AtlasTaskWithholderWatchDog.py +245 -0
  40. pandajedi/jedidog/AtlasWorkflowManagerWatchDog.py +63 -0
  41. pandajedi/jedidog/GenWatchDog.py +21 -0
  42. pandajedi/jedidog/JumboWatchDog.py +147 -0
  43. pandajedi/jedidog/TypicalWatchDogBase.py +122 -0
  44. pandajedi/jedidog/WatchDogBase.py +53 -0
  45. pandajedi/jedidog/__init__.py +0 -0
  46. pandajedi/jediexec/reniceJEDI.py +25 -0
  47. pandajedi/jedigen/AtlasTaskGenerator.py +110 -0
  48. pandajedi/jedigen/TaskGeneratorBase.py +15 -0
  49. pandajedi/jedigen/__init__.py +0 -0
  50. pandajedi/jedimsgprocessor/__init__.py +0 -0
  51. pandajedi/jedimsgprocessor/ai_parameter_adjuster_msg_processor +71 -0
  52. pandajedi/jedimsgprocessor/atlas_idds_msg_processor.py +66 -0
  53. pandajedi/jedimsgprocessor/base_msg_processor.py +33 -0
  54. pandajedi/jedimsgprocessor/forwarding_msg_processor.py +29 -0
  55. pandajedi/jedimsgprocessor/hpo_msg_processor.py +93 -0
  56. pandajedi/jedimsgprocessor/jedi_contents_feeder_msg_processor.py +63 -0
  57. pandajedi/jedimsgprocessor/jedi_job_generator_msg_processor.py +185 -0
  58. pandajedi/jedimsgprocessor/jedi_post_processor_msg_processor.py +82 -0
  59. pandajedi/jedimsgprocessor/kafka_msg_processor.py +44 -0
  60. pandajedi/jedimsgprocessor/panda_callback_msg_processor.py +182 -0
  61. pandajedi/jedimsgprocessor/panda_to_jedi_msg_processor.py +74 -0
  62. pandajedi/jedimsgprocessor/processing_msg_processor.py +134 -0
  63. pandajedi/jedimsgprocessor/status_report_msg_processor.py +129 -0
  64. pandajedi/jedimsgprocessor/tape_carousel_msg_processor.py +118 -0
  65. pandajedi/jedimsgprocessor/workflow_manager_msg_processor.py +89 -0
  66. pandajedi/jediorder/ContentsFeeder.py +785 -0
  67. pandajedi/jediorder/JediDaemon.py +27 -0
  68. pandajedi/jediorder/JediKnight.py +55 -0
  69. pandajedi/jediorder/JediMaster.py +238 -0
  70. pandajedi/jediorder/JediMsgProcessor.py +32 -0
  71. pandajedi/jediorder/JobBroker.py +33 -0
  72. pandajedi/jediorder/JobGenerator.py +2763 -0
  73. pandajedi/jediorder/JobSplitter.py +308 -0
  74. pandajedi/jediorder/JobThrottler.py +28 -0
  75. pandajedi/jediorder/PostProcessor.py +166 -0
  76. pandajedi/jediorder/TaskBroker.py +268 -0
  77. pandajedi/jediorder/TaskCommando.py +334 -0
  78. pandajedi/jediorder/TaskGenerator.py +17 -0
  79. pandajedi/jediorder/TaskRefiner.py +611 -0
  80. pandajedi/jediorder/TaskSetupper.py +17 -0
  81. pandajedi/jediorder/WatchDog.py +79 -0
  82. pandajedi/jediorder/__init__.py +0 -0
  83. pandajedi/jedipprocess/AtlasAnalPostProcessor.py +404 -0
  84. pandajedi/jedipprocess/AtlasPostProcessorUtils.py +50 -0
  85. pandajedi/jedipprocess/AtlasProdPostProcessor.py +178 -0
  86. pandajedi/jedipprocess/GenPostProcessor.py +35 -0
  87. pandajedi/jedipprocess/MailTemplates.py +185 -0
  88. pandajedi/jedipprocess/PostProcessorBase.py +269 -0
  89. pandajedi/jedipprocess/__init__.py +0 -0
  90. pandajedi/jedirefine/AtlasAnalTaskRefiner.py +259 -0
  91. pandajedi/jedirefine/AtlasProdTaskRefiner.py +162 -0
  92. pandajedi/jedirefine/GenTaskRefiner.py +61 -0
  93. pandajedi/jedirefine/RefinerUtils.py +154 -0
  94. pandajedi/jedirefine/TaskRefinerBase.py +996 -0
  95. pandajedi/jedirefine/__init__.py +0 -0
  96. pandajedi/jedisetup/AtlasTaskSetupper.py +309 -0
  97. pandajedi/jedisetup/GenTaskSetupper.py +16 -0
  98. pandajedi/jedisetup/SimpleTaskSetupper.py +152 -0
  99. pandajedi/jedisetup/TaskSetupperBase.py +15 -0
  100. pandajedi/jedisetup/__init__.py +0 -0
  101. pandajedi/jeditest/SchemaChecker.py +41 -0
  102. pandajedi/jeditest/addContWoSoTask.py +54 -0
  103. pandajedi/jeditest/addDataCarouselRequest.py +47 -0
  104. pandajedi/jeditest/addEventServiceTask.py +101 -0
  105. pandajedi/jeditest/addHpoTask.py +81 -0
  106. pandajedi/jeditest/addIDDSStageInRequest.py +56 -0
  107. pandajedi/jeditest/addNonAtlasTask.py +71 -0
  108. pandajedi/jeditest/addNonAtlasTaskEnc.py +64 -0
  109. pandajedi/jeditest/addTestTask.py +124 -0
  110. pandajedi/jeditest/addTestTaskParamToDEFT.py +70 -0
  111. pandajedi/jeditest/addTestTaskWithMerger.py +97 -0
  112. pandajedi/jeditest/brokerTest.py +78 -0
  113. pandajedi/jeditest/cancelDataCarouselRequest.py +35 -0
  114. pandajedi/jeditest/contentsFeederTest.py +28 -0
  115. pandajedi/jeditest/contentsFeederTestWithTask.py +18 -0
  116. pandajedi/jeditest/ddmTest.py +4 -0
  117. pandajedi/jeditest/jobGenForOneTask.py +38 -0
  118. pandajedi/jeditest/jobGeneratorTest.py +43 -0
  119. pandajedi/jeditest/jobSplitterTest.py +104 -0
  120. pandajedi/jeditest/jobThrottlerTest.py +22 -0
  121. pandajedi/jeditest/msgProcessorTest.py +9 -0
  122. pandajedi/jeditest/postProcessorTest.py +18 -0
  123. pandajedi/jeditest/resubmitDataCarouselRequest.py +35 -0
  124. pandajedi/jeditest/resubmitTaskParam.py +20 -0
  125. pandajedi/jeditest/taskBrokerTest.py +22 -0
  126. pandajedi/jeditest/taskBrokerTestWithTask.py +38 -0
  127. pandajedi/jeditest/taskBufferTest.py +5 -0
  128. pandajedi/jeditest/taskCommandoTest.py +28 -0
  129. pandajedi/jeditest/taskRefinerTest.py +30 -0
  130. pandajedi/jeditest/testOneTaskToRefine.py +49 -0
  131. pandajedi/jeditest/testOneTaskToTaskBroker.py +42 -0
  132. pandajedi/jeditest/testPandaDDMMap.py +62 -0
  133. pandajedi/jeditest/updateDataCarouselRequestParams.py +50 -0
  134. pandajedi/jeditest/watchDogTest.py +25 -0
  135. pandajedi/jeditest/workQueueTest.py +22 -0
  136. pandajedi/jedithrottle/AtlasAnalJobThrottler.py +19 -0
  137. pandajedi/jedithrottle/AtlasProdJobThrottler.py +18 -0
  138. pandajedi/jedithrottle/GenJobThrottler.py +26 -0
  139. pandajedi/jedithrottle/JobThrottlerBase.py +496 -0
  140. pandajedi/jedithrottle/__init__.py +0 -0
  141. pandaserver/__init__.py +0 -0
  142. pandaserver/api/__init__.py +0 -0
  143. pandaserver/api/v1/__init__.py +0 -0
  144. pandaserver/api/v1/common.py +339 -0
  145. pandaserver/api/v1/credential_management_api.py +372 -0
  146. pandaserver/api/v1/data_carousel_api.py +264 -0
  147. pandaserver/api/v1/event_api.py +269 -0
  148. pandaserver/api/v1/file_server_api.py +942 -0
  149. pandaserver/api/v1/harvester_api.py +445 -0
  150. pandaserver/api/v1/http_client.py +166 -0
  151. pandaserver/api/v1/idds_api.py +180 -0
  152. pandaserver/api/v1/job_api.py +463 -0
  153. pandaserver/api/v1/metaconfig_api.py +122 -0
  154. pandaserver/api/v1/pilot_api.py +895 -0
  155. pandaserver/api/v1/statistics_api.py +207 -0
  156. pandaserver/api/v1/system_api.py +186 -0
  157. pandaserver/api/v1/task_api.py +1520 -0
  158. pandaserver/api/v1/tests/__init__.py +0 -0
  159. pandaserver/api/v1/tests/credential_management_api_tests.py +116 -0
  160. pandaserver/api/v1/tests/data_carousel_api_tests.py +82 -0
  161. pandaserver/api/v1/tests/harvester_api_tests.py +188 -0
  162. pandaserver/api/v1/tests/idds_api_tests.py +45 -0
  163. pandaserver/api/v1/tests/job_api_tests.py +112 -0
  164. pandaserver/api/v1/tests/pilot_api_tests.py +154 -0
  165. pandaserver/api/v1/tests/statistics_api_tests.py +104 -0
  166. pandaserver/api/v1/tests/system_api_tests.py +58 -0
  167. pandaserver/api/v1/tests/task_api_tests.py +590 -0
  168. pandaserver/api/v1/version.py +2 -0
  169. pandaserver/api/v1/workflow_api.py +136 -0
  170. pandaserver/brokerage/SiteMapper.py +406 -0
  171. pandaserver/brokerage/__init__.py +0 -0
  172. pandaserver/brokerage/broker.py +272 -0
  173. pandaserver/config/__init__.py +0 -0
  174. pandaserver/config/config_utils.py +15 -0
  175. pandaserver/config/daemon_config.py +29 -0
  176. pandaserver/config/mcp_config.py +38 -0
  177. pandaserver/config/panda_config.py +233 -0
  178. pandaserver/configurator/Carbon.py +65 -0
  179. pandaserver/configurator/Configurator.py +908 -0
  180. pandaserver/configurator/__init__.py +0 -0
  181. pandaserver/configurator/aux.py +73 -0
  182. pandaserver/daemons/__init__.py +0 -0
  183. pandaserver/daemons/master.py +110 -0
  184. pandaserver/daemons/master_systemd.py +79 -0
  185. pandaserver/daemons/scripts/__init__.py +0 -0
  186. pandaserver/daemons/scripts/add_main.py +264 -0
  187. pandaserver/daemons/scripts/add_sub.py +221 -0
  188. pandaserver/daemons/scripts/cache_pilots.py +47 -0
  189. pandaserver/daemons/scripts/cache_schedconfig.py +286 -0
  190. pandaserver/daemons/scripts/carbon.py +50 -0
  191. pandaserver/daemons/scripts/configurator.py +94 -0
  192. pandaserver/daemons/scripts/copyArchive.py +1387 -0
  193. pandaserver/daemons/scripts/datasetManager.py +1154 -0
  194. pandaserver/daemons/scripts/dummy_test.py +25 -0
  195. pandaserver/daemons/scripts/evpPD2P.py +144 -0
  196. pandaserver/daemons/scripts/hs_scrapers.py +264 -0
  197. pandaserver/daemons/scripts/metric_collector.py +927 -0
  198. pandaserver/daemons/scripts/panda_activeusers_query.py +80 -0
  199. pandaserver/daemons/scripts/pilotStreaming.py +109 -0
  200. pandaserver/daemons/scripts/process_workflow_files_daemon.py +166 -0
  201. pandaserver/daemons/scripts/recover_lost_files_daemon.py +177 -0
  202. pandaserver/daemons/scripts/task_evaluator.py +405 -0
  203. pandaserver/daemons/scripts/tmpwatch.py +70 -0
  204. pandaserver/daemons/scripts/worker_synchronization.py +97 -0
  205. pandaserver/daemons/utils.py +753 -0
  206. pandaserver/dataservice/DataServiceUtils.py +323 -0
  207. pandaserver/dataservice/ErrorCode.py +25 -0
  208. pandaserver/dataservice/RecoverLostFilesCore.py +322 -0
  209. pandaserver/dataservice/__init__.py +0 -0
  210. pandaserver/dataservice/activator.py +79 -0
  211. pandaserver/dataservice/adder_atlas_plugin.py +1106 -0
  212. pandaserver/dataservice/adder_dummy_plugin.py +31 -0
  213. pandaserver/dataservice/adder_gen.py +954 -0
  214. pandaserver/dataservice/adder_plugin_base.py +27 -0
  215. pandaserver/dataservice/adder_result.py +75 -0
  216. pandaserver/dataservice/adder_simple_plugin.py +160 -0
  217. pandaserver/dataservice/closer.py +247 -0
  218. pandaserver/dataservice/closer_atlas_plugin.py +90 -0
  219. pandaserver/dataservice/ddm.py +1381 -0
  220. pandaserver/dataservice/ddm_handler.py +95 -0
  221. pandaserver/dataservice/dyn_data_distributer.py +824 -0
  222. pandaserver/dataservice/event_lookup_client_ei.py +77 -0
  223. pandaserver/dataservice/event_picker.py +268 -0
  224. pandaserver/dataservice/finisher.py +194 -0
  225. pandaserver/dataservice/setupper.py +225 -0
  226. pandaserver/dataservice/setupper_atlas_plugin.py +1851 -0
  227. pandaserver/dataservice/setupper_dummy_plugin.py +47 -0
  228. pandaserver/dataservice/setupper_plugin_base.py +67 -0
  229. pandaserver/jobdispatcher/DispatcherUtils.py +35 -0
  230. pandaserver/jobdispatcher/ErrorCode.py +19 -0
  231. pandaserver/jobdispatcher/JobDispatcher.py +1625 -0
  232. pandaserver/jobdispatcher/Protocol.py +443 -0
  233. pandaserver/jobdispatcher/Watcher.py +186 -0
  234. pandaserver/jobdispatcher/__init__.py +0 -0
  235. pandaserver/pandamcp/__init__.py +0 -0
  236. pandaserver/pandamcp/mcp_main.py +56 -0
  237. pandaserver/pandamcp/mcp_test_client.py +95 -0
  238. pandaserver/pandamcp/mcp_utils.py +74 -0
  239. pandaserver/proxycache/__init__.py +0 -0
  240. pandaserver/proxycache/panda_proxy_cache.py +240 -0
  241. pandaserver/proxycache/token_cache.py +141 -0
  242. pandaserver/server/.gacl +6 -0
  243. pandaserver/server/panda.py +539 -0
  244. pandaserver/srvcore/CoreUtils.py +572 -0
  245. pandaserver/srvcore/MailUtils.py +106 -0
  246. pandaserver/srvcore/__init__.py +0 -0
  247. pandaserver/srvcore/allowed_methods.py +100 -0
  248. pandaserver/srvcore/oidc_utils.py +152 -0
  249. pandaserver/srvcore/panda_request.py +196 -0
  250. pandaserver/srvcore/srv_msg_utils.py +42 -0
  251. pandaserver/taskbuffer/ConBridge.py +513 -0
  252. pandaserver/taskbuffer/DBProxyPool.py +95 -0
  253. pandaserver/taskbuffer/DataCarousel.py +3053 -0
  254. pandaserver/taskbuffer/DatasetSpec.py +137 -0
  255. pandaserver/taskbuffer/DdmSpec.py +201 -0
  256. pandaserver/taskbuffer/ErrorCode.py +87 -0
  257. pandaserver/taskbuffer/EventServiceUtils.py +396 -0
  258. pandaserver/taskbuffer/FileSpec.py +290 -0
  259. pandaserver/taskbuffer/GlobalShares.py +244 -0
  260. pandaserver/taskbuffer/HarvesterMetricsSpec.py +92 -0
  261. pandaserver/taskbuffer/Initializer.py +82 -0
  262. pandaserver/taskbuffer/InputChunk.py +1080 -0
  263. pandaserver/taskbuffer/JediCacheSpec.py +57 -0
  264. pandaserver/taskbuffer/JediDatasetSpec.py +588 -0
  265. pandaserver/taskbuffer/JediFileSpec.py +242 -0
  266. pandaserver/taskbuffer/JediTaskSpec.py +1687 -0
  267. pandaserver/taskbuffer/JobSpec.py +1006 -0
  268. pandaserver/taskbuffer/JobUtils.py +232 -0
  269. pandaserver/taskbuffer/NucleusSpec.py +116 -0
  270. pandaserver/taskbuffer/OraDBProxy.py +174 -0
  271. pandaserver/taskbuffer/PanDAMsgProcessor.py +8 -0
  272. pandaserver/taskbuffer/PandaDBSchemaInfo.py +18 -0
  273. pandaserver/taskbuffer/ParseJobXML.py +350 -0
  274. pandaserver/taskbuffer/PickleFileSpec.py +18 -0
  275. pandaserver/taskbuffer/PickleJobSpec.py +25 -0
  276. pandaserver/taskbuffer/PrioUtil.py +40 -0
  277. pandaserver/taskbuffer/ProcessGroups.py +111 -0
  278. pandaserver/taskbuffer/ResourceSpec.py +175 -0
  279. pandaserver/taskbuffer/SQLDumper.py +37 -0
  280. pandaserver/taskbuffer/SQLManager.py +60 -0
  281. pandaserver/taskbuffer/SiteSpec.py +304 -0
  282. pandaserver/taskbuffer/SupErrors.py +10 -0
  283. pandaserver/taskbuffer/TaskBuffer.py +2882 -0
  284. pandaserver/taskbuffer/TaskBufferInterface.py +163 -0
  285. pandaserver/taskbuffer/Utils.py +848 -0
  286. pandaserver/taskbuffer/WorkQueue.py +227 -0
  287. pandaserver/taskbuffer/WorkQueueMapper.py +203 -0
  288. pandaserver/taskbuffer/WorkerSpec.py +143 -0
  289. pandaserver/taskbuffer/WrappedCursor.py +487 -0
  290. pandaserver/taskbuffer/WrappedPickle.py +88 -0
  291. pandaserver/taskbuffer/WrappedPostgresConn.py +24 -0
  292. pandaserver/taskbuffer/__init__.py +0 -0
  293. pandaserver/taskbuffer/db_proxy_mods/OraDBProxy_structure.png +0 -0
  294. pandaserver/taskbuffer/db_proxy_mods/__init__.py +7 -0
  295. pandaserver/taskbuffer/db_proxy_mods/base_module.py +841 -0
  296. pandaserver/taskbuffer/db_proxy_mods/data_carousel_module.py +772 -0
  297. pandaserver/taskbuffer/db_proxy_mods/entity_module.py +3804 -0
  298. pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +5661 -0
  299. pandaserver/taskbuffer/db_proxy_mods/job_standalone_module.py +2640 -0
  300. pandaserver/taskbuffer/db_proxy_mods/metrics_module.py +874 -0
  301. pandaserver/taskbuffer/db_proxy_mods/misc_standalone_module.py +4272 -0
  302. pandaserver/taskbuffer/db_proxy_mods/task_complex_module.py +6172 -0
  303. pandaserver/taskbuffer/db_proxy_mods/task_event_module.py +4953 -0
  304. pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +4999 -0
  305. pandaserver/taskbuffer/db_proxy_mods/task_utils_module.py +2063 -0
  306. pandaserver/taskbuffer/db_proxy_mods/worker_module.py +1944 -0
  307. pandaserver/taskbuffer/db_proxy_mods/workflow_module.py +832 -0
  308. pandaserver/taskbuffer/retryModule.py +552 -0
  309. pandaserver/taskbuffer/task_split_rules.py +308 -0
  310. pandaserver/taskbuffer/workflow_processor.py +249 -0
  311. pandaserver/taskbuffer/wrapped_oracle_conn.py +18 -0
  312. pandaserver/test/SchemaChecker.py +40 -0
  313. pandaserver/test/__init__.py +0 -0
  314. pandaserver/test/banUser.py +43 -0
  315. pandaserver/test/boostPrio.py +19 -0
  316. pandaserver/test/boostUser.py +35 -0
  317. pandaserver/test/callbackDDM.py +11 -0
  318. pandaserver/test/daod_on_demand.py +95 -0
  319. pandaserver/test/finishJob.py +184 -0
  320. pandaserver/test/finishTaskJEDI.py +11 -0
  321. pandaserver/test/getJobs.py +53 -0
  322. pandaserver/test/killJob.py +79 -0
  323. pandaserver/test/killJobLowPrio.py +99 -0
  324. pandaserver/test/killJobsInTask.py +56 -0
  325. pandaserver/test/killTask.py +80 -0
  326. pandaserver/test/killTaskJEDI.py +11 -0
  327. pandaserver/test/killUser.py +83 -0
  328. pandaserver/test/lsst/README.txt +7 -0
  329. pandaserver/test/lsst/lsstSubmit.py +139 -0
  330. pandaserver/test/lsst/lsstSubmitMERGEtest.py +141 -0
  331. pandaserver/test/lsst/lsstSubmitPhosim332.py +53 -0
  332. pandaserver/test/lsst/prepare-client-tarball-from-bigpanda-server.sh +41 -0
  333. pandaserver/test/reassignJobs.py +17 -0
  334. pandaserver/test/reassignSite.py +82 -0
  335. pandaserver/test/reassignTask.py +117 -0
  336. pandaserver/test/reassignWaiting.py +39 -0
  337. pandaserver/test/recoverLostFiles.py +7 -0
  338. pandaserver/test/reloadInputDS.py +51 -0
  339. pandaserver/test/sendCommandToJob.py +26 -0
  340. pandaserver/test/setDebugMode.py +38 -0
  341. pandaserver/test/setPriority.py +28 -0
  342. pandaserver/test/testEvgen.py +60 -0
  343. pandaserver/test/testEvgen17.py +58 -0
  344. pandaserver/test/testG4sim.py +81 -0
  345. pandaserver/test/testG4sim17.py +90 -0
  346. pandaserver/test/testGlobalShares.py +131 -0
  347. pandaserver/test/testReco.py +103 -0
  348. pandaserver/test/testSimulReco14.py +98 -0
  349. pandaserver/test/testSiteMap.py +26 -0
  350. pandaserver/test/testUpdateWorkerPilotStatus.py +9 -0
  351. pandaserver/userinterface/Client.py +1686 -0
  352. pandaserver/userinterface/UserIF.py +1799 -0
  353. pandaserver/userinterface/__init__.py +0 -0
  354. pandaserver/workflow/__init__.py +0 -0
  355. pandaserver/workflow/data_handler_plugins/base_data_handler.py +48 -0
  356. pandaserver/workflow/data_handler_plugins/ddm_collection_data_handler.py +100 -0
  357. pandaserver/workflow/data_handler_plugins/panda_task_data_handler.py +140 -0
  358. pandaserver/workflow/pcwl_test.py +122 -0
  359. pandaserver/workflow/pcwl_utils.py +384 -0
  360. pandaserver/workflow/psnakemake_container.json +68 -0
  361. pandaserver/workflow/psnakemake_task.json +62 -0
  362. pandaserver/workflow/psnakemake_test.py +60 -0
  363. pandaserver/workflow/snakeparser/__init__.py +3 -0
  364. pandaserver/workflow/snakeparser/extensions.py +74 -0
  365. pandaserver/workflow/snakeparser/log.py +53 -0
  366. pandaserver/workflow/snakeparser/names.py +3 -0
  367. pandaserver/workflow/snakeparser/parser.py +332 -0
  368. pandaserver/workflow/snakeparser/utils.py +29 -0
  369. pandaserver/workflow/step_handler_plugins/base_step_handler.py +85 -0
  370. pandaserver/workflow/step_handler_plugins/panda_task_step_handler.py +280 -0
  371. pandaserver/workflow/workflow_base.py +531 -0
  372. pandaserver/workflow/workflow_core.py +1974 -0
  373. pandaserver/workflow/workflow_core_smoketest.py +388 -0
  374. pandaserver/workflow/workflow_parser.py +222 -0
  375. pandaserver/workflow/workflow_utils.py +1047 -0
@@ -0,0 +1,101 @@
1
+ import sys
2
+ import uuid
3
+
4
+ from userinterface import Client
5
+
6
+ site = sys.argv[1]
7
+
8
+ dsName = "mc12_8TeV.189659.gg2VVPythia8_AU2CT10_ggH125p5_VV_2mu2numu_m2l4_2pt3.evgen.EVNT.e2872_tid01461041_00"
9
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
10
+
11
+ taskParamMap = {}
12
+
13
+ # taskParamMap['nFilesPerJob'] = 1
14
+ taskParamMap["nFiles"] = 1
15
+ taskParamMap["nEventsPerFile"] = 5000
16
+ taskParamMap["nEventsPerWorker"] = 10 # 5000
17
+ taskParamMap["nEventsPerJob"] = 1000
18
+ taskParamMap["nEvents"] = 1000
19
+ # taskParamMap['skipScout'] = True
20
+ taskParamMap["objectStore"] = "root^atlas-objectstore.cern.ch^/atlas/eventservice"
21
+ taskParamMap["taskName"] = str(uuid.uuid4())
22
+ taskParamMap["userName"] = "pandasrv1"
23
+ taskParamMap["vo"] = "atlas"
24
+ taskParamMap["taskPriority"] = 900
25
+ # taskParamMap['reqID'] = reqIdx
26
+ taskParamMap["architecture"] = "x86_64-slc5-gcc43-opt"
27
+ taskParamMap["transUses"] = "Atlas-17.2.11"
28
+ taskParamMap["transHome"] = "AtlasProduction-17.2.11.8"
29
+ taskParamMap["transPath"] = "AtlasG4_tf.py"
30
+ taskParamMap["processingType"] = "simul"
31
+ taskParamMap["prodSourceLabel"] = "ptest"
32
+ taskParamMap["taskType"] = "prod"
33
+ taskParamMap["workingGroup"] = "AP_Higgs"
34
+ taskParamMap["coreCount"] = 1
35
+ taskParamMap["walltime"] = 1
36
+ taskParamMap["cloud"] = "US"
37
+ taskParamMap["site"] = site
38
+ taskParamMap["log"] = {
39
+ "dataset": logDatasetName,
40
+ "type": "template",
41
+ "param_type": "log",
42
+ "token": "ATLASDATADISK",
43
+ "offset": 1000,
44
+ "value": f"{logDatasetName}.${{SN}}.log.tgz",
45
+ }
46
+ outDatasetName = f"panda.jeditest.HITS.{uuid.uuid4()}"
47
+
48
+
49
+ taskParamMap["jobParameters"] = [
50
+ {
51
+ "type": "constant",
52
+ "value": "--AMITag s1831 --DBRelease=current --athenaopts=--preloadlib=${ATLASMKLLIBDIR_PRELOAD}/libimf.so --conditionsTag=OFLCOND-MC12-SIM-00",
53
+ },
54
+ {
55
+ "type": "template",
56
+ "value": "firstEvent=${FIRSTEVENT}",
57
+ "param_type": "number",
58
+ },
59
+ {
60
+ "type": "constant",
61
+ "value": "--geometryVersion=ATLAS-GEO-21-02-02_VALIDATION",
62
+ },
63
+ {
64
+ "type": "template",
65
+ "param_type": "input",
66
+ "value": "--inputEvgenFile=${IN}",
67
+ "dataset": dsName,
68
+ },
69
+ {
70
+ "type": "template",
71
+ "value": "--maxEvents=${MAXEVENTS}",
72
+ "param_type": "number",
73
+ },
74
+ {
75
+ "type": "template",
76
+ "param_type": "output",
77
+ "token": "ATLASDATADISK",
78
+ "value": f" --outputHitsFile={outDatasetName}.${{SN}}.pool.root",
79
+ "dataset": outDatasetName,
80
+ },
81
+ {
82
+ "type": "constant",
83
+ "value": "--physicsList=QGSP_BERT --postInclude=RecJobTransforms/UseFrontierFallbackDBRelease.py --preInclude=SimulationJobOptions/preInclude.FrozenShowersFCalOnly.py,SimulationJobOptions/preInclude.BeamPipeKill.py",
84
+ },
85
+ {
86
+ "type": "template",
87
+ "value": "--skipEvents=${SKIPEVENTS}",
88
+ "param_type": "number",
89
+ },
90
+ {
91
+ "type": "template",
92
+ "value": "--randomSeed=${RNDMSEED}",
93
+ "param_type": "number",
94
+ },
95
+ ]
96
+
97
+ taskParamMap["esmergeSpec"] = {}
98
+ taskParamMap["esmergeSpec"]["transPath"] = "Merge_trf.py"
99
+ taskParamMap["esmergeSpec"]["jobParameters"] = "aaa bbb"
100
+
101
+ print(Client.insertTaskParams(taskParamMap))
@@ -0,0 +1,81 @@
1
+ import uuid
2
+ from urllib.parse import quote
3
+
4
+ from userinterface import Client
5
+
6
+ taskParamMap = {}
7
+
8
+ taskParamMap["noInput"] = True
9
+ taskParamMap["nEventsPerJob"] = 1
10
+ taskParamMap["nEvents"] = 5
11
+ taskParamMap["taskName"] = str(uuid.uuid4())
12
+ taskParamMap["userName"] = "pandasrv1"
13
+ taskParamMap["vo"] = "atlas"
14
+ taskParamMap["taskPriority"] = 1000
15
+ taskParamMap["reqID"] = 12345
16
+ taskParamMap["architecture"] = ""
17
+ taskParamMap["hpoWorkflow"] = True
18
+ taskParamMap["transUses"] = ""
19
+ taskParamMap["transHome"] = ""
20
+ taskParamMap["transPath"] = "http://pandaserver.cern.ch:25080/trf/user/runHPO-00-00-01"
21
+ taskParamMap["processingType"] = "simul"
22
+ taskParamMap["prodSourceLabel"] = "test"
23
+ taskParamMap["useLocalIO"] = 1
24
+ taskParamMap["taskType"] = "prod"
25
+ taskParamMap["workingGroup"] = "AP_HPO"
26
+ taskParamMap["coreCount"] = 1
27
+ taskParamMap["site"] = "BNL_PROD_UCORE"
28
+ taskParamMap["nucleus"] = "CERN-PROD"
29
+ taskParamMap["cloud"] = "WORLD"
30
+
31
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
32
+ outDatasetName = f"panda.jeditest.HPO.{uuid.uuid4()}"
33
+
34
+ taskParamMap["log"] = {
35
+ "dataset": logDatasetName,
36
+ "type": "template",
37
+ "param_type": "log",
38
+ "token": "ddd:.*DATADISK",
39
+ "destination": "(type=DATADISK)\(dontkeeplog=True)",
40
+ "offset": 1000,
41
+ "value": f"{logDatasetName}.${{SN}}.log.tgz",
42
+ }
43
+
44
+ taskParamMap["hpoRequestData"] = {
45
+ "sandbox": "gitlab-registry.cern.ch/zhangruihpc/steeringcontainer:latest",
46
+ "executable": "docker",
47
+ "arguments": '/bin/bash -c "hpogrid generate --n_point=%NUM_POINTS ' "--max_point=%MAX_POINTS --infile=$PWD/%IN --outfile=$PWD/%OUT " '-l=nevergrad"',
48
+ "output_json": "output.json",
49
+ "max_points": 10,
50
+ "num_points_per_generation": 2,
51
+ }
52
+
53
+ taskParamMap["container_name"] = "docker://gitlab-registry.cern.ch/zhangruihpc/evaluationcontainer:mlflow"
54
+
55
+ taskParamMap["jobParameters"] = [
56
+ {"type": "constant", "value": f"-o out.json -j \"\" -p \"{quote('bash ./exec_in_container.sh')}\""},
57
+ {"type": "constant", "value": "--writeInputToTxt IN_DATA:input_ds.json --inSampleFile input_sample.json"},
58
+ {"type": "constant", "value": "-a aaa.tgz --sourceURL https://aipanda048.cern.ch:25443"},
59
+ {"type": "constant", "value": "--inMap \"{'IN_DATA': ${IN_DATA/T}}\""},
60
+ {
61
+ "type": "template",
62
+ "param_type": "input",
63
+ "value": '-i "${IN_DATA/T}"',
64
+ "dataset": "mc16_13TeV.501103.MGPy8EG_StauStauDirect_220p0_1p0_TFilt.merge.EVNT.e8102_e7400_tid21342682_00",
65
+ "attributes": "nosplit,repeat",
66
+ },
67
+ {
68
+ "type": "template",
69
+ "param_type": "output",
70
+ "token": "ATLASDATADISK",
71
+ "value": "$JEDITASKID.metrics.${SN}.tgz",
72
+ "dataset": outDatasetName,
73
+ "hidden": True,
74
+ },
75
+ {
76
+ "type": "constant",
77
+ "value": "--outMetricsFile=${OUTPUT0}^metrics.tgz",
78
+ },
79
+ ]
80
+
81
+ print(Client.insertTaskParams(taskParamMap))
@@ -0,0 +1,56 @@
1
+ import sys
2
+
3
+ from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
4
+ from pandajedi.jediddm.DDMInterface import DDMInterface
5
+ from pandaserver.taskbuffer.DataCarousel import (
6
+ DataCarouselInterface,
7
+ DataCarouselRequestSpec,
8
+ DataCarouselRequestStatus,
9
+ )
10
+
11
+ vo = "atlas"
12
+ task_id = int(sys.argv[1])
13
+ request_id = int(sys.argv[2])
14
+
15
+ print("set tbIF")
16
+ tbIF = JediTaskBufferInterface()
17
+ tbIF.setupInterface(max_size=1)
18
+
19
+ print("set ddmIF")
20
+ ddmIF = DDMInterface()
21
+ ddmIF.setupInterface()
22
+
23
+ print("set DCIF")
24
+ data_carousel_interface = DataCarouselInterface(tbIF, ddmIF.getInterface(vo))
25
+ if data_carousel_interface is None:
26
+ # data carousel interface is undefined
27
+ errStr = f"data carousel interface is undefined for vo={vo}"
28
+ print(errStr)
29
+ sys.exit(1)
30
+
31
+ print("query DB for data carousel request")
32
+ sql = (
33
+ f"SELECT req.* "
34
+ f"FROM ATLAS_PANDA.data_carousel_requests req, ATLAS_PANDA.data_carousel_relations rel "
35
+ f"WHERE req.request_id=rel.request_id "
36
+ f"AND rel.request_id=:request_id AND rel.task_id=:task_id "
37
+ )
38
+ var_map = {
39
+ ":task_id": task_id,
40
+ ":request_id": request_id,
41
+ }
42
+ res_list = tbIF.querySQL(sql, var_map)
43
+
44
+ if res_list:
45
+ for res in res_list:
46
+ print("got request; submit iDDS stage-in request")
47
+ # make request spec
48
+ dc_req_spec = DataCarouselRequestSpec()
49
+ dc_req_spec.pack(res)
50
+ # submit iDDS stage-in reqeust
51
+ ret = data_carousel_interface._submit_idds_stagein_request(task_id, dc_req_spec)
52
+ print(f"Done submit to iDDS; iDDS_requestID={ret}")
53
+ break
54
+ else:
55
+ print(f"Got no request: {res_list}")
56
+ sys.exit(0)
@@ -0,0 +1,71 @@
1
+ import uuid
2
+
3
+ try:
4
+ from pandatools import Client
5
+ except ImportError:
6
+ from userinterface import Client
7
+
8
+ inFileList = ["file1", "file2", "file3"]
9
+
10
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
11
+
12
+ taskParamMap = {}
13
+
14
+ taskParamMap["nFilesPerJob"] = 1
15
+ taskParamMap["nFiles"] = len(inFileList)
16
+ # taskParamMap['nEventsPerInputFile'] = 10000
17
+ # taskParamMap['nEventsPerJob'] = 10000
18
+ # taskParamMap['nEvents'] = 25000
19
+ taskParamMap["noInput"] = True
20
+ taskParamMap["pfnList"] = inFileList
21
+ # taskParamMap['mergeOutput'] = True
22
+ taskParamMap["taskName"] = str(uuid.uuid4())
23
+ taskParamMap["userName"] = "someone"
24
+ taskParamMap["vo"] = "wlcg"
25
+ taskParamMap["taskPriority"] = 900
26
+ # taskParamMap['reqID'] = reqIdx
27
+ taskParamMap["architecture"] = "power9"
28
+ taskParamMap["transUses"] = "A"
29
+ taskParamMap["transHome"] = "B"
30
+ taskParamMap["transPath"] = "executable"
31
+ taskParamMap["processingType"] = "step1"
32
+ taskParamMap["prodSourceLabel"] = "test"
33
+ taskParamMap["taskType"] = "test"
34
+ taskParamMap["workingGroup"] = "groupA"
35
+ # taskParamMap['coreCount'] = 1
36
+ # taskParamMap['walltime'] = 1
37
+ taskParamMap["cloud"] = "NA"
38
+ taskParamMap["site"] = "TEST_PQ"
39
+ taskParamMap["log"] = {
40
+ "dataset": logDatasetName,
41
+ "type": "template",
42
+ "param_type": "log",
43
+ "token": "local",
44
+ "destination": "local",
45
+ "value": f"{logDatasetName}.${{SN}}.log.tgz",
46
+ }
47
+ outDatasetName = f"panda.jeditest.{uuid.uuid4()}"
48
+
49
+
50
+ taskParamMap["jobParameters"] = [
51
+ {
52
+ "type": "constant",
53
+ "value": '-i "${IN/T}"',
54
+ },
55
+ {"type": "constant", "value": "ecmEnergy=8000 runNumber=12345"},
56
+ {
57
+ "type": "template",
58
+ "param_type": "output",
59
+ "token": "local",
60
+ "destination": "local",
61
+ "value": f"outputEVNTFile={outDatasetName}.${{SN}}.root",
62
+ "dataset": outDatasetName,
63
+ "offset": 1000,
64
+ },
65
+ {
66
+ "type": "constant",
67
+ "value": "aaaa",
68
+ },
69
+ ]
70
+
71
+ print(Client.insertTaskParams(taskParamMap))
@@ -0,0 +1,64 @@
1
+ import sys
2
+ import uuid
3
+
4
+ from userinterface import Client
5
+
6
+ inFileList = ["file1"]
7
+
8
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
9
+
10
+ taskParamMap = {}
11
+
12
+ taskParamMap["nFilesPerJob"] = 1
13
+ taskParamMap["nFiles"] = len(inFileList)
14
+ # taskParamMap['nEventsPerInputFile'] = 10000
15
+ # taskParamMap['nEventsPerJob'] = 10000
16
+ # taskParamMap['nEvents'] = 25000
17
+ taskParamMap["noInput"] = True
18
+ taskParamMap["pfnList"] = inFileList
19
+ # taskParamMap['mergeOutput'] = True
20
+ taskParamMap["taskName"] = str(uuid.uuid4())
21
+ taskParamMap["userName"] = "someone"
22
+ taskParamMap["vo"] = "wlcg"
23
+ taskParamMap["taskPriority"] = 900
24
+ # taskParamMap['reqID'] = reqIdx
25
+ taskParamMap["architecture"] = ""
26
+ taskParamMap["transUses"] = ""
27
+ taskParamMap["transHome"] = None
28
+ # taskParamMap['transPath'] = 'https://atlpan.web.cern.ch/atlpan/bash-c'
29
+ taskParamMap["transPath"] = "https://atlpan.web.cern.ch/atlpan/bash-c-enc"
30
+ taskParamMap["encJobParams"] = True
31
+ taskParamMap["processingType"] = "step1"
32
+ taskParamMap["prodSourceLabel"] = "test"
33
+ taskParamMap["taskType"] = "test"
34
+ taskParamMap["workingGroup"] = "lsst"
35
+ # taskParamMap['coreCount'] = 1
36
+ # taskParamMap['walltime'] = 1
37
+ taskParamMap["cloud"] = "LSST"
38
+ taskParamMap["site"] = "DOMA_LSST_GOOGLE_TEST"
39
+ # taskParamMap['site'] = 'DOMA_LSST_SLAC_TEST'
40
+
41
+ # taskParamMap['ramCount'] = 1000
42
+ taskParamMap["ramUnit"] = "MB"
43
+
44
+ """
45
+ taskParamMap['log'] = {'dataset': logDatasetName,
46
+ 'type':'template',
47
+ 'param_type':'log',
48
+ 'token':'local',
49
+ 'destination':'local',
50
+ 'value':'{0}.${{SN}}.log.tgz'.format(logDatasetName)}
51
+ """
52
+
53
+ taskParamMap["jobParameters"] = [
54
+ {
55
+ "type": "constant",
56
+ "value": "echo aaa; ls; echo",
57
+ },
58
+ {
59
+ "type": "constant",
60
+ "value": "\"'${IN/L}'\"",
61
+ },
62
+ ]
63
+
64
+ print(Client.insertTaskParams(taskParamMap))
@@ -0,0 +1,124 @@
1
+ import sys
2
+ import uuid
3
+
4
+ from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
5
+ from pandaserver.taskbuffer.JediDatasetSpec import JediDatasetSpec
6
+ from pandaserver.taskbuffer.JediTaskSpec import JediTaskSpec
7
+
8
+ tbIF = JediTaskBufferInterface()
9
+ tbIF.setupInterface()
10
+
11
+ task = JediTaskSpec()
12
+ task.jediTaskID = sys.argv[1]
13
+ task.taskName = f"pandatest.{uuid.uuid4()}"
14
+ task.status = "defined"
15
+ task.userName = "pandasrv1"
16
+ task.vo = "atlas"
17
+ task.prodSourceLabel = "managed"
18
+ task.taskPriority = 100
19
+ task.currentPriority = task.taskPriority
20
+ task.architecture = "i686-slc5-gcc43-opt"
21
+ task.transUses = "Atlas-17.2.7"
22
+ task.transHome = "AtlasProduction-17.2.8.10"
23
+ task.transPath = "Reco_trf.py"
24
+ task.workQueue_ID = 3
25
+
26
+ tbIF.insertTask_JEDI(task)
27
+
28
+
29
+ ds = JediDatasetSpec()
30
+ ds.jediTaskID = task.jediTaskID
31
+ if len(sys.argv) > 2:
32
+ ds.datasetName = sys.argv[2]
33
+ else:
34
+ ds.datasetName = "data12_8TeV.00214651.physics_Egamma.merge.AOD.f489_m1261"
35
+ ds.type = "input"
36
+ ds.vo = task.vo
37
+ ds.cloud = "US"
38
+ ds.streamName = "IN"
39
+ ds.status = "defined"
40
+ ds.nFiles = 0
41
+ ds.nFilesUsed = 0
42
+ ds.nFilesFinished = 0
43
+ ds.nFilesFailed = 0
44
+
45
+ st, datasetID = tbIF.insertDataset_JEDI(ds)
46
+
47
+ ds = JediDatasetSpec()
48
+ ds.jediTaskID = task.jediTaskID
49
+ ds.datasetName = "ddo.000001.Atlas.Ideal.DBRelease.v220701"
50
+ ds.type = "input"
51
+ ds.vo = task.vo
52
+ ds.cloud = "US"
53
+ ds.streamName = "DBR"
54
+ ds.status = "defined"
55
+ ds.nFiles = 0
56
+ ds.nFilesUsed = 0
57
+ ds.nFilesFinished = 0
58
+ ds.nFilesFailed = 0
59
+ ds.masterID = datasetID
60
+ ds.attributes = "repeat,nosplit"
61
+
62
+ tbIF.insertDataset_JEDI(ds)
63
+
64
+
65
+ ds = JediDatasetSpec()
66
+ ds.jediTaskID = task.jediTaskID
67
+ ds.datasetName = f"panda.jeditest.GEN.{uuid.uuid4()}"
68
+ ds.type = "output"
69
+ ds.vo = task.vo
70
+ ds.cloud = "US"
71
+ ds.streamName = "OUT"
72
+ ds.status = "defined"
73
+ ds.nFiles = 0
74
+ ds.nFilesUsed = 0
75
+ ds.nFilesFinished = 0
76
+ ds.nFilesFailed = 0
77
+
78
+ st, datasetID = tbIF.insertDataset_JEDI(ds)
79
+
80
+ tbIF.insertOutputTemplate_JEDI(
81
+ [
82
+ {
83
+ "jediTaskID": task.jediTaskID,
84
+ "datasetID": datasetID,
85
+ "filenameTemplate": f"{ds.datasetName}.${{SN}}.pool.root",
86
+ "serialNr": 1,
87
+ "streamName": "OUT",
88
+ "outtype": ds.type,
89
+ }
90
+ ]
91
+ )
92
+
93
+ ds = JediDatasetSpec()
94
+ ds.jediTaskID = task.jediTaskID
95
+ ds.datasetName = f"panda.jeditest.log.{uuid.uuid4()}"
96
+ ds.type = "log"
97
+ ds.vo = task.vo
98
+ ds.cloud = "US"
99
+ ds.streamName = "LOG"
100
+ ds.status = "defined"
101
+ ds.nFiles = 0
102
+ ds.nFilesUsed = 0
103
+ ds.nFilesFinished = 0
104
+ ds.nFilesFailed = 0
105
+
106
+ st, datasetID = tbIF.insertDataset_JEDI(ds)
107
+
108
+ tbIF.insertOutputTemplate_JEDI(
109
+ [
110
+ {
111
+ "jediTaskID": task.jediTaskID,
112
+ "datasetID": datasetID,
113
+ "filenameTemplate": f"{ds.datasetName}.${{SN}}.log.tgz",
114
+ "serialNr": 1,
115
+ "streamName": "LOG",
116
+ "outtype": ds.type,
117
+ }
118
+ ]
119
+ )
120
+
121
+ tbIF.insertJobParamsTemplate_JEDI(
122
+ task.jediTaskID,
123
+ 'inputAODFile=${IN} maxEvents=1000 RunNumber=213816 autoConfiguration=everything preExec="from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag="BTagCalibALL-07-02"" DBRelease=${DBR} AMITag=p1462 outputNTUP_EMBLLDNFile=${OUT}',
124
+ )
@@ -0,0 +1,70 @@
1
+ import sys
2
+
3
+ try:
4
+ metaID = sys.argv[1]
5
+ except Exception:
6
+ metaID = None
7
+ import json
8
+ import uuid
9
+
10
+ taskParamMap = {}
11
+ taskParamMap["taskName"] = str(uuid.uuid4())
12
+ taskParamMap["userName"] = "pandasrv1"
13
+ taskParamMap["vo"] = "atlas"
14
+ taskParamMap["taskPriority"] = 100
15
+ taskParamMap["architecture"] = "i686-slc5-gcc43-opt"
16
+ taskParamMap["transUses"] = "Atlas-17.2.7"
17
+ taskParamMap["transHome"] = "AtlasProduction-17.2.8.10"
18
+ taskParamMap["transPath"] = "Reco_trf.py"
19
+ taskParamMap["processingType"] = "reco"
20
+ taskParamMap["prodSourceLabel"] = "test"
21
+ taskParamMap["taskType"] = "prod"
22
+ taskParamMap["workingGroup"] = "AP_Higgs"
23
+ taskParamMap["coreCount"] = 1
24
+ taskParamMap["cloud"] = "US"
25
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
26
+ taskParamMap["log"] = {
27
+ "dataset": logDatasetName,
28
+ "type": "template",
29
+ "param_type": "log",
30
+ "token": "ATLASDATADISK",
31
+ "value": f"{logDatasetName}.${{SN}}.log.tgz",
32
+ }
33
+ outDatasetName = f"panda.jeditest.NTUP_EMBLLDN.{uuid.uuid4()}"
34
+ taskParamMap["jobParameters"] = [
35
+ {
36
+ "type": "template",
37
+ "param_type": "input",
38
+ "value": "inputAODFile=${IN}",
39
+ "dataset": "data12_8TeV.00214651.physics_Egamma.merge.AOD.f489_m1261",
40
+ },
41
+ {
42
+ "type": "constant",
43
+ "value": 'maxEvents=1000 RunNumber=213816 autoConfiguration=everything preExec="from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag="BTagCalibALL-07-02""',
44
+ },
45
+ {
46
+ "type": "template",
47
+ "param_type": "input",
48
+ "value": "DBRelease=${DBR}",
49
+ "dataset": "ddo.000001.Atlas.Ideal.DBRelease.v220701",
50
+ "attributes": "repeat,nosplit",
51
+ },
52
+ {"type": "constant", "value": "AMITag=p1462"},
53
+ {
54
+ "type": "template",
55
+ "param_type": "output",
56
+ "token": "ATLASDATADISK",
57
+ "value": f"outputNTUP_EMBLLDNFile={outDatasetName}.${{SN}}.pool.root",
58
+ "dataset": outDatasetName,
59
+ },
60
+ ]
61
+
62
+ jonStr = json.dumps(taskParamMap)
63
+
64
+ from pandajedi.jedicore.JediTaskBufferInterface import ( # noqa: E402
65
+ JediTaskBufferInterface,
66
+ )
67
+
68
+ tbIF = JediTaskBufferInterface()
69
+ tbIF.setupInterface()
70
+ tbIF.insertTaskParams_JEDI(taskParamMap["vo"], taskParamMap["prodSourceLabel"], taskParamMap["userName"], taskParamMap["taskName"], jonStr)
@@ -0,0 +1,97 @@
1
+ import sys
2
+
3
+ try:
4
+ metaID = sys.argv[1]
5
+ except Exception:
6
+ metaID = None
7
+ import json
8
+ import uuid
9
+
10
+ from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
11
+
12
+ tbIF = JediTaskBufferInterface()
13
+ tbIF.setupInterface()
14
+
15
+ reqIdx = 100000000
16
+ dsName = "mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00"
17
+ logDatasetName = f"panda.jeditest.log.{uuid.uuid4()}"
18
+
19
+ taskParamMap = {}
20
+
21
+ taskParamMap["nFilesPerJob"] = 1
22
+ taskParamMap["nFiles"] = 2
23
+ taskParamMap["nEventsPerInputFile"] = 10000
24
+ taskParamMap["nEventsPerJob"] = 10000
25
+ # taskParamMap['nEvents'] = 25000
26
+ # taskParamMap['skipScout'] = True
27
+ taskParamMap["taskName"] = str(uuid.uuid4())
28
+ taskParamMap["userName"] = "pandasrv1"
29
+ taskParamMap["vo"] = "atlas"
30
+ taskParamMap["taskPriority"] = 900
31
+ # taskParamMap['reqID'] = reqIdx
32
+ taskParamMap["architecture"] = "i686-slc5-gcc43-opt"
33
+ taskParamMap["transUses"] = "Atlas-17.2.2"
34
+ taskParamMap["transHome"] = "AtlasProduction-17.2.2.2"
35
+ taskParamMap["transPath"] = "Generate_trf.py"
36
+ taskParamMap["processingType"] = "evgen"
37
+ taskParamMap["prodSourceLabel"] = "test"
38
+ taskParamMap["taskType"] = "prod"
39
+ taskParamMap["workingGroup"] = "AP_Higgs"
40
+ # taskParamMap['coreCount'] = 1
41
+ # taskParamMap['walltime'] = 1
42
+ taskParamMap["cloud"] = "FR"
43
+ # taskParamMap['site'] = 'FZK-LCG2'
44
+ taskParamMap["log"] = {
45
+ "dataset": logDatasetName,
46
+ "type": "template",
47
+ "param_type": "log",
48
+ "token": "ATLASDATADISK",
49
+ "offset": 1000,
50
+ "value": f"{logDatasetName}.${{SN}}.log.tgz",
51
+ }
52
+ outDatasetName = f"panda.jeditest.EVNT.{uuid.uuid4()}"
53
+
54
+
55
+ taskParamMap["jobParameters"] = [
56
+ {
57
+ "type": "template",
58
+ "param_type": "input",
59
+ "value": "inputGenConfFile=${IN}",
60
+ "dataset": dsName,
61
+ },
62
+ {"type": "constant", "value": "ecmEnergy=8000 runNumber=12345"},
63
+ {
64
+ "type": "template",
65
+ "value": "maxEvents=${MAXEVENTS}",
66
+ "param_type": "number",
67
+ },
68
+ {"type": "template", "value": "randomSeed=${RNDMSEED}", "param_type": "number", "offset": 1000},
69
+ {
70
+ "type": "constant",
71
+ "value": "jobConfig=MC12JobOptions/MC12.161559.PowHegPythia8_AU2CT10_ggH145_tautaull.py",
72
+ },
73
+ {
74
+ "type": "template",
75
+ "param_type": "output",
76
+ "token": "ATLASDATADISK",
77
+ "value": f"outputEVNTFile={outDatasetName}.${{SN}}.pool.root",
78
+ "dataset": outDatasetName,
79
+ "offset": 1000,
80
+ },
81
+ {
82
+ "type": "constant",
83
+ "value": "evgenJobOpts=MC12JobOpts-00-01-06_v1.tar.gz",
84
+ },
85
+ ]
86
+
87
+ # params to merge output
88
+ taskParamMap["mergeOutput"] = True
89
+ taskParamMap["mergeSpec"] = {}
90
+ taskParamMap["mergeSpec"]["transPath"] = "HITSMerge_tf.py"
91
+ taskParamMap["mergeSpec"][
92
+ "jobParameters"
93
+ ] = "--AMITag s1776 --DBRelease=current --autoConfiguration=everything --outputHitsFile=${OUTPUT0} --inputHitsFile=${TRN_OUTPUT0} --inputLogsFile=${TRN_LOG0}"
94
+
95
+ jonStr = json.dumps(taskParamMap)
96
+
97
+ tbIF.insertTaskParams_JEDI(taskParamMap["vo"], taskParamMap["prodSourceLabel"], taskParamMap["userName"], taskParamMap["taskName"], jonStr)