wmglobalqueue 2.4.5.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. Utils/CPMetrics.py +270 -0
  2. Utils/CertTools.py +100 -0
  3. Utils/EmailAlert.py +50 -0
  4. Utils/ExtendedUnitTestCase.py +62 -0
  5. Utils/FileTools.py +182 -0
  6. Utils/IteratorTools.py +80 -0
  7. Utils/MathUtils.py +31 -0
  8. Utils/MemoryCache.py +119 -0
  9. Utils/Patterns.py +24 -0
  10. Utils/Pipeline.py +137 -0
  11. Utils/PortForward.py +97 -0
  12. Utils/ProcFS.py +112 -0
  13. Utils/ProcessStats.py +194 -0
  14. Utils/PythonVersion.py +17 -0
  15. Utils/Signals.py +36 -0
  16. Utils/TemporaryEnvironment.py +27 -0
  17. Utils/Throttled.py +227 -0
  18. Utils/Timers.py +130 -0
  19. Utils/Timestamps.py +86 -0
  20. Utils/TokenManager.py +143 -0
  21. Utils/Tracing.py +60 -0
  22. Utils/TwPrint.py +98 -0
  23. Utils/Utilities.py +318 -0
  24. Utils/__init__.py +11 -0
  25. Utils/wmcoreDTools.py +707 -0
  26. WMCore/ACDC/Collection.py +57 -0
  27. WMCore/ACDC/CollectionTypes.py +12 -0
  28. WMCore/ACDC/CouchCollection.py +67 -0
  29. WMCore/ACDC/CouchFileset.py +238 -0
  30. WMCore/ACDC/CouchService.py +73 -0
  31. WMCore/ACDC/DataCollectionService.py +485 -0
  32. WMCore/ACDC/Fileset.py +94 -0
  33. WMCore/ACDC/__init__.py +11 -0
  34. WMCore/Algorithms/Alarm.py +39 -0
  35. WMCore/Algorithms/MathAlgos.py +274 -0
  36. WMCore/Algorithms/MiscAlgos.py +67 -0
  37. WMCore/Algorithms/ParseXMLFile.py +115 -0
  38. WMCore/Algorithms/Permissions.py +27 -0
  39. WMCore/Algorithms/Singleton.py +58 -0
  40. WMCore/Algorithms/SubprocessAlgos.py +129 -0
  41. WMCore/Algorithms/__init__.py +7 -0
  42. WMCore/Cache/GenericDataCache.py +98 -0
  43. WMCore/Cache/WMConfigCache.py +572 -0
  44. WMCore/Cache/__init__.py +0 -0
  45. WMCore/Configuration.py +659 -0
  46. WMCore/DAOFactory.py +47 -0
  47. WMCore/DataStructs/File.py +177 -0
  48. WMCore/DataStructs/Fileset.py +140 -0
  49. WMCore/DataStructs/Job.py +182 -0
  50. WMCore/DataStructs/JobGroup.py +142 -0
  51. WMCore/DataStructs/JobPackage.py +49 -0
  52. WMCore/DataStructs/LumiList.py +734 -0
  53. WMCore/DataStructs/Mask.py +219 -0
  54. WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py +197 -0
  55. WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py +92 -0
  56. WMCore/DataStructs/MathStructs/SummaryHistogram.py +117 -0
  57. WMCore/DataStructs/MathStructs/__init__.py +0 -0
  58. WMCore/DataStructs/Pickleable.py +24 -0
  59. WMCore/DataStructs/Run.py +256 -0
  60. WMCore/DataStructs/Subscription.py +175 -0
  61. WMCore/DataStructs/WMObject.py +47 -0
  62. WMCore/DataStructs/WorkUnit.py +112 -0
  63. WMCore/DataStructs/Workflow.py +60 -0
  64. WMCore/DataStructs/__init__.py +8 -0
  65. WMCore/Database/CMSCouch.py +1430 -0
  66. WMCore/Database/ConfigDBMap.py +29 -0
  67. WMCore/Database/CouchMonitoring.py +450 -0
  68. WMCore/Database/CouchUtils.py +118 -0
  69. WMCore/Database/DBCore.py +198 -0
  70. WMCore/Database/DBCreator.py +113 -0
  71. WMCore/Database/DBExceptionHandler.py +59 -0
  72. WMCore/Database/DBFactory.py +117 -0
  73. WMCore/Database/DBFormatter.py +177 -0
  74. WMCore/Database/Dialects.py +13 -0
  75. WMCore/Database/ExecuteDAO.py +327 -0
  76. WMCore/Database/MongoDB.py +241 -0
  77. WMCore/Database/MySQL/Destroy.py +42 -0
  78. WMCore/Database/MySQL/ListUserContent.py +20 -0
  79. WMCore/Database/MySQL/__init__.py +9 -0
  80. WMCore/Database/MySQLCore.py +132 -0
  81. WMCore/Database/Oracle/Destroy.py +56 -0
  82. WMCore/Database/Oracle/ListUserContent.py +19 -0
  83. WMCore/Database/Oracle/__init__.py +9 -0
  84. WMCore/Database/ResultSet.py +44 -0
  85. WMCore/Database/Transaction.py +91 -0
  86. WMCore/Database/__init__.py +9 -0
  87. WMCore/Database/ipy_profile_couch.py +438 -0
  88. WMCore/GlobalWorkQueue/CherryPyThreads/CleanUpTask.py +29 -0
  89. WMCore/GlobalWorkQueue/CherryPyThreads/HeartbeatMonitor.py +105 -0
  90. WMCore/GlobalWorkQueue/CherryPyThreads/LocationUpdateTask.py +28 -0
  91. WMCore/GlobalWorkQueue/CherryPyThreads/ReqMgrInteractionTask.py +35 -0
  92. WMCore/GlobalWorkQueue/CherryPyThreads/__init__.py +0 -0
  93. WMCore/GlobalWorkQueue/__init__.py +0 -0
  94. WMCore/GroupUser/CouchObject.py +127 -0
  95. WMCore/GroupUser/Decorators.py +51 -0
  96. WMCore/GroupUser/Group.py +33 -0
  97. WMCore/GroupUser/Interface.py +73 -0
  98. WMCore/GroupUser/User.py +96 -0
  99. WMCore/GroupUser/__init__.py +11 -0
  100. WMCore/Lexicon.py +836 -0
  101. WMCore/REST/Auth.py +202 -0
  102. WMCore/REST/CherryPyPeriodicTask.py +166 -0
  103. WMCore/REST/Error.py +333 -0
  104. WMCore/REST/Format.py +642 -0
  105. WMCore/REST/HeartbeatMonitorBase.py +90 -0
  106. WMCore/REST/Main.py +636 -0
  107. WMCore/REST/Server.py +2435 -0
  108. WMCore/REST/Services.py +24 -0
  109. WMCore/REST/Test.py +120 -0
  110. WMCore/REST/Tools.py +38 -0
  111. WMCore/REST/Validation.py +250 -0
  112. WMCore/REST/__init__.py +1 -0
  113. WMCore/ReqMgr/DataStructs/RequestStatus.py +209 -0
  114. WMCore/ReqMgr/DataStructs/RequestType.py +13 -0
  115. WMCore/ReqMgr/DataStructs/__init__.py +0 -0
  116. WMCore/ReqMgr/__init__.py +1 -0
  117. WMCore/Services/AlertManager/AlertManagerAPI.py +111 -0
  118. WMCore/Services/AlertManager/__init__.py +0 -0
  119. WMCore/Services/CRIC/CRIC.py +238 -0
  120. WMCore/Services/CRIC/__init__.py +0 -0
  121. WMCore/Services/DBS/DBS3Reader.py +1044 -0
  122. WMCore/Services/DBS/DBSConcurrency.py +44 -0
  123. WMCore/Services/DBS/DBSErrors.py +112 -0
  124. WMCore/Services/DBS/DBSReader.py +23 -0
  125. WMCore/Services/DBS/DBSUtils.py +166 -0
  126. WMCore/Services/DBS/DBSWriterObjects.py +381 -0
  127. WMCore/Services/DBS/ProdException.py +133 -0
  128. WMCore/Services/DBS/__init__.py +8 -0
  129. WMCore/Services/FWJRDB/FWJRDBAPI.py +118 -0
  130. WMCore/Services/FWJRDB/__init__.py +0 -0
  131. WMCore/Services/HTTPS/HTTPSAuthHandler.py +66 -0
  132. WMCore/Services/HTTPS/__init__.py +0 -0
  133. WMCore/Services/LogDB/LogDB.py +201 -0
  134. WMCore/Services/LogDB/LogDBBackend.py +191 -0
  135. WMCore/Services/LogDB/LogDBExceptions.py +11 -0
  136. WMCore/Services/LogDB/LogDBReport.py +85 -0
  137. WMCore/Services/LogDB/__init__.py +0 -0
  138. WMCore/Services/MSPileup/__init__.py +0 -0
  139. WMCore/Services/MSUtils/MSUtils.py +54 -0
  140. WMCore/Services/MSUtils/__init__.py +0 -0
  141. WMCore/Services/McM/McM.py +173 -0
  142. WMCore/Services/McM/__init__.py +8 -0
  143. WMCore/Services/MonIT/Grafana.py +133 -0
  144. WMCore/Services/MonIT/__init__.py +0 -0
  145. WMCore/Services/PyCondor/PyCondorAPI.py +154 -0
  146. WMCore/Services/PyCondor/__init__.py +0 -0
  147. WMCore/Services/ReqMgr/ReqMgr.py +261 -0
  148. WMCore/Services/ReqMgr/__init__.py +0 -0
  149. WMCore/Services/ReqMgrAux/ReqMgrAux.py +419 -0
  150. WMCore/Services/ReqMgrAux/__init__.py +0 -0
  151. WMCore/Services/RequestDB/RequestDBReader.py +267 -0
  152. WMCore/Services/RequestDB/RequestDBWriter.py +39 -0
  153. WMCore/Services/RequestDB/__init__.py +0 -0
  154. WMCore/Services/Requests.py +624 -0
  155. WMCore/Services/Rucio/Rucio.py +1290 -0
  156. WMCore/Services/Rucio/RucioUtils.py +74 -0
  157. WMCore/Services/Rucio/__init__.py +0 -0
  158. WMCore/Services/RucioConMon/RucioConMon.py +121 -0
  159. WMCore/Services/RucioConMon/__init__.py +0 -0
  160. WMCore/Services/Service.py +400 -0
  161. WMCore/Services/StompAMQ/__init__.py +0 -0
  162. WMCore/Services/TagCollector/TagCollector.py +155 -0
  163. WMCore/Services/TagCollector/XMLUtils.py +98 -0
  164. WMCore/Services/TagCollector/__init__.py +0 -0
  165. WMCore/Services/UUIDLib.py +13 -0
  166. WMCore/Services/UserFileCache/UserFileCache.py +160 -0
  167. WMCore/Services/UserFileCache/__init__.py +8 -0
  168. WMCore/Services/WMAgent/WMAgent.py +63 -0
  169. WMCore/Services/WMAgent/__init__.py +0 -0
  170. WMCore/Services/WMArchive/CMSSWMetrics.py +526 -0
  171. WMCore/Services/WMArchive/DataMap.py +463 -0
  172. WMCore/Services/WMArchive/WMArchive.py +33 -0
  173. WMCore/Services/WMArchive/__init__.py +0 -0
  174. WMCore/Services/WMBS/WMBS.py +97 -0
  175. WMCore/Services/WMBS/__init__.py +0 -0
  176. WMCore/Services/WMStats/DataStruct/RequestInfoCollection.py +300 -0
  177. WMCore/Services/WMStats/DataStruct/__init__.py +0 -0
  178. WMCore/Services/WMStats/WMStatsPycurl.py +145 -0
  179. WMCore/Services/WMStats/WMStatsReader.py +445 -0
  180. WMCore/Services/WMStats/WMStatsWriter.py +273 -0
  181. WMCore/Services/WMStats/__init__.py +0 -0
  182. WMCore/Services/WMStatsServer/WMStatsServer.py +134 -0
  183. WMCore/Services/WMStatsServer/__init__.py +0 -0
  184. WMCore/Services/WorkQueue/WorkQueue.py +492 -0
  185. WMCore/Services/WorkQueue/__init__.py +0 -0
  186. WMCore/Services/__init__.py +8 -0
  187. WMCore/Services/pycurl_manager.py +574 -0
  188. WMCore/WMBase.py +50 -0
  189. WMCore/WMConnectionBase.py +164 -0
  190. WMCore/WMException.py +183 -0
  191. WMCore/WMExceptions.py +269 -0
  192. WMCore/WMFactory.py +76 -0
  193. WMCore/WMInit.py +377 -0
  194. WMCore/WMLogging.py +104 -0
  195. WMCore/WMSpec/ConfigSectionTree.py +442 -0
  196. WMCore/WMSpec/Persistency.py +135 -0
  197. WMCore/WMSpec/Steps/BuildMaster.py +87 -0
  198. WMCore/WMSpec/Steps/BuildTools.py +201 -0
  199. WMCore/WMSpec/Steps/Builder.py +97 -0
  200. WMCore/WMSpec/Steps/Diagnostic.py +89 -0
  201. WMCore/WMSpec/Steps/Emulator.py +62 -0
  202. WMCore/WMSpec/Steps/ExecuteMaster.py +208 -0
  203. WMCore/WMSpec/Steps/Executor.py +210 -0
  204. WMCore/WMSpec/Steps/StepFactory.py +213 -0
  205. WMCore/WMSpec/Steps/TaskEmulator.py +75 -0
  206. WMCore/WMSpec/Steps/Template.py +204 -0
  207. WMCore/WMSpec/Steps/Templates/AlcaHarvest.py +76 -0
  208. WMCore/WMSpec/Steps/Templates/CMSSW.py +613 -0
  209. WMCore/WMSpec/Steps/Templates/DQMUpload.py +59 -0
  210. WMCore/WMSpec/Steps/Templates/DeleteFiles.py +70 -0
  211. WMCore/WMSpec/Steps/Templates/LogArchive.py +84 -0
  212. WMCore/WMSpec/Steps/Templates/LogCollect.py +105 -0
  213. WMCore/WMSpec/Steps/Templates/StageOut.py +105 -0
  214. WMCore/WMSpec/Steps/Templates/__init__.py +10 -0
  215. WMCore/WMSpec/Steps/WMExecutionFailure.py +21 -0
  216. WMCore/WMSpec/Steps/__init__.py +8 -0
  217. WMCore/WMSpec/Utilities.py +63 -0
  218. WMCore/WMSpec/WMSpecErrors.py +12 -0
  219. WMCore/WMSpec/WMStep.py +347 -0
  220. WMCore/WMSpec/WMTask.py +1997 -0
  221. WMCore/WMSpec/WMWorkload.py +2288 -0
  222. WMCore/WMSpec/WMWorkloadTools.py +382 -0
  223. WMCore/WMSpec/__init__.py +9 -0
  224. WMCore/WorkQueue/DataLocationMapper.py +273 -0
  225. WMCore/WorkQueue/DataStructs/ACDCBlock.py +47 -0
  226. WMCore/WorkQueue/DataStructs/Block.py +48 -0
  227. WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py +148 -0
  228. WMCore/WorkQueue/DataStructs/WorkQueueElement.py +274 -0
  229. WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py +152 -0
  230. WMCore/WorkQueue/DataStructs/WorkQueueElementsSummary.py +185 -0
  231. WMCore/WorkQueue/DataStructs/__init__.py +0 -0
  232. WMCore/WorkQueue/Policy/End/EndPolicyInterface.py +44 -0
  233. WMCore/WorkQueue/Policy/End/SingleShot.py +22 -0
  234. WMCore/WorkQueue/Policy/End/__init__.py +32 -0
  235. WMCore/WorkQueue/Policy/PolicyInterface.py +17 -0
  236. WMCore/WorkQueue/Policy/Start/Block.py +258 -0
  237. WMCore/WorkQueue/Policy/Start/Dataset.py +180 -0
  238. WMCore/WorkQueue/Policy/Start/MonteCarlo.py +131 -0
  239. WMCore/WorkQueue/Policy/Start/ResubmitBlock.py +171 -0
  240. WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +316 -0
  241. WMCore/WorkQueue/Policy/Start/__init__.py +34 -0
  242. WMCore/WorkQueue/Policy/__init__.py +57 -0
  243. WMCore/WorkQueue/WMBSHelper.py +772 -0
  244. WMCore/WorkQueue/WorkQueue.py +1237 -0
  245. WMCore/WorkQueue/WorkQueueBackend.py +750 -0
  246. WMCore/WorkQueue/WorkQueueBase.py +39 -0
  247. WMCore/WorkQueue/WorkQueueExceptions.py +44 -0
  248. WMCore/WorkQueue/WorkQueueReqMgrInterface.py +278 -0
  249. WMCore/WorkQueue/WorkQueueUtils.py +130 -0
  250. WMCore/WorkQueue/__init__.py +13 -0
  251. WMCore/Wrappers/JsonWrapper/JSONThunker.py +342 -0
  252. WMCore/Wrappers/JsonWrapper/__init__.py +7 -0
  253. WMCore/Wrappers/__init__.py +6 -0
  254. WMCore/__init__.py +10 -0
  255. wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-patch +15 -0
  256. wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-unpatch +8 -0
  257. wmglobalqueue-2.4.5.1.data/data/bin/wmc-httpd +3 -0
  258. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/.couchapprc +1 -0
  259. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/README.md +40 -0
  260. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/index.html +264 -0
  261. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/ElementInfoByWorkflow.js +96 -0
  262. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/StuckElementInfo.js +57 -0
  263. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/WorkloadInfoTable.js +80 -0
  264. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/dataTable.js +70 -0
  265. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/namespace.js +23 -0
  266. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/style/main.css +75 -0
  267. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/couchapp.json +4 -0
  268. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/childQueueFilter.js +13 -0
  269. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/filterDeletedDocs.js +3 -0
  270. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/queueFilter.js +11 -0
  271. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/language +1 -0
  272. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/mustache.js +333 -0
  273. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/validate.js +27 -0
  274. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/workqueue_utils.js +61 -0
  275. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/elementsDetail.js +28 -0
  276. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/filter.js +86 -0
  277. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/stuckElements.js +38 -0
  278. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workRestrictions.js +153 -0
  279. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workflowSummary.js +28 -0
  280. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/rewrites.json +73 -0
  281. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/redirect.js +23 -0
  282. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/status.js +40 -0
  283. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/ElementSummaryByWorkflow.html +27 -0
  284. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/StuckElementSummary.html +26 -0
  285. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/TaskStatus.html +23 -0
  286. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/WorkflowSummary.html +27 -0
  287. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/workqueue-common-lib.html +2 -0
  288. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib-remote.html +16 -0
  289. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib.html +18 -0
  290. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/updates/in-place.js +50 -0
  291. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/validate_doc_update.js +8 -0
  292. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.couch.app.js +235 -0
  293. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.pathbinder.js +173 -0
  294. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/map.js +8 -0
  295. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/reduce.js +2 -0
  296. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/map.js +8 -0
  297. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/reduce.js +2 -0
  298. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/map.js +8 -0
  299. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/reduce.js +2 -0
  300. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/map.js +11 -0
  301. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/reduce.js +1 -0
  302. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/availableByPriority/map.js +6 -0
  303. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/conflicts/map.js +5 -0
  304. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elements/map.js +5 -0
  305. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByData/map.js +8 -0
  306. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParent/map.js +8 -0
  307. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParentData/map.js +8 -0
  308. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByPileupData/map.js +8 -0
  309. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByStatus/map.js +8 -0
  310. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsBySubscription/map.js +6 -0
  311. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/map.js +8 -0
  312. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/reduce.js +3 -0
  313. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsDetailByWorkflowAndStatus/map.js +26 -0
  314. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/map.js +10 -0
  315. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/reduce.js +1 -0
  316. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/map.js +6 -0
  317. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/reduce.js +1 -0
  318. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/map.js +6 -0
  319. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/reduce.js +1 -0
  320. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/map.js +6 -0
  321. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/reduce.js +1 -0
  322. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/map.js +6 -0
  323. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/reduce.js +1 -0
  324. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/map.js +6 -0
  325. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/reduce.js +1 -0
  326. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/map.js +6 -0
  327. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/reduce.js +1 -0
  328. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/openRequests/map.js +6 -0
  329. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/recent-items/map.js +5 -0
  330. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/map.js +6 -0
  331. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/reduce.js +1 -0
  332. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/specsByWorkflow/map.js +5 -0
  333. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/stuckElements/map.js +38 -0
  334. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/map.js +12 -0
  335. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/reduce.js +3 -0
  336. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/map.js +6 -0
  337. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/reduce.js +2 -0
  338. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/map.js +6 -0
  339. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/reduce.js +2 -0
  340. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/map.js +9 -0
  341. wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/reduce.js +10 -0
  342. wmglobalqueue-2.4.5.1.dist-info/METADATA +26 -0
  343. wmglobalqueue-2.4.5.1.dist-info/RECORD +347 -0
  344. wmglobalqueue-2.4.5.1.dist-info/WHEEL +5 -0
  345. wmglobalqueue-2.4.5.1.dist-info/licenses/LICENSE +202 -0
  346. wmglobalqueue-2.4.5.1.dist-info/licenses/NOTICE +16 -0
  347. wmglobalqueue-2.4.5.1.dist-info/top_level.txt +2 -0
@@ -0,0 +1,261 @@
1
+ from builtins import str, bytes
2
+ from future.utils import viewitems
3
+
4
+ import json
5
+ import logging
6
+
7
+ from WMCore.Services.Service import Service
8
+ from WMCore.Cache.GenericDataCache import MemoryCacheStruct
9
+
10
+
11
+ class ReqMgr(Service):
12
+ """
13
+ API for dealing with retrieving information from RequestManager dataservice
14
+
15
+ """
16
+
17
+ def __init__(self, url, header=None, logger=None):
18
+ """
19
+ responseType will be either xml or json
20
+ """
21
+
22
+ httpDict = {}
23
+ header = header or {}
24
+ # url is end point
25
+ httpDict['endpoint'] = "%s/data" % url
26
+ httpDict['logger'] = logger if logger else logging.getLogger()
27
+
28
+ # cherrypy converts request.body to params when content type is set
29
+ # application/x-www-form-urlencodeds
30
+ httpDict.setdefault("content_type", 'application/json')
31
+ httpDict.setdefault('cacheduration', 0)
32
+ httpDict.setdefault("accept_type", "application/json")
33
+ httpDict.update(header)
34
+ self.encoder = json.dumps
35
+ Service.__init__(self, httpDict)
36
+ # This is only for the unittest: never set it true unless it is unittest
37
+ self._noStale = False
38
+
39
+ def _getResult(self, callname, clearCache=True, args=None, verb="GET",
40
+ encoder=json.loads, decoder=json.loads, contentType=None):
41
+ """
42
+ _getResult_
43
+ """
44
+ result = ''
45
+ cfile = callname.replace("/", "_")
46
+ if clearCache:
47
+ self.clearCache(cfile, args, verb)
48
+
49
+ f = self.refreshCache(cfile, callname, args, encoder=encoder,
50
+ verb=verb, contentType=contentType)
51
+ result = f.read()
52
+ f.close()
53
+
54
+ if result and decoder:
55
+ result = decoder(result)
56
+
57
+ return result['result']
58
+
59
+ def _createQuery(self, queryDict):
60
+ """
61
+ _createQuery
62
+ :param queryDict: is the format of {name: values} fair. value can be sting, int or list
63
+ :type queryDict: dict
64
+ :returns: url query string
65
+ """
66
+ if self._noStale:
67
+ args = "_nostale=true&"
68
+ else:
69
+ args = ""
70
+ for name, values in viewitems(queryDict):
71
+ if isinstance(values, (str, bytes, int)):
72
+ values = [values]
73
+ for val in values:
74
+ args += '%s=%s&' % (name, val)
75
+
76
+ return args.rstrip('&')
77
+
78
+ def getParentLocks(self):
79
+ """
80
+ _getParentLocks_
81
+
82
+ A public method to return the parent locks from ReqMgr.
83
+ :returns: A list of datasets
84
+
85
+ """
86
+ callname = 'parentlocks'
87
+ result = self._getResult(callname, clearCache=True, verb="GET")
88
+ return result[0][callname]
89
+
90
+ def getRequestByNames(self, names):
91
+
92
+ """
93
+ _getRequestByNames_
94
+
95
+ :param names: list or sting of request name(s)
96
+ :type statusList: list, str
97
+ :returns: list of dict or list of request names depending on the detail value
98
+ -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
99
+ 'BlockWhitelist': [],
100
+ 'CMSSWVersion': 'CMSSW_4_4_2_patch2',
101
+ ....
102
+ '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
103
+ 'inputMode': 'couchDB'}}]
104
+ TODO: need proper error handling if status is not 200 from orignal reporting.
105
+ """
106
+
107
+ query = self._createQuery({'name': names})
108
+ callname = 'request?%s' % query
109
+ return self._getResult(callname, verb="GET")
110
+
111
+ def getRequestByStatus(self, statusList, detail=True):
112
+ """
113
+ _getRequestByStatus_
114
+
115
+ :param statusList: list of status
116
+ :type statusList: list
117
+ :param detail: boolean of request list.
118
+ :type detail: boolean
119
+ :returns: list of dict or list of request names depending on the detail value
120
+ -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
121
+ 'BlockWhitelist': [],
122
+ 'CMSSWVersion': 'CMSSW_4_4_2_patch2',
123
+ ....
124
+ '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
125
+ 'inputMode': 'couchDB'}}]
126
+ TODO: need proper error handling if status is not 200 from orignal reporting.
127
+ """
128
+
129
+ query = self._createQuery({'status': statusList, 'detail': detail})
130
+ callname = 'request?%s' % query
131
+ return self._getResult(callname, verb="GET")
132
+
133
+ def getRequestTasks(self, name):
134
+
135
+ """
136
+ _getRequestTasks_
137
+
138
+ :param name: request name
139
+ :type string: str
140
+ :returns: list of dict or list of request names depending on the detail value
141
+ -- {result:[{requestTask}, {requestTask}]}
142
+ """
143
+
144
+ callname = 'splitting/%s' % name
145
+ return self._getResult(callname, verb="GET")
146
+
147
+ def getConfig(self, name):
148
+
149
+ """
150
+ _getConfig_
151
+
152
+ :param name: request name
153
+ :type string: str
154
+ :returns: list of dict or list of request names depending on the detail value
155
+ -- {result:[config_string]}
156
+ """
157
+
158
+ callname = 'workload_config/%s' % name
159
+ return self._getResult(callname, verb="GET")
160
+
161
+ def insertRequests(self, requestDict):
162
+ """
163
+ _insertRequests_
164
+
165
+ :param requestDict: request argument dictionary
166
+ :type requestDict: dict
167
+ :returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
168
+ 'BlockWhitelist': [],
169
+ 'CMSSWVersion': 'CMSSW_4_4_2_patch2',
170
+ ....
171
+ '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
172
+ 'inputMode': 'couchDB'}}]
173
+ TODO: need proper error handling if status is not 200 from orignal reporting.
174
+ """
175
+ return self["requests"].post('request', requestDict)[0]['result']
176
+
177
+ def updateRequestStatus(self, request, status):
178
+ """
179
+ _updateRequestStatus_
180
+
181
+ :param request: request(workflow name)
182
+ :type reqeust: str
183
+ :param status: status of workflow to update (i.e. 'assigned')
184
+ :type status: str
185
+ :returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
186
+ 'BlockWhitelist': [],
187
+ 'CMSSWVersion': 'CMSSW_4_4_2_patch2',
188
+ ....
189
+ '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
190
+ 'inputMode': 'couchDB'}}]
191
+ TODO: need proper error handling if status is not 200 from orignal reporting.
192
+ """
193
+
194
+ status = {"RequestStatus": status}
195
+ status["RequestName"] = request
196
+ return self["requests"].put('request', status)[0]['result']
197
+
198
+ def updateRequestStats(self, request, stats):
199
+ """
200
+ put initial stats for request
201
+ param: stats: dict of {'total_jobs': 100, 'input_lumis': 100,
202
+ 'input_events': 100, 'input_num_files': 100}
203
+ specific to ReqMgr app. not implemented for T0
204
+ """
205
+ # this stats dict will be validated on the server side
206
+ self.updateRequestProperty(request, stats)
207
+
208
+ def updateRequestProperty(self, request, propDict):
209
+ """
210
+ _updateRequestProperty_
211
+ :param request: request(workflow name)
212
+ :type reqeust: str
213
+ :param propDict: request property with key value -- {"SiteWhitelist": ["ABC"], "SiteBlacklist": ["A"], "RequestStatus": "assigned"}
214
+ :type propDict: dict
215
+ :returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
216
+ 'BlockWhitelist': [],
217
+ 'CMSSWVersion': 'CMSSW_4_4_2_patch2',
218
+ ....
219
+ '_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
220
+ 'inputMode': 'couchDB'}}]
221
+ """
222
+ propDict["RequestName"] = request
223
+ return self["requests"].put('request/%s' % request, propDict)[0]['result']
224
+
225
+ def getAbortedAndForceCompleteRequestsFromMemoryCache(self, expire=0):
226
+ """
227
+ _getAbortedAndForceCompleteRequestsFromMemoryCache_
228
+ """
229
+
230
+ maskStates = ["aborted", "aborted-completed", "force-complete"]
231
+ return self.getRequestByStatusFromMemoryCache(maskStates, expire)
232
+
233
+ def getRequestByStatusFromMemoryCache(self, statusList, expire=0):
234
+
235
+ return MemoryCacheStruct(expire=expire, func=self.getRequestByStatus, initCacheValue=[],
236
+ logger=self['logger'], kwargs={'statusList': statusList, "detail": False})
237
+
238
+ def cloneRequest(self, requestName, overwrittenParams=None):
239
+ """
240
+ _cloneRequest_
241
+
242
+ :param requestName: request name which need to be cloned
243
+ :type reqeust: str
244
+ :param overwrittenParams: status of workflow to update (i.e. 'assigned')
245
+ :type overwrittenParams: dict (dictionary format of parameters need to be overwritten)
246
+ :returns list of dict [{'request': cloned_request_name}]
247
+ """
248
+ params = overwrittenParams or {}
249
+ uri = 'request/clone/%s' % requestName
250
+ return self["requests"].post(uri, params)[0]['result']
251
+
252
+ def getGenericRequestInfo(self, queryDict):
253
+ """
254
+ _getGenericRequestInfo_
255
+ :param queryDict: query dictionary which will be converted to url query string.
256
+ :type queryDict: dict
257
+ :returns list of dict [{'request': { any info request by mask}]
258
+ """
259
+ query = self._createQuery(queryDict)
260
+ callname = 'request?%s' % query
261
+ return self._getResult(callname, verb="GET")
File without changes
@@ -0,0 +1,419 @@
1
+ from __future__ import division, print_function
2
+
3
+ import json
4
+ import logging
5
+
6
+ from Utils.Utilities import diskUse
7
+ from WMCore.Cache.GenericDataCache import MemoryCacheStruct
8
+ from WMCore.Services.Service import Service
9
+
10
+
11
+ class ReqMgrAux(Service):
12
+ """
13
+ API for dealing with retrieving information from RequestManager dataservice
14
+
15
+ """
16
+
17
+ def __init__(self, url, httpDict=None, logger=None):
18
+ """
19
+ responseType will be either xml or json
20
+ """
21
+
22
+ httpDict = httpDict or {}
23
+ # url is end point
24
+ httpDict['endpoint'] = "%s/data" % url
25
+ httpDict['logger'] = logger if logger else logging.getLogger()
26
+
27
+ # cherrypy converts request.body to params when content type is set
28
+ # application/x-www-form-urlencodeds
29
+ httpDict.setdefault("content_type", 'application/json')
30
+ httpDict.setdefault('cacheduration', 0)
31
+ self.cacheExpire = httpDict['cacheduration']
32
+ httpDict.setdefault("accept_type", "application/json")
33
+ self.encoder = json.dumps
34
+ Service.__init__(self, httpDict)
35
+ # This is only for the unittest: never set it true unless it is unittest
36
+ self._noStale = False
37
+
38
+ def _getResult(self, callname, clearCache=False, args=None, verb="GET",
39
+ encoder=json.loads, decoder=json.loads, contentType=None):
40
+ """
41
+ _getResult_
42
+ """
43
+ cfile = callname.replace("/", "_")
44
+ if clearCache:
45
+ self.clearCache(cfile, args, verb)
46
+
47
+ f = self.refreshCache(cfile, callname, args, encoder=encoder,
48
+ verb=verb, contentType=contentType)
49
+ result = f.read()
50
+ f.close()
51
+
52
+ if result and decoder:
53
+ result = decoder(result)
54
+
55
+ return result['result']
56
+
57
+ def _getDataFromMemoryCache(self, callname):
58
+ cache = MemoryCacheStruct(expire=self.cacheExpire, func=self._getResult, initCacheValue={},
59
+ logger=self['logger'], kwargs={'callname': callname, "verb": "GET"})
60
+ return cache.getData()
61
+
62
+ def getCMSSWVersion(self):
63
+ """
64
+ get dictionary format of architecture and cmssw versions
65
+ i.e.
66
+ {"slc5_amd64_gcc462": [
67
+ "CMSSW_5_3_4_patch1",
68
+ "CMSSW_5_2_9",
69
+ "CMSSW_5_3_4_patch2",
70
+ "CMSSW_5_3_3_cand1_patch1"],
71
+ ...
72
+ }
73
+ """
74
+ return self._getDataFromMemoryCache('cmsswversions')
75
+
76
+ def _updateRecords(self, callName, resource, kwparams):
77
+ """
78
+ Fetches the original document, locally update it according to
79
+ the key/value pairs provided and update the document.
80
+ :param callName: resource to be requested
81
+ :param resource: name of the resource/document to be updated
82
+ :param kwparams: a dictionary with the content to be updated
83
+ :return: a dictionary with the CouchDB response
84
+ """
85
+ apiMap = {'wmagentconfig': self.getWMAgentConfig,
86
+ 'campaignconfig': self.getCampaignConfig,
87
+ 'transferinfo': self.getTransferInfo}
88
+
89
+ thisDoc = apiMap[callName](resource)
90
+ # getWMAgentConfig method returns directly the document, while the others
91
+ # return a list of document(s)
92
+ if isinstance(thisDoc, (list, set)):
93
+ thisDoc = thisDoc[0]
94
+ thisDoc.update(kwparams)
95
+ return self["requests"].put("%s/%s" % (callName, resource), thisDoc)[0]['result']
96
+
97
+ def populateCMSSWVersion(self, tcUrl, **kwargs):
98
+ """
99
+ Query TagCollector and update the CMSSW versions document in Couch
100
+ :return: a boolean with the result of the operation
101
+ """
102
+ from WMCore.Services.TagCollector.TagCollector import TagCollector
103
+ cmsswVersions = TagCollector(tcUrl, **kwargs).releases_by_architecture()
104
+ resp = self["requests"].put('cmsswversions', cmsswVersions)[0]['result']
105
+
106
+ if resp and resp[0].get("ok", False):
107
+ self["logger"].info("CMSSW document successfully updated.")
108
+ return True
109
+
110
+ msg = "Failed to update CMSSW document. Response: %s" % resp
111
+ self["logger"].warning(msg)
112
+ return False
113
+
114
+ def getWMAgentConfig(self, agentName):
115
+ """
116
+ retrieve agent configuration reqmgr aux db.
117
+ """
118
+ agentConfig = self._getDataFromMemoryCache('wmagentconfig/%s' % agentName)
119
+
120
+ if len(agentConfig) != 1:
121
+ # something wrong with database record. returns default value for both case.
122
+ self["logger"].warning("agent config is not correct: %s" % agentConfig)
123
+ return {}
124
+
125
+ return agentConfig[0]
126
+
127
+ def postWMAgentConfig(self, agentName, agentConfig):
128
+ """
129
+ Create a new WMAgent configuration file in ReqMgrAux.
130
+ If document already exists, nothing happens.
131
+ """
132
+ return self["requests"].post('wmagentconfig/%s' % agentName, agentConfig)[0]['result']
133
+
134
+ def updateWMAgentConfig(self, agentName, content, inPlace=False):
135
+ """
136
+ Update the corresponding agent configuration with the content
137
+ provided, replacing the old document.
138
+ If inPlace is set to True, then a modification of the current
139
+ document is performed, according to the key/value pairs provided.
140
+ :param agentName: name of the agent/document in couch
141
+ :param content: a dictionary with the data to be updated
142
+ :param inPlace: a boolean defining whether to perform a replace
143
+ or a update modification
144
+ :return: a boolean with the result of the operation
145
+ """
146
+ api = 'wmagentconfig'
147
+ if inPlace:
148
+ resp = self._updateRecords(api, agentName, content)
149
+ else:
150
+ resp = self["requests"].put("%s/%s" % (api, agentName), content)[0]['result']
151
+
152
+ if resp and resp[0].get("ok", False):
153
+ self["logger"].info("Update in-place: %s for agent: %s was successful.", inPlace, agentName)
154
+ return True
155
+
156
+ msg = "Failed to update agent: %s in-place: %s. Response: %s" % (agentName, inPlace, resp)
157
+ self["logger"].warning(msg)
158
+ return False
159
+
160
+ def getCampaignConfig(self, campaignName):
161
+ """
162
+ get campaign config for transferor function in unified ReqMgr2MS.
163
+ """
164
+ return self._getDataFromMemoryCache('campaignconfig/%s' % campaignName)
165
+
166
+ def postCampaignConfig(self, campaignName, campaignConfig):
167
+ """
168
+ Create a new campaign configuration document
169
+
170
+ :param campaignName
171
+ :type basestringg
172
+ :param campaignConfig:
173
+ :type dict - only can replace whole campaign no partial parameters
174
+
175
+ campaignName: "HIRun2015":
176
+ campaignConfig: {
177
+ "go": true,
178
+ "labels" : ["02May2016","25Aug2016"],
179
+ "overflow" : {"PRIM" : {}},
180
+ "DDMcopies": {
181
+ "all" : { "N" : 2 }
182
+ },
183
+ "custodial_override" : ["DQMIO"],
184
+ "fractionpass": 1.0,
185
+ "lumisize" : -1,
186
+ "maxcopies" : 1,
187
+ "custodial": "T1_FR_CCIN2P3_MSS",
188
+ "parameters" :{
189
+ "NonCustodialSites" : ["T2_US_Vanderbilt"],
190
+ "SiteBlacklist": [
191
+ "T1_US_FNAL",
192
+ "T2_US_Purdue",
193
+ "T2_US_Caltech",
194
+ "T2_US_Florida",
195
+ "T2_US_Nebraska",
196
+ "T2_US_UCSD",
197
+ "T2_US_Wisconsin"
198
+ ]
199
+ }
200
+ }
201
+
202
+ :return: CouchDB response dictionary
203
+ """
204
+ return self["requests"].post('campaignconfig/%s' % campaignName, campaignConfig)[0]['result']
205
+
206
+ def updateCampaignConfig(self, campaignName, content, inPlace=False):
207
+ """
208
+ Update the corresponding campaign configuration with the content
209
+ provided, replacing the old document.
210
+ If inPlace is set to True, then a modification of the current
211
+ document is performed, according to the key/value pairs provided.
212
+ :param campaignName: name of the campaign document in couch
213
+ :param content: a dictionary with the data to be updated
214
+ :param inPlace: a boolean defining whether to perform a replace
215
+ or a update modification
216
+ :return: a boolean with the result of the operation
217
+ """
218
+ api = 'campaignconfig'
219
+ if inPlace:
220
+ resp = self._updateRecords(api, campaignName, content)
221
+ else:
222
+ resp = self["requests"].put("%s/%s" % (api, campaignName), content)[0]['result']
223
+
224
+ if resp and resp[0].get("ok", False):
225
+ self["logger"].info("Update in-place: %s for campaign: %s was successful.", inPlace, campaignName)
226
+ return True
227
+ msg = "Failed to update campaign: %s in-place: %s. Response: %s" % (campaignName, inPlace, resp)
228
+ self["logger"].warning(msg)
229
+ return False
230
+
231
+ def getUnifiedConfig(self, docName=None):
232
+ """
233
+ Retrieve a Unified document type from the auxiliary db.
234
+ If docName is not provided, then fetch the default UNIFIED_CONFIG doc
235
+ """
236
+ if docName:
237
+ unifiedConfig = self._getDataFromMemoryCache('unifiedconfig/%s' % docName)
238
+ else:
239
+ unifiedConfig = self._getDataFromMemoryCache('unifiedconfig')
240
+
241
+ if not unifiedConfig:
242
+ self["logger"].warning("Unified configuration document not found. Result: %s", unifiedConfig)
243
+
244
+ return unifiedConfig
245
+
246
+ def postUnifiedConfig(self, unifiedConfig, docName=None):
247
+ """
248
+ Create a new unified configuration document
249
+ """
250
+ if docName:
251
+ return self["requests"].post('unifiedconfig/%s' % docName, unifiedConfig)[0]['result']
252
+ return self["requests"].post('unifiedconfig', unifiedConfig)[0]['result']
253
+
254
+ def updateUnifiedConfig(self, content, docName=None):
255
+ """
256
+ Update the unified configuration with the content provided, replacing
257
+ the old document.
258
+ :param content: a dictionary with the data to be updated
259
+ :return: a boolean with the result of the operation
260
+ """
261
+ api = 'unifiedconfig'
262
+ if docName:
263
+ resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
264
+ else:
265
+ resp = self["requests"].put("%s" % api, content)[0]['result']
266
+
267
+ if resp and resp[0].get("ok", False):
268
+ self["logger"].info("Unified configuration successfully updated.")
269
+ return True
270
+
271
+ self["logger"].warning("Failed to update the unified configuration. Response: %s", resp)
272
+ return False
273
+
274
+ def getTransferInfo(self, docName):
275
+ """
276
+ get a workflow transfer document, to be used by unified ReqMgr2MS.
277
+ """
278
+ return self._getDataFromMemoryCache('transferinfo/%s' % docName)
279
+
280
+ def postTransferInfo(self, docName, transferInfo):
281
+ """
282
+ Create a new workflow transfer document
283
+
284
+ :param docName: the name of the document to be created
285
+ :param transferInfo: a dictionary with the document content
286
+ :return: CouchDB response dictionary
287
+ """
288
+ return self["requests"].post('transferinfo/%s' % docName, transferInfo)[0]['result']
289
+
290
+ def updateTransferInfo(self, docName, content, inPlace=False):
291
+ """
292
+ Update the corresponding workflow transfer document with the content
293
+ provided, replacing the old one.
294
+ If inPlace is set to True, then a modification of the current
295
+ document is performed, according to the key/value pairs provided.
296
+ :param docName: name of the transfer document in couch
297
+ :param content: a dictionary with the data to be updated
298
+ :param inPlace: a boolean defining whether to perform a replace
299
+ or a update modification
300
+ :return: a boolean with the result of the operation
301
+ """
302
+ api = 'transferinfo'
303
+ if inPlace:
304
+ resp = self._updateRecords(api, docName, content)
305
+ else:
306
+ resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
307
+
308
+ if resp and resp[0].get("ok", False):
309
+ self["logger"].info("Update in-place: %s for transfer doc: %s was successful.", inPlace, docName)
310
+ return True
311
+ msg = "Failed to update transfer doc: %s in-place: %s. Response: %s" % (docName, inPlace, resp)
312
+ self["logger"].warning(msg)
313
+ return False
314
+
315
+ def updateParentLocks(self, content, docName=None):
316
+ """
317
+ Update the list of locked parent datasets with the content provided, replacing
318
+ the old document.
319
+ :param content: a dictionary with the data to be updated
320
+ :return: a boolean with the result of the operation
321
+ """
322
+ api = 'parentlocks'
323
+ if docName:
324
+ resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
325
+ else:
326
+ resp = self["requests"].put("%s" % api, content)[0]['result']
327
+
328
+ if resp and resp[0].get("ok", False):
329
+ self["logger"].info("Parent dataset locks successfully updated.")
330
+ return True
331
+
332
+ self["logger"].warning("Failed to update parent dataset locks. Response: %s", resp)
333
+ return False
334
+
335
+ def getParentLocks(self):
336
+ """
337
+ get the list of parent locks
338
+ """
339
+ return self._getDataFromMemoryCache('parentlocks')
340
+
341
+ def deleteConfigDoc(self, docType, docName):
342
+ """
343
+ Given a document type and a document name, delete it from the
344
+ Couch auxiliary DB.
345
+ :param docType: a string with the document type to be deleted, which
346
+ corresponds to the same API name
347
+ :param docName: the name of the document to be deleted
348
+ :return: response dictionary from the couch operation
349
+ """
350
+ allowedValues = ('wmagentconfig', 'campaignconfig', 'unifiedconfig', 'transferinfo')
351
+ if docType not in allowedValues:
352
+ msg = "Document type: '%s' not allowed for deletion." % docType
353
+ msg += " Supported documents are: %s" % allowedValues
354
+ self["logger"].warning(msg)
355
+ else:
356
+ return self["requests"].delete('%s/%s' % (docType, docName))[0]['result']
357
+
358
+
359
+ AUXDB_AGENT_CONFIG_CACHE = {}
360
+
361
+
362
+ # function to check whether agent is should be in draining status.
363
+ def isDrainMode(config):
364
+ """
365
+ config is loaded from WMAgentConfig in local config
366
+ agentDrainMode is boolean value. (if it is passed update the WMAgentConfig in couchdb)
367
+ """
368
+ if hasattr(config, "Tier0Feeder"):
369
+ return False
370
+
371
+ global AUXDB_AGENT_CONFIG_CACHE
372
+
373
+ reqMgrAux = ReqMgrAux(config.General.ReqMgr2ServiceURL)
374
+ agentConfig = reqMgrAux.getWMAgentConfig(config.Agent.hostName)
375
+ if "UserDrainMode" in agentConfig and "AgentDrainMode" in agentConfig:
376
+ AUXDB_AGENT_CONFIG_CACHE = agentConfig
377
+ return agentConfig["UserDrainMode"] or agentConfig["AgentDrainMode"]
378
+ # if the cache is empty this will raise Key not exist exception.
379
+ return AUXDB_AGENT_CONFIG_CACHE["UserDrainMode"] or AUXDB_AGENT_CONFIG_CACHE["AgentDrainMode"]
380
+
381
+
382
+ def listDiskUsageOverThreshold(config, updateDB):
383
+ """
384
+ check whether disk usage is over threshold,
385
+ return list of disk paths
386
+ if updateDB is True update the aux couch db value.
387
+ This function contains both check an update to avoide multiple db calls.
388
+ """
389
+ defaultDiskThreshold = 85
390
+ defaultIgnoredDisks = []
391
+ if hasattr(config, "Tier0Feeder"):
392
+ # get the value from config.
393
+ ignoredDisks = getattr(config.AgentStatusWatcher, "ignoreDisks", defaultIgnoredDisks)
394
+ diskUseThreshold = getattr(config.AgentStatusWatcher, "diskUseThreshold", defaultDiskThreshold)
395
+ t0Flag = True
396
+ else:
397
+ reqMgrAux = ReqMgrAux(config.General.ReqMgr2ServiceURL)
398
+ agentConfig = reqMgrAux.getWMAgentConfig(config.Agent.hostName)
399
+ diskUseThreshold = agentConfig.get("DiskUseThreshold", defaultDiskThreshold)
400
+ ignoredDisks = agentConfig.get("IgnoreDisks", defaultIgnoredDisks)
401
+ t0Flag = False
402
+
403
+ # Disk space warning
404
+ diskUseList = diskUse()
405
+ overThresholdDisks = []
406
+ for disk in diskUseList:
407
+ if (float(disk['percent'].strip('%')) >= diskUseThreshold and
408
+ disk['mounted'] not in ignoredDisks):
409
+ overThresholdDisks.append(disk)
410
+
411
+ if updateDB and not t0Flag:
412
+ agentDrainMode = bool(len(overThresholdDisks))
413
+ if agentConfig and (agentDrainMode != agentConfig.get("AgentDrainMode")):
414
+ if agentConfig.get("UserDrainMode") is True:
415
+ logging.info("Agent is already in UserDrainMode: True mode. No need to update.")
416
+ else:
417
+ reqMgrAux.updateWMAgentConfig(config.Agent.hostName, {"AgentDrainMode": agentDrainMode}, inPlace=True)
418
+
419
+ return overThresholdDisks
File without changes