wmglobalqueue 2.3.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of wmglobalqueue might be problematic. Click here for more details.

Files changed (345) hide show
  1. Utils/CPMetrics.py +270 -0
  2. Utils/CertTools.py +62 -0
  3. Utils/EmailAlert.py +50 -0
  4. Utils/ExtendedUnitTestCase.py +62 -0
  5. Utils/FileTools.py +182 -0
  6. Utils/IteratorTools.py +80 -0
  7. Utils/MathUtils.py +31 -0
  8. Utils/MemoryCache.py +119 -0
  9. Utils/Patterns.py +24 -0
  10. Utils/Pipeline.py +137 -0
  11. Utils/PortForward.py +97 -0
  12. Utils/ProcessStats.py +103 -0
  13. Utils/PythonVersion.py +17 -0
  14. Utils/Signals.py +36 -0
  15. Utils/TemporaryEnvironment.py +27 -0
  16. Utils/Throttled.py +227 -0
  17. Utils/Timers.py +130 -0
  18. Utils/Timestamps.py +86 -0
  19. Utils/TokenManager.py +143 -0
  20. Utils/Tracing.py +60 -0
  21. Utils/TwPrint.py +98 -0
  22. Utils/Utilities.py +308 -0
  23. Utils/__init__.py +11 -0
  24. WMCore/ACDC/Collection.py +57 -0
  25. WMCore/ACDC/CollectionTypes.py +12 -0
  26. WMCore/ACDC/CouchCollection.py +67 -0
  27. WMCore/ACDC/CouchFileset.py +238 -0
  28. WMCore/ACDC/CouchService.py +73 -0
  29. WMCore/ACDC/DataCollectionService.py +485 -0
  30. WMCore/ACDC/Fileset.py +94 -0
  31. WMCore/ACDC/__init__.py +11 -0
  32. WMCore/Algorithms/Alarm.py +39 -0
  33. WMCore/Algorithms/MathAlgos.py +274 -0
  34. WMCore/Algorithms/MiscAlgos.py +67 -0
  35. WMCore/Algorithms/ParseXMLFile.py +115 -0
  36. WMCore/Algorithms/Permissions.py +27 -0
  37. WMCore/Algorithms/Singleton.py +58 -0
  38. WMCore/Algorithms/SubprocessAlgos.py +129 -0
  39. WMCore/Algorithms/__init__.py +7 -0
  40. WMCore/Cache/GenericDataCache.py +98 -0
  41. WMCore/Cache/WMConfigCache.py +572 -0
  42. WMCore/Cache/__init__.py +0 -0
  43. WMCore/Configuration.py +651 -0
  44. WMCore/DAOFactory.py +47 -0
  45. WMCore/DataStructs/File.py +177 -0
  46. WMCore/DataStructs/Fileset.py +140 -0
  47. WMCore/DataStructs/Job.py +182 -0
  48. WMCore/DataStructs/JobGroup.py +142 -0
  49. WMCore/DataStructs/JobPackage.py +49 -0
  50. WMCore/DataStructs/LumiList.py +734 -0
  51. WMCore/DataStructs/Mask.py +219 -0
  52. WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py +197 -0
  53. WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py +92 -0
  54. WMCore/DataStructs/MathStructs/SummaryHistogram.py +117 -0
  55. WMCore/DataStructs/MathStructs/__init__.py +0 -0
  56. WMCore/DataStructs/Pickleable.py +24 -0
  57. WMCore/DataStructs/Run.py +256 -0
  58. WMCore/DataStructs/Subscription.py +175 -0
  59. WMCore/DataStructs/WMObject.py +47 -0
  60. WMCore/DataStructs/WorkUnit.py +112 -0
  61. WMCore/DataStructs/Workflow.py +60 -0
  62. WMCore/DataStructs/__init__.py +8 -0
  63. WMCore/Database/CMSCouch.py +1349 -0
  64. WMCore/Database/ConfigDBMap.py +29 -0
  65. WMCore/Database/CouchUtils.py +118 -0
  66. WMCore/Database/DBCore.py +198 -0
  67. WMCore/Database/DBCreator.py +113 -0
  68. WMCore/Database/DBExceptionHandler.py +57 -0
  69. WMCore/Database/DBFactory.py +110 -0
  70. WMCore/Database/DBFormatter.py +177 -0
  71. WMCore/Database/Dialects.py +13 -0
  72. WMCore/Database/ExecuteDAO.py +327 -0
  73. WMCore/Database/MongoDB.py +241 -0
  74. WMCore/Database/MySQL/Destroy.py +42 -0
  75. WMCore/Database/MySQL/ListUserContent.py +20 -0
  76. WMCore/Database/MySQL/__init__.py +9 -0
  77. WMCore/Database/MySQLCore.py +132 -0
  78. WMCore/Database/Oracle/Destroy.py +56 -0
  79. WMCore/Database/Oracle/ListUserContent.py +19 -0
  80. WMCore/Database/Oracle/__init__.py +9 -0
  81. WMCore/Database/ResultSet.py +44 -0
  82. WMCore/Database/Transaction.py +91 -0
  83. WMCore/Database/__init__.py +9 -0
  84. WMCore/Database/ipy_profile_couch.py +438 -0
  85. WMCore/GlobalWorkQueue/CherryPyThreads/CleanUpTask.py +29 -0
  86. WMCore/GlobalWorkQueue/CherryPyThreads/HeartbeatMonitor.py +105 -0
  87. WMCore/GlobalWorkQueue/CherryPyThreads/LocationUpdateTask.py +28 -0
  88. WMCore/GlobalWorkQueue/CherryPyThreads/ReqMgrInteractionTask.py +35 -0
  89. WMCore/GlobalWorkQueue/CherryPyThreads/__init__.py +0 -0
  90. WMCore/GlobalWorkQueue/__init__.py +0 -0
  91. WMCore/GroupUser/CouchObject.py +127 -0
  92. WMCore/GroupUser/Decorators.py +51 -0
  93. WMCore/GroupUser/Group.py +33 -0
  94. WMCore/GroupUser/Interface.py +73 -0
  95. WMCore/GroupUser/User.py +96 -0
  96. WMCore/GroupUser/__init__.py +11 -0
  97. WMCore/Lexicon.py +836 -0
  98. WMCore/REST/Auth.py +202 -0
  99. WMCore/REST/CherryPyPeriodicTask.py +166 -0
  100. WMCore/REST/Error.py +333 -0
  101. WMCore/REST/Format.py +642 -0
  102. WMCore/REST/HeartbeatMonitorBase.py +90 -0
  103. WMCore/REST/Main.py +623 -0
  104. WMCore/REST/Server.py +2435 -0
  105. WMCore/REST/Services.py +24 -0
  106. WMCore/REST/Test.py +120 -0
  107. WMCore/REST/Tools.py +38 -0
  108. WMCore/REST/Validation.py +250 -0
  109. WMCore/REST/__init__.py +1 -0
  110. WMCore/ReqMgr/DataStructs/RequestStatus.py +209 -0
  111. WMCore/ReqMgr/DataStructs/RequestType.py +13 -0
  112. WMCore/ReqMgr/DataStructs/__init__.py +0 -0
  113. WMCore/ReqMgr/__init__.py +1 -0
  114. WMCore/Services/AlertManager/AlertManagerAPI.py +111 -0
  115. WMCore/Services/AlertManager/__init__.py +0 -0
  116. WMCore/Services/CRIC/CRIC.py +238 -0
  117. WMCore/Services/CRIC/__init__.py +0 -0
  118. WMCore/Services/DBS/DBS3Reader.py +1044 -0
  119. WMCore/Services/DBS/DBSConcurrency.py +44 -0
  120. WMCore/Services/DBS/DBSErrors.py +113 -0
  121. WMCore/Services/DBS/DBSReader.py +23 -0
  122. WMCore/Services/DBS/DBSUtils.py +139 -0
  123. WMCore/Services/DBS/DBSWriterObjects.py +381 -0
  124. WMCore/Services/DBS/ProdException.py +133 -0
  125. WMCore/Services/DBS/__init__.py +8 -0
  126. WMCore/Services/FWJRDB/FWJRDBAPI.py +118 -0
  127. WMCore/Services/FWJRDB/__init__.py +0 -0
  128. WMCore/Services/HTTPS/HTTPSAuthHandler.py +66 -0
  129. WMCore/Services/HTTPS/__init__.py +0 -0
  130. WMCore/Services/LogDB/LogDB.py +201 -0
  131. WMCore/Services/LogDB/LogDBBackend.py +191 -0
  132. WMCore/Services/LogDB/LogDBExceptions.py +11 -0
  133. WMCore/Services/LogDB/LogDBReport.py +85 -0
  134. WMCore/Services/LogDB/__init__.py +0 -0
  135. WMCore/Services/MSPileup/__init__.py +0 -0
  136. WMCore/Services/MSUtils/MSUtils.py +54 -0
  137. WMCore/Services/MSUtils/__init__.py +0 -0
  138. WMCore/Services/McM/McM.py +173 -0
  139. WMCore/Services/McM/__init__.py +8 -0
  140. WMCore/Services/MonIT/Grafana.py +133 -0
  141. WMCore/Services/MonIT/__init__.py +0 -0
  142. WMCore/Services/PyCondor/PyCondorAPI.py +154 -0
  143. WMCore/Services/PyCondor/PyCondorUtils.py +105 -0
  144. WMCore/Services/PyCondor/__init__.py +0 -0
  145. WMCore/Services/ReqMgr/ReqMgr.py +261 -0
  146. WMCore/Services/ReqMgr/__init__.py +0 -0
  147. WMCore/Services/ReqMgrAux/ReqMgrAux.py +419 -0
  148. WMCore/Services/ReqMgrAux/__init__.py +0 -0
  149. WMCore/Services/RequestDB/RequestDBReader.py +267 -0
  150. WMCore/Services/RequestDB/RequestDBWriter.py +39 -0
  151. WMCore/Services/RequestDB/__init__.py +0 -0
  152. WMCore/Services/Requests.py +624 -0
  153. WMCore/Services/Rucio/Rucio.py +1287 -0
  154. WMCore/Services/Rucio/RucioUtils.py +74 -0
  155. WMCore/Services/Rucio/__init__.py +0 -0
  156. WMCore/Services/RucioConMon/RucioConMon.py +128 -0
  157. WMCore/Services/RucioConMon/__init__.py +0 -0
  158. WMCore/Services/Service.py +400 -0
  159. WMCore/Services/StompAMQ/__init__.py +0 -0
  160. WMCore/Services/TagCollector/TagCollector.py +155 -0
  161. WMCore/Services/TagCollector/XMLUtils.py +98 -0
  162. WMCore/Services/TagCollector/__init__.py +0 -0
  163. WMCore/Services/UUIDLib.py +13 -0
  164. WMCore/Services/UserFileCache/UserFileCache.py +160 -0
  165. WMCore/Services/UserFileCache/__init__.py +8 -0
  166. WMCore/Services/WMAgent/WMAgent.py +63 -0
  167. WMCore/Services/WMAgent/__init__.py +0 -0
  168. WMCore/Services/WMArchive/CMSSWMetrics.py +526 -0
  169. WMCore/Services/WMArchive/DataMap.py +463 -0
  170. WMCore/Services/WMArchive/WMArchive.py +33 -0
  171. WMCore/Services/WMArchive/__init__.py +0 -0
  172. WMCore/Services/WMBS/WMBS.py +97 -0
  173. WMCore/Services/WMBS/__init__.py +0 -0
  174. WMCore/Services/WMStats/DataStruct/RequestInfoCollection.py +300 -0
  175. WMCore/Services/WMStats/DataStruct/__init__.py +0 -0
  176. WMCore/Services/WMStats/WMStatsPycurl.py +145 -0
  177. WMCore/Services/WMStats/WMStatsReader.py +445 -0
  178. WMCore/Services/WMStats/WMStatsWriter.py +273 -0
  179. WMCore/Services/WMStats/__init__.py +0 -0
  180. WMCore/Services/WMStatsServer/WMStatsServer.py +134 -0
  181. WMCore/Services/WMStatsServer/__init__.py +0 -0
  182. WMCore/Services/WorkQueue/WorkQueue.py +492 -0
  183. WMCore/Services/WorkQueue/__init__.py +0 -0
  184. WMCore/Services/__init__.py +8 -0
  185. WMCore/Services/pycurl_manager.py +574 -0
  186. WMCore/WMBase.py +50 -0
  187. WMCore/WMConnectionBase.py +164 -0
  188. WMCore/WMException.py +183 -0
  189. WMCore/WMExceptions.py +269 -0
  190. WMCore/WMFactory.py +76 -0
  191. WMCore/WMInit.py +228 -0
  192. WMCore/WMLogging.py +108 -0
  193. WMCore/WMSpec/ConfigSectionTree.py +442 -0
  194. WMCore/WMSpec/Persistency.py +135 -0
  195. WMCore/WMSpec/Steps/BuildMaster.py +87 -0
  196. WMCore/WMSpec/Steps/BuildTools.py +201 -0
  197. WMCore/WMSpec/Steps/Builder.py +97 -0
  198. WMCore/WMSpec/Steps/Diagnostic.py +89 -0
  199. WMCore/WMSpec/Steps/Emulator.py +62 -0
  200. WMCore/WMSpec/Steps/ExecuteMaster.py +208 -0
  201. WMCore/WMSpec/Steps/Executor.py +210 -0
  202. WMCore/WMSpec/Steps/StepFactory.py +213 -0
  203. WMCore/WMSpec/Steps/TaskEmulator.py +75 -0
  204. WMCore/WMSpec/Steps/Template.py +204 -0
  205. WMCore/WMSpec/Steps/Templates/AlcaHarvest.py +76 -0
  206. WMCore/WMSpec/Steps/Templates/CMSSW.py +613 -0
  207. WMCore/WMSpec/Steps/Templates/DQMUpload.py +59 -0
  208. WMCore/WMSpec/Steps/Templates/DeleteFiles.py +70 -0
  209. WMCore/WMSpec/Steps/Templates/LogArchive.py +84 -0
  210. WMCore/WMSpec/Steps/Templates/LogCollect.py +105 -0
  211. WMCore/WMSpec/Steps/Templates/StageOut.py +105 -0
  212. WMCore/WMSpec/Steps/Templates/__init__.py +10 -0
  213. WMCore/WMSpec/Steps/WMExecutionFailure.py +21 -0
  214. WMCore/WMSpec/Steps/__init__.py +8 -0
  215. WMCore/WMSpec/Utilities.py +63 -0
  216. WMCore/WMSpec/WMSpecErrors.py +12 -0
  217. WMCore/WMSpec/WMStep.py +347 -0
  218. WMCore/WMSpec/WMTask.py +1980 -0
  219. WMCore/WMSpec/WMWorkload.py +2288 -0
  220. WMCore/WMSpec/WMWorkloadTools.py +370 -0
  221. WMCore/WMSpec/__init__.py +9 -0
  222. WMCore/WorkQueue/DataLocationMapper.py +273 -0
  223. WMCore/WorkQueue/DataStructs/ACDCBlock.py +47 -0
  224. WMCore/WorkQueue/DataStructs/Block.py +48 -0
  225. WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py +148 -0
  226. WMCore/WorkQueue/DataStructs/WorkQueueElement.py +274 -0
  227. WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py +152 -0
  228. WMCore/WorkQueue/DataStructs/WorkQueueElementsSummary.py +185 -0
  229. WMCore/WorkQueue/DataStructs/__init__.py +0 -0
  230. WMCore/WorkQueue/Policy/End/EndPolicyInterface.py +44 -0
  231. WMCore/WorkQueue/Policy/End/SingleShot.py +22 -0
  232. WMCore/WorkQueue/Policy/End/__init__.py +32 -0
  233. WMCore/WorkQueue/Policy/PolicyInterface.py +17 -0
  234. WMCore/WorkQueue/Policy/Start/Block.py +258 -0
  235. WMCore/WorkQueue/Policy/Start/Dataset.py +180 -0
  236. WMCore/WorkQueue/Policy/Start/MonteCarlo.py +131 -0
  237. WMCore/WorkQueue/Policy/Start/ResubmitBlock.py +171 -0
  238. WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +316 -0
  239. WMCore/WorkQueue/Policy/Start/__init__.py +34 -0
  240. WMCore/WorkQueue/Policy/__init__.py +57 -0
  241. WMCore/WorkQueue/WMBSHelper.py +772 -0
  242. WMCore/WorkQueue/WorkQueue.py +1237 -0
  243. WMCore/WorkQueue/WorkQueueBackend.py +750 -0
  244. WMCore/WorkQueue/WorkQueueBase.py +39 -0
  245. WMCore/WorkQueue/WorkQueueExceptions.py +44 -0
  246. WMCore/WorkQueue/WorkQueueReqMgrInterface.py +278 -0
  247. WMCore/WorkQueue/WorkQueueUtils.py +130 -0
  248. WMCore/WorkQueue/__init__.py +13 -0
  249. WMCore/Wrappers/JsonWrapper/JSONThunker.py +342 -0
  250. WMCore/Wrappers/JsonWrapper/__init__.py +7 -0
  251. WMCore/Wrappers/__init__.py +6 -0
  252. WMCore/__init__.py +10 -0
  253. wmglobalqueue-2.3.10.data/data/bin/wmc-dist-patch +15 -0
  254. wmglobalqueue-2.3.10.data/data/bin/wmc-dist-unpatch +8 -0
  255. wmglobalqueue-2.3.10.data/data/bin/wmc-httpd +3 -0
  256. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/.couchapprc +1 -0
  257. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/README.md +40 -0
  258. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/index.html +264 -0
  259. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/ElementInfoByWorkflow.js +96 -0
  260. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/StuckElementInfo.js +57 -0
  261. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/WorkloadInfoTable.js +80 -0
  262. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/dataTable.js +70 -0
  263. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/namespace.js +23 -0
  264. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/style/main.css +75 -0
  265. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/couchapp.json +4 -0
  266. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/childQueueFilter.js +13 -0
  267. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/filterDeletedDocs.js +3 -0
  268. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/queueFilter.js +11 -0
  269. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/language +1 -0
  270. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/mustache.js +333 -0
  271. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/validate.js +27 -0
  272. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/workqueue_utils.js +61 -0
  273. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/elementsDetail.js +28 -0
  274. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/filter.js +86 -0
  275. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/stuckElements.js +38 -0
  276. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/workRestrictions.js +153 -0
  277. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/workflowSummary.js +28 -0
  278. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/rewrites.json +73 -0
  279. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/shows/redirect.js +23 -0
  280. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/shows/status.js +40 -0
  281. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/ElementSummaryByWorkflow.html +27 -0
  282. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/StuckElementSummary.html +26 -0
  283. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/TaskStatus.html +23 -0
  284. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/WorkflowSummary.html +27 -0
  285. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/workqueue-common-lib.html +2 -0
  286. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib-remote.html +16 -0
  287. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib.html +18 -0
  288. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/updates/in-place.js +50 -0
  289. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/validate_doc_update.js +8 -0
  290. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.couch.app.js +235 -0
  291. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.pathbinder.js +173 -0
  292. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeData/map.js +8 -0
  293. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeData/reduce.js +2 -0
  294. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeParentData/map.js +8 -0
  295. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeParentData/reduce.js +2 -0
  296. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activePileupData/map.js +8 -0
  297. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activePileupData/reduce.js +2 -0
  298. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/analyticsData/map.js +11 -0
  299. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/analyticsData/reduce.js +1 -0
  300. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/availableByPriority/map.js +6 -0
  301. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/conflicts/map.js +5 -0
  302. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elements/map.js +5 -0
  303. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByData/map.js +8 -0
  304. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByParent/map.js +8 -0
  305. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByParentData/map.js +8 -0
  306. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByPileupData/map.js +8 -0
  307. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByStatus/map.js +8 -0
  308. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsBySubscription/map.js +6 -0
  309. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/map.js +8 -0
  310. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/reduce.js +3 -0
  311. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsDetailByWorkflowAndStatus/map.js +26 -0
  312. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/map.js +10 -0
  313. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/reduce.js +1 -0
  314. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/map.js +6 -0
  315. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/reduce.js +1 -0
  316. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/map.js +6 -0
  317. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/reduce.js +1 -0
  318. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/map.js +6 -0
  319. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/reduce.js +1 -0
  320. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByRequest/map.js +6 -0
  321. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByRequest/reduce.js +1 -0
  322. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatus/map.js +6 -0
  323. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatus/reduce.js +1 -0
  324. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/map.js +6 -0
  325. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/reduce.js +1 -0
  326. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/openRequests/map.js +6 -0
  327. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/recent-items/map.js +5 -0
  328. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/map.js +6 -0
  329. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/reduce.js +1 -0
  330. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/specsByWorkflow/map.js +5 -0
  331. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/stuckElements/map.js +38 -0
  332. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/map.js +12 -0
  333. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/reduce.js +3 -0
  334. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrl/map.js +6 -0
  335. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrl/reduce.js +2 -0
  336. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/map.js +6 -0
  337. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/reduce.js +2 -0
  338. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/workflowSummary/map.js +9 -0
  339. wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/workflowSummary/reduce.js +10 -0
  340. wmglobalqueue-2.3.10.dist-info/LICENSE +202 -0
  341. wmglobalqueue-2.3.10.dist-info/METADATA +24 -0
  342. wmglobalqueue-2.3.10.dist-info/NOTICE +16 -0
  343. wmglobalqueue-2.3.10.dist-info/RECORD +345 -0
  344. wmglobalqueue-2.3.10.dist-info/WHEEL +5 -0
  345. wmglobalqueue-2.3.10.dist-info/top_level.txt +2 -0
WMCore/REST/Main.py ADDED
@@ -0,0 +1,623 @@
1
+ """ wmc-httpd [-q] { -r | --restart } [-l LOG-FILE] CONFIG-FILE
2
+ wmc-httpd [-q] { -v | --verify } [-l LOG-FILE] CONFIG-FILE
3
+ wmc-httpd [-q] { -s | --status } CONFIG-FILE
4
+ wmc-httpd [-q] { -k | --kill } CONFIG-FILE
5
+
6
+ Manages a web server application. Loads configuration and all views, starting
7
+ up an appropriately configured CherryPy instance. Views are loaded dynamically
8
+ and can be turned on/off via configuration file."""
9
+
10
+ from __future__ import print_function
11
+ from builtins import object
12
+ from future.utils import viewitems
13
+ from future import standard_library
14
+ standard_library.install_aliases()
15
+
16
+ import errno
17
+ import logging
18
+ import os
19
+ import os.path
20
+ import re
21
+ import signal
22
+ import socket
23
+ import sys
24
+ import _thread
25
+ import time
26
+ import traceback
27
+ from argparse import ArgumentParser
28
+ from io import BytesIO, StringIO
29
+ from glob import glob
30
+ from subprocess import Popen, PIPE
31
+ from pprint import pformat
32
+
33
+ import cherrypy
34
+ from cherrypy import Application
35
+ from cherrypy._cplogging import LogManager
36
+ from cherrypy.lib import profiler
37
+
38
+ ### Tools is needed for CRABServer startup: it sets up the tools attributes
39
+ import WMCore.REST.Tools
40
+ from WMCore.Configuration import ConfigSection, loadConfigurationFile
41
+ from Utils.Utilities import lowerCmsHeaders
42
+ from Utils.PythonVersion import PY2
43
+
44
+ #: Terminal controls to switch to "OK" status message colour.
45
+ COLOR_OK = "\033[0;32m"
46
+
47
+ #: Terminal controls to switch to "warning" status message colour.
48
+ COLOR_WARN = "\033[0;31m"
49
+
50
+ #: Terminal controls to restore normal message colour.
51
+ COLOR_NORMAL = "\033[0;39m"
52
+
53
+
54
+ def sig_terminate(signum=None, frame=None):
55
+ """Termination signal handler.
56
+
57
+ CherryPy termination signal handling is broken, the handler does not
58
+ take the right number of arguments. This is our own fixed handler
59
+ to terminate the web server gracefully; in theory it could be
60
+ removed when CherryPy is fixed, but we attach other signals here
61
+ and print a logging entry."""
62
+ cherrypy.log("INFO: exiting server from termination signal %d" % signum, severity=logging.INFO)
63
+ cherrypy.engine.exit()
64
+
65
+
66
+ def sig_reload(signum=None, frame=None):
67
+ """SIGHUP handler to restart the server.
68
+
69
+ This just adds some logging compared to the CherryPy signal handler."""
70
+ cherrypy.log("INFO: restarting server from hang-up signal %d" % signum, severity=logging.INFO)
71
+ cherrypy.engine.restart()
72
+
73
+
74
+ def sig_graceful(signum=None, frame=None):
75
+ """SIGUSR1 handler to restart the server gracefully.
76
+
77
+ This just adds some logging compared to the CherryPy signal handler."""
78
+ cherrypy.log("INFO: restarting server gracefully from signal %d" % signum, severity=logging.INFO)
79
+ cherrypy.engine.graceful()
80
+
81
+
82
+ class ProfiledApp(Application):
83
+ """Wrapper CherryPy Application object which generates aggregated
84
+ profiles for the component on each call. Note that there needs to
85
+ be an instance of this for each mount point to be profiled."""
86
+
87
+ def __init__(self, app, path):
88
+ Application.__init__(self, app.root, app.script_name, app.config)
89
+ self.profiler = profiler.ProfileAggregator(path)
90
+
91
+ def __call__(self, env, handler):
92
+ def gather(): return Application.__call__(self, env, handler)
93
+
94
+ return self.profiler.run(gather)
95
+
96
+
97
+ class Logger(LogManager):
98
+ """Custom logger to record information in format we prefer."""
99
+
100
+ def __init__(self, *args, **kwargs):
101
+ self.host = socket.gethostname()
102
+ LogManager.__init__(self, *args, **kwargs)
103
+
104
+ def access(self):
105
+ """Record one client access."""
106
+ request = cherrypy.request
107
+ remote = request.remote
108
+ response = cherrypy.response
109
+ inheaders = lowerCmsHeaders(request.headers)
110
+ outheaders = response.headers
111
+ wfile = request.wsgi_environ.get('cherrypy.wfile', None)
112
+ nout = (wfile and wfile.bytes_written) or outheaders.get('Content-Length', 0)
113
+ if hasattr(request, 'start_time'):
114
+ delta_time = (time.time() - request.start_time) * 1e6
115
+ else:
116
+ delta_time = 0
117
+ msg = ('%(t)s %(H)s %(h)s "%(r)s" %(s)s'
118
+ ' [data: %(i)s in %(b)s out %(T).0f us ]'
119
+ ' [auth: %(AS)s "%(AU)s" "%(AC)s" ]'
120
+ ' [ref: "%(f)s" "%(a)s" ]') % \
121
+ {'t': self.time(),
122
+ 'H': self.host,
123
+ 'h': remote.name or remote.ip,
124
+ 'r': request.request_line,
125
+ 's': response.status,
126
+ # request.rfile.rfile.bytes_read is a custom CMS web
127
+ # cherrypy patch not always available, hence the test
128
+ 'i': (getattr(request.rfile, 'rfile', None)
129
+ and getattr(request.rfile.rfile, "bytes_read", None)
130
+ and request.rfile.rfile.bytes_read) or "-",
131
+ 'b': nout or "-",
132
+ 'T': delta_time,
133
+ 'AS': inheaders.get("cms-auth-status", "-"),
134
+ 'AU': inheaders.get("cms-auth-cert", inheaders.get("cms-auth-host", "")),
135
+ 'AC': getattr(request.cookie.get("cms-auth", None), "value", ""),
136
+ 'f': inheaders.get("Referer", ""),
137
+ 'a': inheaders.get("User-Agent", "")}
138
+ self.access_log.log(logging.INFO, msg)
139
+ self.access_log.propagate = False # to avoid duplicate records on the log
140
+ self.error_log.propagate = False # to avoid duplicate records on the log
141
+
142
+
143
+ class RESTMain(object):
144
+ """Base class for the core CherryPy main application object.
145
+
146
+ The :class:`~.RESTMain` implements basic functionality of a CherryPy-based
147
+ server. Most users will want the fully functional :class:`~.RESTDaemon`
148
+ instead; but in some cases such as tests and other single-shot jobs which
149
+ don't require a daemon process this class is useful in its own right.
150
+
151
+ The class implements the methods required to configure, but not run, a
152
+ CherryPy server set up with an application configuration.
153
+
154
+ The main application object takes the server configuration and state
155
+ directory as parametres. It provides methods to create full CherryPy
156
+ serer and configure the application based on configuration description."""
157
+
158
+ def __init__(self, config, statedir):
159
+ """Prepare the server.
160
+
161
+ :arg config: server configuration
162
+ :arg str statedir: server state directory."""
163
+ self.config = config
164
+ self.appname = config.main.application.lower()
165
+ self.appconfig = config.section_(self.appname)
166
+ self.srvconfig = config.section_("main")
167
+ self.statedir = statedir
168
+ self.hostname = socket.getfqdn().lower()
169
+ self.silent = False
170
+ self.extensions = {}
171
+ self.views = {}
172
+
173
+ def validate_config(self):
174
+ """Check the server configuration has the required sections."""
175
+ for key in ('admin', 'description', 'title'):
176
+ if not hasattr(self.appconfig, key):
177
+ raise RuntimeError("'%s' required in application configuration" % key)
178
+
179
+ def setup_server(self):
180
+ """Configure CherryPy server from application configuration.
181
+
182
+ Traverses the server configuration portion and applies parameters
183
+ known to be for CherryPy to the CherryPy server configuration.
184
+ These are: engine, hooks, log, request, respose, server, tools,
185
+ wsgi, checker.
186
+
187
+ Also applies pseudo-parameters ``thread_stack_size`` (default: 128kB)
188
+ and ``sys_check_interval`` (default: 10000). The former sets the
189
+ default stack size to desired value, to avoid excessively large
190
+ thread stacks -- typical operating system default is 8 MB, which
191
+ adds up rather a lot for lots of server threads. The latter sets
192
+ python's ``sys.setcheckinterval``; the default is to increase this
193
+ to avoid unnecessarily frequent checks for python's GIL, global
194
+ interpreter lock. In general we want each thread to complete as
195
+ quickly as possible without making unnecessary checks."""
196
+ cpconfig = cherrypy.config
197
+
198
+ # Determine server local base.
199
+ port = getattr(self.srvconfig, 'port', 8080)
200
+ local_base = getattr(self.srvconfig, 'local_base', socket.gethostname())
201
+ if local_base.find(':') == -1:
202
+ local_base = '%s:%d' % (local_base, port)
203
+
204
+ # Set default server configuration.
205
+ cherrypy.log = Logger()
206
+
207
+ cpconfig.update({'server.max_request_body_size': 0})
208
+ cpconfig.update({'server.environment': 'production'})
209
+ cpconfig.update({'server.socket_host': '0.0.0.0'})
210
+ cpconfig.update({'server.socket_port': port})
211
+ cpconfig.update({'server.socket_queue_size': 100})
212
+ cpconfig.update({'server.thread_pool': 100})
213
+ cpconfig.update({'tools.proxy.on': True})
214
+ cpconfig.update({'tools.proxy.base': local_base})
215
+ cpconfig.update({'tools.time.on': True})
216
+ cpconfig.update({'engine.autoreload.on': False})
217
+ cpconfig.update({'request.show_tracebacks': False})
218
+ cpconfig.update({'request.methods_with_bodies': ("POST", "PUT", "DELETE")})
219
+ _thread.stack_size(getattr(self.srvconfig, 'thread_stack_size', 128 * 1024))
220
+ # read sys_check_interval from server config which sets in instructions
221
+ # as we previously used sys.setcheckinterval
222
+ interval = getattr(self.srvconfig, 'sys_check_interval', 10000)
223
+ # set check interval in seconds for sys.setswitchinterval
224
+ sys.setswitchinterval(interval/1000)
225
+ self.silent = getattr(self.srvconfig, 'silent', False)
226
+
227
+ # Apply any override options from app config file.
228
+ for section in ('engine', 'hooks', 'log', 'request', 'response',
229
+ 'server', 'tools', 'wsgi', 'checker'):
230
+ if not hasattr(self.srvconfig, section):
231
+ continue
232
+ for opt, value in viewitems(getattr(self.srvconfig, section).dictionary_()):
233
+ if isinstance(value, ConfigSection):
234
+ for xopt, xvalue in viewitems(value.dictionary_()):
235
+ cpconfig.update({"%s.%s.%s" % (section, opt, xopt): xvalue})
236
+ elif isinstance(value, str) or isinstance(value, int):
237
+ cpconfig.update({"%s.%s" % (section, opt): value})
238
+ else:
239
+ raise RuntimeError("%s.%s should be string or int, got %s"
240
+ % (section, opt, type(value)))
241
+
242
+ # Apply security customisation.
243
+ if hasattr(self.srvconfig, 'authz_defaults'):
244
+ defsec = self.srvconfig.authz_defaults
245
+ cpconfig.update({'tools.cms_auth.on': True})
246
+ cpconfig.update({'tools.cms_auth.role': defsec['role']})
247
+ cpconfig.update({'tools.cms_auth.group': defsec['group']})
248
+ cpconfig.update({'tools.cms_auth.site': defsec['site']})
249
+
250
+ if hasattr(self.srvconfig, 'authz_policy'):
251
+ cpconfig.update({'tools.cms_auth.policy': self.srvconfig.authz_policy})
252
+ cherrypy.log("INFO: final CherryPy configuration: %s" % pformat(cpconfig))
253
+
254
+ def install_application(self):
255
+ """Install application and its components from the configuration."""
256
+ index = self.srvconfig.index
257
+
258
+ # First instantiate non-view extensions.
259
+ if getattr(self.config, 'extensions', None):
260
+ for ext in self.config.extensions:
261
+ name = ext._internal_name
262
+ if not self.silent:
263
+ cherrypy.log("INFO: instantiating extension %s" % name)
264
+ module_name, class_name = ext.object.rsplit(".", 1)
265
+ module = __import__(module_name, globals(), locals(), [class_name])
266
+ obj = getattr(module, class_name)(self, ext)
267
+ self.extensions[name] = obj
268
+
269
+ # Then instantiate views and mount them to cherrypy. If the view is
270
+ # designated as the index, create it as an application, profiled one
271
+ # if server profiling was requested. Otherwise just mount it as a
272
+ # normal server content object. Force tracebacks off for everything.
273
+ for view in self.config.views:
274
+ name = view._internal_name
275
+ path = "/%s" % self.appname + ((name != index and "/%s" % name) or "")
276
+ if not self.silent:
277
+ cherrypy.log("INFO: loading %s into %s" % (name, path))
278
+ module_name, class_name = view.object.rsplit(".", 1)
279
+ module = __import__(module_name, globals(), locals(), [class_name])
280
+ obj = getattr(module, class_name)(self, view, path)
281
+ app = Application(obj, path, {"/": {"request.show_tracebacks": False}})
282
+ if getattr(self.srvconfig, 'profile', False):
283
+ profdir = "%s/profile" % self.statedir
284
+ if not os.path.exists(profdir):
285
+ os.makedirs(profdir)
286
+ app = ProfiledApp(app, profdir)
287
+ cherrypy.tree.mount(app)
288
+ self.views[name] = obj
289
+
290
+
291
+ class RESTDaemon(RESTMain):
292
+ """Web server object.
293
+
294
+ The `RESTDaemon` represents the web server daemon. It provides all
295
+ services for starting, stopping and checking the status of the daemon,
296
+ as well as running the main loop.
297
+
298
+ The class implements all the methods required for proper unix
299
+ daemonisation, including maintaing a PID file for the process group
300
+ and correct progressively more aggressive signals sent to terminate
301
+ the daemon. Starting multiple daemons in same directory is refused.
302
+
303
+ The daemon takes the server configuration as a parametre. When the
304
+ server is started, it creates a CherryPy server and configuration
305
+ from the application config contents."""
306
+
307
+ def __init__(self, config, statedir):
308
+ """Initialise the daemon.
309
+
310
+ :arg config: server configuration
311
+ :arg str statedir: server state directory."""
312
+ RESTMain.__init__(self, config, statedir)
313
+ self.pidfile = "%s/%s.pid" % (self.statedir, self.appname)
314
+ self.logfile = ["rotatelogs", "%s/%s-%%Y%%m%%d.log" % (self.statedir, self.appname), "86400"]
315
+
316
+ def daemon_pid(self):
317
+ """Check if there is a daemon running, and if so return its pid.
318
+
319
+ Reads the pid file from the daemon work directory, if any. If a
320
+ non-empty pid file exists, checks if a process by that PGID exists.
321
+ If yes, reports the daemon as running, otherwise reports either a
322
+ stale daemon no longer running or no deamon running at all.
323
+
324
+ :returns: A tuple (running, pgid). The first value will be True if a
325
+ running daemon was found, in which case pid will be its PGID. The
326
+ first value will be false otherwise, and pgid will be either None
327
+ if no pid file was found, or an integer if there was a stale file."""
328
+ pid = None
329
+ try:
330
+ with open(self.pidfile) as fd:
331
+ pid = int(fd.readline())
332
+ os.killpg(pid, 0)
333
+ return (True, pid)
334
+ except:
335
+ return (False, pid)
336
+
337
+ def kill_daemon(self, silent=False):
338
+ """Check if the daemon is running, and if so kill it.
339
+
340
+ If there is no daemon running and no pid file, does nothing. If there
341
+ is a pid file but no such process running, removes the stale pid file.
342
+ Otherwise sends progressively more lethal signals at intervals to the
343
+ daemon process until it quits.
344
+
345
+ The signals are always sent to the entire process group, and signals
346
+ will keep on getting sent as long as at least one process from the
347
+ daemon process group is still alive. If for some reason the group
348
+ cannot be killed otherwise, sends SIGKILL to the group in the end.
349
+
350
+ The message about removing a stale pid file cannot be silenced. All
351
+ other messages are squelched if `silent` is True.
352
+
353
+ :arg bool silent: do not print any messages if True."""
354
+ running, pid = self.daemon_pid()
355
+ if not running:
356
+ if pid != None:
357
+ print("Removing stale pid file %s" % self.pidfile)
358
+ os.remove(self.pidfile)
359
+ else:
360
+ if not silent:
361
+ print("%s not running, not killing" % self.appname)
362
+ else:
363
+ if not silent:
364
+ sys.stdout.write("Killing %s pgid %d " % (self.appname, pid))
365
+ sys.stdout.flush()
366
+
367
+ dead = False
368
+ for sig, grace in ((signal.SIGINT, .5), (signal.SIGINT, 1),
369
+ (signal.SIGINT, 3), (signal.SIGINT, 5),
370
+ (signal.SIGKILL, 0)):
371
+ try:
372
+ if not silent:
373
+ sys.stdout.write(".")
374
+ sys.stdout.flush()
375
+ os.killpg(pid, sig)
376
+ time.sleep(grace)
377
+ os.killpg(pid, 0)
378
+ except OSError as e:
379
+ if e.errno == errno.ESRCH:
380
+ dead = True
381
+ break
382
+ raise
383
+
384
+ if not dead:
385
+ try:
386
+ os.killpg(pid, signal.SIGKILL)
387
+ except:
388
+ pass
389
+
390
+ if not silent:
391
+ sys.stdout.write("\n")
392
+
393
+ def start_daemon(self):
394
+ """Start the deamon."""
395
+ # This REST server can be run in the frontend i.e. interactively (e.g. to use pdb)
396
+ # by setting an env. var. via: export DONT_DAEMONIZE_REST=True
397
+ # if the variable is missing or set to any other value, code starts normally as a daemon
398
+ daemonize = os.getenv('DONT_DAEMONIZE_REST', 'False') != 'True'
399
+
400
+ os.chdir(self.statedir)
401
+
402
+ if daemonize:
403
+ # Redirect all output to the logging daemon.
404
+ devnull = open(os.devnull, "w")
405
+ if isinstance(self.logfile, list):
406
+ subproc = Popen(self.logfile, stdin=PIPE, stdout=devnull, stderr=devnull,
407
+ bufsize=0, close_fds=True, shell=False)
408
+ logger = subproc.stdin
409
+ elif isinstance(self.logfile, str):
410
+ # if a unix pipe is set as the logfile, it must be opened to append to the end of the file
411
+ # if file/pipe does not exist, create it
412
+ logger = open(self.logfile, "a")
413
+ else:
414
+ raise TypeError("'logfile' must be a string or array")
415
+ os.dup2(logger.fileno(), sys.stdout.fileno())
416
+ os.dup2(logger.fileno(), sys.stderr.fileno())
417
+ os.dup2(devnull.fileno(), sys.stdin.fileno())
418
+ logger.close()
419
+ devnull.close()
420
+
421
+ # First fork. Discard the parent.
422
+ pid = os.fork()
423
+ if pid > 0:
424
+ os._exit(0)
425
+
426
+ # Establish as a daemon, set process group / session id.
427
+ os.setsid()
428
+
429
+ # Second fork. The child does the work, discard the second parent.
430
+ pid = os.fork()
431
+ if pid > 0:
432
+ os._exit(0)
433
+
434
+ # Save process group id to pid file, then run real worker.
435
+ with open(self.pidfile, "w") as pidObj:
436
+ pidObj.write("%d\n" % os.getpgid(0))
437
+
438
+ # following code is executed both in daemon and not daemon mode
439
+ error = False
440
+ try:
441
+ self.run()
442
+ except Exception as e:
443
+ error = True
444
+ if PY2:
445
+ trace = BytesIO()
446
+ else:
447
+ trace = StringIO()
448
+ traceback.print_exc(file=trace)
449
+ cherrypy.log("ERROR: terminating due to error: %s" % trace.getvalue())
450
+
451
+ # Remove pid file once we are done.
452
+ try:
453
+ os.remove(self.pidfile)
454
+ except:
455
+ pass
456
+
457
+ # Exit
458
+ sys.exit((error and 1) or 0)
459
+
460
+ def run(self):
461
+ """Run the server daemon main loop."""
462
+ # Fork. The child always exits the loop and executes the code below
463
+ # to run the server proper. The parent monitors the child, and if
464
+ # it exits abnormally, restarts it, otherwise exits completely with
465
+ # the child's exit code.
466
+ daemonize = os.getenv('DONT_DAEMONIZE_REST', 'False') != 'True'
467
+ if daemonize:
468
+ cherrypy.log("WATCHDOG: starting server daemon (pid %d)" % os.getpid())
469
+ while True:
470
+ serverpid = os.fork()
471
+ if not serverpid: break
472
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
473
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
474
+ signal.signal(signal.SIGQUIT, signal.SIG_IGN)
475
+ (xpid, exitrc) = os.waitpid(serverpid, 0)
476
+ (exitcode, exitsigno, exitcore) = (exitrc >> 8, exitrc & 127, exitrc & 128)
477
+ retval = (exitsigno and ("signal %d" % exitsigno)) or str(exitcode)
478
+ retmsg = retval + ((exitcore and " (core dumped)") or "")
479
+ restart = (exitsigno > 0 and exitsigno not in (2, 3, 15))
480
+ cherrypy.log("WATCHDOG: server exited with exit code %s%s"
481
+ % (retmsg, (restart and "... restarting") or ""))
482
+
483
+ if not restart:
484
+ sys.exit((exitsigno and 1) or exitcode)
485
+
486
+ for pidfile in glob("%s/*/*pid" % self.statedir):
487
+ if os.path.exists(pidfile):
488
+ with open(pidfile) as fd:
489
+ pid = int(fd.readline())
490
+ os.remove(pidfile)
491
+ cherrypy.log("WATCHDOG: killing slave server %d" % pid)
492
+ try:
493
+ os.kill(pid, 9)
494
+ except:
495
+ pass
496
+
497
+ # Run. Override signal handlers after CherryPy has itself started and
498
+ # installed its own handlers. To achieve this we need to start the
499
+ # server in non-blocking mode, fiddle with, than ask server to block.
500
+ self.validate_config()
501
+ self.setup_server()
502
+ self.install_application()
503
+ cherrypy.log("INFO: starting server in %s" % self.statedir)
504
+ cherrypy.config.update({'log.screen': bool(getattr(self.srvconfig, "log_screen", True))})
505
+ cherrypy.engine.start()
506
+ signal.signal(signal.SIGHUP, sig_reload)
507
+ signal.signal(signal.SIGUSR1, sig_graceful)
508
+ signal.signal(signal.SIGTERM, sig_terminate)
509
+ signal.signal(signal.SIGQUIT, sig_terminate)
510
+ signal.signal(signal.SIGINT, sig_terminate)
511
+ cherrypy.engine.block()
512
+
513
+
514
+ def main():
515
+ # Re-exec if we don't have unbuffered i/o. This is essential to get server
516
+ # to output its logs synchronous to its operation, such that log output does
517
+ # not remain buffered in the python server. This is particularly important
518
+ # when infrequently accessed server redirects output to 'rotatelogs'.
519
+ if 'PYTHONUNBUFFERED' not in os.environ:
520
+ os.environ['PYTHONUNBUFFERED'] = "1"
521
+ if PY2:
522
+ os.execvp("python", ["python"] + sys.argv)
523
+ else:
524
+ os.execvp("python3", ["python3"] + sys.argv)
525
+
526
+ opt = ArgumentParser(usage=__doc__)
527
+ opt.add_argument("-q", "--quiet", action="store_true", dest="quiet", default=False,
528
+ help="be quiet, don't print unnecessary output")
529
+ opt.add_argument("-v", "--verify", action="store_true", dest="verify", default=False,
530
+ help="verify daemon is running, restart if not")
531
+ opt.add_argument("-s", "--status", action="store_true", dest="status", default=False,
532
+ help="check if the server monitor daemon is running")
533
+ opt.add_argument("-k", "--kill", action="store_true", dest="kill", default=False,
534
+ help="kill any existing already running daemon")
535
+ opt.add_argument("-r", "--restart", action="store_true", dest="restart", default=False,
536
+ help="restart, kill any existing running daemon first")
537
+ opt.add_argument("-d", "--dir", dest="statedir", metavar="DIR", default=os.getcwd(),
538
+ help="server state directory (default: current working directory)")
539
+ opt.add_argument("-l", "--log", dest="logfile", metavar="DEST", default=None,
540
+ help="log to DEST, via pipe if DEST begins with '|', otherwise a file")
541
+ opts, args = opt.parse_known_args()
542
+
543
+ if len(args) != 1:
544
+ print("%s: exactly one configuration file required" % sys.argv[0], file=sys.stderr)
545
+ sys.exit(1)
546
+
547
+ if not os.path.isfile(args[0]) or not os.access(args[0], os.R_OK):
548
+ print("%s: %s: invalid configuration file" % (sys.argv[0], args[0]), file=sys.stderr)
549
+ sys.exit(1)
550
+
551
+ if not opts.statedir or \
552
+ not os.path.isdir(opts.statedir) or \
553
+ not os.access(opts.statedir, os.W_OK):
554
+ print("%s: %s: invalid state directory" % (sys.argv[0], opts.statedir), file=sys.stderr)
555
+ sys.exit(1)
556
+
557
+ # Create server object.
558
+ cfg = loadConfigurationFile(args[0])
559
+ app = cfg.main.application.lower()
560
+ server = RESTDaemon(cfg, opts.statedir)
561
+
562
+ # Now actually execute the task.
563
+ if opts.status:
564
+ # Show status of running daemon, including exit code matching the
565
+ # daemon status: 0 = running, 1 = not running, 2 = not running but
566
+ # there is a stale pid file. If silent don't print out anything
567
+ # but still return the right exit code.
568
+ running, pid = server.daemon_pid()
569
+ if running:
570
+ if not opts.quiet:
571
+ print("%s is %sRUNNING%s, PID %d" \
572
+ % (app, COLOR_OK, COLOR_NORMAL, pid))
573
+ sys.exit(0)
574
+ elif pid != None:
575
+ if not opts.quiet:
576
+ print("%s is %sNOT RUNNING%s, stale PID %d" \
577
+ % (app, COLOR_WARN, COLOR_NORMAL, pid))
578
+ sys.exit(2)
579
+ else:
580
+ if not opts.quiet:
581
+ print("%s is %sNOT RUNNING%s" \
582
+ % (app, COLOR_WARN, COLOR_NORMAL))
583
+ sys.exit(1)
584
+
585
+ elif opts.kill:
586
+ # Stop any previously running daemon. If quiet squelch messages,
587
+ # except removal of stale pid file cannot be silenced.
588
+ server.kill_daemon(silent=opts.quiet)
589
+
590
+ else:
591
+ # We are handling a server start, in one of many possible ways:
592
+ # normal start, restart (= kill any previous daemon), or verify
593
+ # (= if daemon is running leave it alone, otherwise start).
594
+
595
+ # Convert 'verify' to 'restart' if the server isn't running.
596
+ if opts.verify:
597
+ opts.restart = True
598
+ if server.daemon_pid()[0]:
599
+ sys.exit(0)
600
+
601
+ # If restarting, kill any previous daemon, otherwise complain if
602
+ # there is a daemon already running here. Starting overlapping
603
+ # daemons is not supported because pid file would be overwritten
604
+ # and we'd lose track of the previous daemon.
605
+ if opts.restart:
606
+ server.kill_daemon(silent=opts.quiet)
607
+ else:
608
+ running, pid = server.daemon_pid()
609
+ if running:
610
+ print("Refusing to start over an already running daemon, pid %d" % pid, file=sys.stderr)
611
+ sys.exit(1)
612
+
613
+ # If we are (re)starting and were given a log file option, convert
614
+ # the logfile option to a list if it looks like a pipe request, i.e.
615
+ # starts with "|", such as "|rotatelogs foo/bar-%Y%m%d.log".
616
+ if opts.logfile:
617
+ if opts.logfile.startswith("|"):
618
+ server.logfile = re.split(r"\s+", opts.logfile[1:])
619
+ else:
620
+ server.logfile = opts.logfile
621
+
622
+ # Actually start the daemon now.
623
+ server.start_daemon()