wmglobalqueue 2.3.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wmglobalqueue might be problematic. Click here for more details.
- Utils/CPMetrics.py +270 -0
- Utils/CertTools.py +62 -0
- Utils/EmailAlert.py +50 -0
- Utils/ExtendedUnitTestCase.py +62 -0
- Utils/FileTools.py +182 -0
- Utils/IteratorTools.py +80 -0
- Utils/MathUtils.py +31 -0
- Utils/MemoryCache.py +119 -0
- Utils/Patterns.py +24 -0
- Utils/Pipeline.py +137 -0
- Utils/PortForward.py +97 -0
- Utils/ProcessStats.py +103 -0
- Utils/PythonVersion.py +17 -0
- Utils/Signals.py +36 -0
- Utils/TemporaryEnvironment.py +27 -0
- Utils/Throttled.py +227 -0
- Utils/Timers.py +130 -0
- Utils/Timestamps.py +86 -0
- Utils/TokenManager.py +143 -0
- Utils/Tracing.py +60 -0
- Utils/TwPrint.py +98 -0
- Utils/Utilities.py +308 -0
- Utils/__init__.py +11 -0
- WMCore/ACDC/Collection.py +57 -0
- WMCore/ACDC/CollectionTypes.py +12 -0
- WMCore/ACDC/CouchCollection.py +67 -0
- WMCore/ACDC/CouchFileset.py +238 -0
- WMCore/ACDC/CouchService.py +73 -0
- WMCore/ACDC/DataCollectionService.py +485 -0
- WMCore/ACDC/Fileset.py +94 -0
- WMCore/ACDC/__init__.py +11 -0
- WMCore/Algorithms/Alarm.py +39 -0
- WMCore/Algorithms/MathAlgos.py +274 -0
- WMCore/Algorithms/MiscAlgos.py +67 -0
- WMCore/Algorithms/ParseXMLFile.py +115 -0
- WMCore/Algorithms/Permissions.py +27 -0
- WMCore/Algorithms/Singleton.py +58 -0
- WMCore/Algorithms/SubprocessAlgos.py +129 -0
- WMCore/Algorithms/__init__.py +7 -0
- WMCore/Cache/GenericDataCache.py +98 -0
- WMCore/Cache/WMConfigCache.py +572 -0
- WMCore/Cache/__init__.py +0 -0
- WMCore/Configuration.py +651 -0
- WMCore/DAOFactory.py +47 -0
- WMCore/DataStructs/File.py +177 -0
- WMCore/DataStructs/Fileset.py +140 -0
- WMCore/DataStructs/Job.py +182 -0
- WMCore/DataStructs/JobGroup.py +142 -0
- WMCore/DataStructs/JobPackage.py +49 -0
- WMCore/DataStructs/LumiList.py +734 -0
- WMCore/DataStructs/Mask.py +219 -0
- WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py +197 -0
- WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py +92 -0
- WMCore/DataStructs/MathStructs/SummaryHistogram.py +117 -0
- WMCore/DataStructs/MathStructs/__init__.py +0 -0
- WMCore/DataStructs/Pickleable.py +24 -0
- WMCore/DataStructs/Run.py +256 -0
- WMCore/DataStructs/Subscription.py +175 -0
- WMCore/DataStructs/WMObject.py +47 -0
- WMCore/DataStructs/WorkUnit.py +112 -0
- WMCore/DataStructs/Workflow.py +60 -0
- WMCore/DataStructs/__init__.py +8 -0
- WMCore/Database/CMSCouch.py +1349 -0
- WMCore/Database/ConfigDBMap.py +29 -0
- WMCore/Database/CouchUtils.py +118 -0
- WMCore/Database/DBCore.py +198 -0
- WMCore/Database/DBCreator.py +113 -0
- WMCore/Database/DBExceptionHandler.py +57 -0
- WMCore/Database/DBFactory.py +110 -0
- WMCore/Database/DBFormatter.py +177 -0
- WMCore/Database/Dialects.py +13 -0
- WMCore/Database/ExecuteDAO.py +327 -0
- WMCore/Database/MongoDB.py +241 -0
- WMCore/Database/MySQL/Destroy.py +42 -0
- WMCore/Database/MySQL/ListUserContent.py +20 -0
- WMCore/Database/MySQL/__init__.py +9 -0
- WMCore/Database/MySQLCore.py +132 -0
- WMCore/Database/Oracle/Destroy.py +56 -0
- WMCore/Database/Oracle/ListUserContent.py +19 -0
- WMCore/Database/Oracle/__init__.py +9 -0
- WMCore/Database/ResultSet.py +44 -0
- WMCore/Database/Transaction.py +91 -0
- WMCore/Database/__init__.py +9 -0
- WMCore/Database/ipy_profile_couch.py +438 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/CleanUpTask.py +29 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/HeartbeatMonitor.py +105 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/LocationUpdateTask.py +28 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/ReqMgrInteractionTask.py +35 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/__init__.py +0 -0
- WMCore/GlobalWorkQueue/__init__.py +0 -0
- WMCore/GroupUser/CouchObject.py +127 -0
- WMCore/GroupUser/Decorators.py +51 -0
- WMCore/GroupUser/Group.py +33 -0
- WMCore/GroupUser/Interface.py +73 -0
- WMCore/GroupUser/User.py +96 -0
- WMCore/GroupUser/__init__.py +11 -0
- WMCore/Lexicon.py +836 -0
- WMCore/REST/Auth.py +202 -0
- WMCore/REST/CherryPyPeriodicTask.py +166 -0
- WMCore/REST/Error.py +333 -0
- WMCore/REST/Format.py +642 -0
- WMCore/REST/HeartbeatMonitorBase.py +90 -0
- WMCore/REST/Main.py +623 -0
- WMCore/REST/Server.py +2435 -0
- WMCore/REST/Services.py +24 -0
- WMCore/REST/Test.py +120 -0
- WMCore/REST/Tools.py +38 -0
- WMCore/REST/Validation.py +250 -0
- WMCore/REST/__init__.py +1 -0
- WMCore/ReqMgr/DataStructs/RequestStatus.py +209 -0
- WMCore/ReqMgr/DataStructs/RequestType.py +13 -0
- WMCore/ReqMgr/DataStructs/__init__.py +0 -0
- WMCore/ReqMgr/__init__.py +1 -0
- WMCore/Services/AlertManager/AlertManagerAPI.py +111 -0
- WMCore/Services/AlertManager/__init__.py +0 -0
- WMCore/Services/CRIC/CRIC.py +238 -0
- WMCore/Services/CRIC/__init__.py +0 -0
- WMCore/Services/DBS/DBS3Reader.py +1044 -0
- WMCore/Services/DBS/DBSConcurrency.py +44 -0
- WMCore/Services/DBS/DBSErrors.py +113 -0
- WMCore/Services/DBS/DBSReader.py +23 -0
- WMCore/Services/DBS/DBSUtils.py +139 -0
- WMCore/Services/DBS/DBSWriterObjects.py +381 -0
- WMCore/Services/DBS/ProdException.py +133 -0
- WMCore/Services/DBS/__init__.py +8 -0
- WMCore/Services/FWJRDB/FWJRDBAPI.py +118 -0
- WMCore/Services/FWJRDB/__init__.py +0 -0
- WMCore/Services/HTTPS/HTTPSAuthHandler.py +66 -0
- WMCore/Services/HTTPS/__init__.py +0 -0
- WMCore/Services/LogDB/LogDB.py +201 -0
- WMCore/Services/LogDB/LogDBBackend.py +191 -0
- WMCore/Services/LogDB/LogDBExceptions.py +11 -0
- WMCore/Services/LogDB/LogDBReport.py +85 -0
- WMCore/Services/LogDB/__init__.py +0 -0
- WMCore/Services/MSPileup/__init__.py +0 -0
- WMCore/Services/MSUtils/MSUtils.py +54 -0
- WMCore/Services/MSUtils/__init__.py +0 -0
- WMCore/Services/McM/McM.py +173 -0
- WMCore/Services/McM/__init__.py +8 -0
- WMCore/Services/MonIT/Grafana.py +133 -0
- WMCore/Services/MonIT/__init__.py +0 -0
- WMCore/Services/PyCondor/PyCondorAPI.py +154 -0
- WMCore/Services/PyCondor/PyCondorUtils.py +105 -0
- WMCore/Services/PyCondor/__init__.py +0 -0
- WMCore/Services/ReqMgr/ReqMgr.py +261 -0
- WMCore/Services/ReqMgr/__init__.py +0 -0
- WMCore/Services/ReqMgrAux/ReqMgrAux.py +419 -0
- WMCore/Services/ReqMgrAux/__init__.py +0 -0
- WMCore/Services/RequestDB/RequestDBReader.py +267 -0
- WMCore/Services/RequestDB/RequestDBWriter.py +39 -0
- WMCore/Services/RequestDB/__init__.py +0 -0
- WMCore/Services/Requests.py +624 -0
- WMCore/Services/Rucio/Rucio.py +1287 -0
- WMCore/Services/Rucio/RucioUtils.py +74 -0
- WMCore/Services/Rucio/__init__.py +0 -0
- WMCore/Services/RucioConMon/RucioConMon.py +128 -0
- WMCore/Services/RucioConMon/__init__.py +0 -0
- WMCore/Services/Service.py +400 -0
- WMCore/Services/StompAMQ/__init__.py +0 -0
- WMCore/Services/TagCollector/TagCollector.py +155 -0
- WMCore/Services/TagCollector/XMLUtils.py +98 -0
- WMCore/Services/TagCollector/__init__.py +0 -0
- WMCore/Services/UUIDLib.py +13 -0
- WMCore/Services/UserFileCache/UserFileCache.py +160 -0
- WMCore/Services/UserFileCache/__init__.py +8 -0
- WMCore/Services/WMAgent/WMAgent.py +63 -0
- WMCore/Services/WMAgent/__init__.py +0 -0
- WMCore/Services/WMArchive/CMSSWMetrics.py +526 -0
- WMCore/Services/WMArchive/DataMap.py +463 -0
- WMCore/Services/WMArchive/WMArchive.py +33 -0
- WMCore/Services/WMArchive/__init__.py +0 -0
- WMCore/Services/WMBS/WMBS.py +97 -0
- WMCore/Services/WMBS/__init__.py +0 -0
- WMCore/Services/WMStats/DataStruct/RequestInfoCollection.py +300 -0
- WMCore/Services/WMStats/DataStruct/__init__.py +0 -0
- WMCore/Services/WMStats/WMStatsPycurl.py +145 -0
- WMCore/Services/WMStats/WMStatsReader.py +445 -0
- WMCore/Services/WMStats/WMStatsWriter.py +273 -0
- WMCore/Services/WMStats/__init__.py +0 -0
- WMCore/Services/WMStatsServer/WMStatsServer.py +134 -0
- WMCore/Services/WMStatsServer/__init__.py +0 -0
- WMCore/Services/WorkQueue/WorkQueue.py +492 -0
- WMCore/Services/WorkQueue/__init__.py +0 -0
- WMCore/Services/__init__.py +8 -0
- WMCore/Services/pycurl_manager.py +574 -0
- WMCore/WMBase.py +50 -0
- WMCore/WMConnectionBase.py +164 -0
- WMCore/WMException.py +183 -0
- WMCore/WMExceptions.py +269 -0
- WMCore/WMFactory.py +76 -0
- WMCore/WMInit.py +228 -0
- WMCore/WMLogging.py +108 -0
- WMCore/WMSpec/ConfigSectionTree.py +442 -0
- WMCore/WMSpec/Persistency.py +135 -0
- WMCore/WMSpec/Steps/BuildMaster.py +87 -0
- WMCore/WMSpec/Steps/BuildTools.py +201 -0
- WMCore/WMSpec/Steps/Builder.py +97 -0
- WMCore/WMSpec/Steps/Diagnostic.py +89 -0
- WMCore/WMSpec/Steps/Emulator.py +62 -0
- WMCore/WMSpec/Steps/ExecuteMaster.py +208 -0
- WMCore/WMSpec/Steps/Executor.py +210 -0
- WMCore/WMSpec/Steps/StepFactory.py +213 -0
- WMCore/WMSpec/Steps/TaskEmulator.py +75 -0
- WMCore/WMSpec/Steps/Template.py +204 -0
- WMCore/WMSpec/Steps/Templates/AlcaHarvest.py +76 -0
- WMCore/WMSpec/Steps/Templates/CMSSW.py +613 -0
- WMCore/WMSpec/Steps/Templates/DQMUpload.py +59 -0
- WMCore/WMSpec/Steps/Templates/DeleteFiles.py +70 -0
- WMCore/WMSpec/Steps/Templates/LogArchive.py +84 -0
- WMCore/WMSpec/Steps/Templates/LogCollect.py +105 -0
- WMCore/WMSpec/Steps/Templates/StageOut.py +105 -0
- WMCore/WMSpec/Steps/Templates/__init__.py +10 -0
- WMCore/WMSpec/Steps/WMExecutionFailure.py +21 -0
- WMCore/WMSpec/Steps/__init__.py +8 -0
- WMCore/WMSpec/Utilities.py +63 -0
- WMCore/WMSpec/WMSpecErrors.py +12 -0
- WMCore/WMSpec/WMStep.py +347 -0
- WMCore/WMSpec/WMTask.py +1980 -0
- WMCore/WMSpec/WMWorkload.py +2288 -0
- WMCore/WMSpec/WMWorkloadTools.py +370 -0
- WMCore/WMSpec/__init__.py +9 -0
- WMCore/WorkQueue/DataLocationMapper.py +273 -0
- WMCore/WorkQueue/DataStructs/ACDCBlock.py +47 -0
- WMCore/WorkQueue/DataStructs/Block.py +48 -0
- WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py +148 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElement.py +274 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py +152 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementsSummary.py +185 -0
- WMCore/WorkQueue/DataStructs/__init__.py +0 -0
- WMCore/WorkQueue/Policy/End/EndPolicyInterface.py +44 -0
- WMCore/WorkQueue/Policy/End/SingleShot.py +22 -0
- WMCore/WorkQueue/Policy/End/__init__.py +32 -0
- WMCore/WorkQueue/Policy/PolicyInterface.py +17 -0
- WMCore/WorkQueue/Policy/Start/Block.py +258 -0
- WMCore/WorkQueue/Policy/Start/Dataset.py +180 -0
- WMCore/WorkQueue/Policy/Start/MonteCarlo.py +131 -0
- WMCore/WorkQueue/Policy/Start/ResubmitBlock.py +171 -0
- WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +316 -0
- WMCore/WorkQueue/Policy/Start/__init__.py +34 -0
- WMCore/WorkQueue/Policy/__init__.py +57 -0
- WMCore/WorkQueue/WMBSHelper.py +772 -0
- WMCore/WorkQueue/WorkQueue.py +1237 -0
- WMCore/WorkQueue/WorkQueueBackend.py +750 -0
- WMCore/WorkQueue/WorkQueueBase.py +39 -0
- WMCore/WorkQueue/WorkQueueExceptions.py +44 -0
- WMCore/WorkQueue/WorkQueueReqMgrInterface.py +278 -0
- WMCore/WorkQueue/WorkQueueUtils.py +130 -0
- WMCore/WorkQueue/__init__.py +13 -0
- WMCore/Wrappers/JsonWrapper/JSONThunker.py +342 -0
- WMCore/Wrappers/JsonWrapper/__init__.py +7 -0
- WMCore/Wrappers/__init__.py +6 -0
- WMCore/__init__.py +10 -0
- wmglobalqueue-2.3.10.data/data/bin/wmc-dist-patch +15 -0
- wmglobalqueue-2.3.10.data/data/bin/wmc-dist-unpatch +8 -0
- wmglobalqueue-2.3.10.data/data/bin/wmc-httpd +3 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/.couchapprc +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/README.md +40 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/index.html +264 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/ElementInfoByWorkflow.js +96 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/StuckElementInfo.js +57 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/WorkloadInfoTable.js +80 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/dataTable.js +70 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/js/namespace.js +23 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/_attachments/style/main.css +75 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/couchapp.json +4 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/childQueueFilter.js +13 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/filterDeletedDocs.js +3 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/filters/queueFilter.js +11 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/language +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/mustache.js +333 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/validate.js +27 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lib/workqueue_utils.js +61 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/elementsDetail.js +28 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/filter.js +86 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/stuckElements.js +38 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/workRestrictions.js +153 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/lists/workflowSummary.js +28 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/rewrites.json +73 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/shows/redirect.js +23 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/shows/status.js +40 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/ElementSummaryByWorkflow.html +27 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/StuckElementSummary.html +26 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/TaskStatus.html +23 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/WorkflowSummary.html +27 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/workqueue-common-lib.html +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib-remote.html +16 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib.html +18 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/updates/in-place.js +50 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/validate_doc_update.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.couch.app.js +235 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.pathbinder.js +173 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeData/reduce.js +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeParentData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activeParentData/reduce.js +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activePileupData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/activePileupData/reduce.js +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/analyticsData/map.js +11 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/analyticsData/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/availableByPriority/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/conflicts/map.js +5 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elements/map.js +5 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByParent/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByParentData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByPileupData/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByStatus/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsBySubscription/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/map.js +8 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/reduce.js +3 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/elementsDetailByWorkflowAndStatus/map.js +26 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/map.js +10 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByRequest/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByRequest/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatus/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatus/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/openRequests/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/recent-items/map.js +5 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/reduce.js +1 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/specsByWorkflow/map.js +5 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/stuckElements/map.js +38 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/map.js +12 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/reduce.js +3 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrl/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrl/reduce.js +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/map.js +6 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/reduce.js +2 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/workflowSummary/map.js +9 -0
- wmglobalqueue-2.3.10.data/data/data/couchapps/WorkQueue/views/workflowSummary/reduce.js +10 -0
- wmglobalqueue-2.3.10.dist-info/LICENSE +202 -0
- wmglobalqueue-2.3.10.dist-info/METADATA +24 -0
- wmglobalqueue-2.3.10.dist-info/NOTICE +16 -0
- wmglobalqueue-2.3.10.dist-info/RECORD +345 -0
- wmglobalqueue-2.3.10.dist-info/WHEEL +5 -0
- wmglobalqueue-2.3.10.dist-info/top_level.txt +2 -0
WMCore/REST/Main.py
ADDED
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
""" wmc-httpd [-q] { -r | --restart } [-l LOG-FILE] CONFIG-FILE
|
|
2
|
+
wmc-httpd [-q] { -v | --verify } [-l LOG-FILE] CONFIG-FILE
|
|
3
|
+
wmc-httpd [-q] { -s | --status } CONFIG-FILE
|
|
4
|
+
wmc-httpd [-q] { -k | --kill } CONFIG-FILE
|
|
5
|
+
|
|
6
|
+
Manages a web server application. Loads configuration and all views, starting
|
|
7
|
+
up an appropriately configured CherryPy instance. Views are loaded dynamically
|
|
8
|
+
and can be turned on/off via configuration file."""
|
|
9
|
+
|
|
10
|
+
from __future__ import print_function
|
|
11
|
+
from builtins import object
|
|
12
|
+
from future.utils import viewitems
|
|
13
|
+
from future import standard_library
|
|
14
|
+
standard_library.install_aliases()
|
|
15
|
+
|
|
16
|
+
import errno
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import os.path
|
|
20
|
+
import re
|
|
21
|
+
import signal
|
|
22
|
+
import socket
|
|
23
|
+
import sys
|
|
24
|
+
import _thread
|
|
25
|
+
import time
|
|
26
|
+
import traceback
|
|
27
|
+
from argparse import ArgumentParser
|
|
28
|
+
from io import BytesIO, StringIO
|
|
29
|
+
from glob import glob
|
|
30
|
+
from subprocess import Popen, PIPE
|
|
31
|
+
from pprint import pformat
|
|
32
|
+
|
|
33
|
+
import cherrypy
|
|
34
|
+
from cherrypy import Application
|
|
35
|
+
from cherrypy._cplogging import LogManager
|
|
36
|
+
from cherrypy.lib import profiler
|
|
37
|
+
|
|
38
|
+
### Tools is needed for CRABServer startup: it sets up the tools attributes
|
|
39
|
+
import WMCore.REST.Tools
|
|
40
|
+
from WMCore.Configuration import ConfigSection, loadConfigurationFile
|
|
41
|
+
from Utils.Utilities import lowerCmsHeaders
|
|
42
|
+
from Utils.PythonVersion import PY2
|
|
43
|
+
|
|
44
|
+
#: Terminal controls to switch to "OK" status message colour.
|
|
45
|
+
COLOR_OK = "\033[0;32m"
|
|
46
|
+
|
|
47
|
+
#: Terminal controls to switch to "warning" status message colour.
|
|
48
|
+
COLOR_WARN = "\033[0;31m"
|
|
49
|
+
|
|
50
|
+
#: Terminal controls to restore normal message colour.
|
|
51
|
+
COLOR_NORMAL = "\033[0;39m"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def sig_terminate(signum=None, frame=None):
|
|
55
|
+
"""Termination signal handler.
|
|
56
|
+
|
|
57
|
+
CherryPy termination signal handling is broken, the handler does not
|
|
58
|
+
take the right number of arguments. This is our own fixed handler
|
|
59
|
+
to terminate the web server gracefully; in theory it could be
|
|
60
|
+
removed when CherryPy is fixed, but we attach other signals here
|
|
61
|
+
and print a logging entry."""
|
|
62
|
+
cherrypy.log("INFO: exiting server from termination signal %d" % signum, severity=logging.INFO)
|
|
63
|
+
cherrypy.engine.exit()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def sig_reload(signum=None, frame=None):
|
|
67
|
+
"""SIGHUP handler to restart the server.
|
|
68
|
+
|
|
69
|
+
This just adds some logging compared to the CherryPy signal handler."""
|
|
70
|
+
cherrypy.log("INFO: restarting server from hang-up signal %d" % signum, severity=logging.INFO)
|
|
71
|
+
cherrypy.engine.restart()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def sig_graceful(signum=None, frame=None):
|
|
75
|
+
"""SIGUSR1 handler to restart the server gracefully.
|
|
76
|
+
|
|
77
|
+
This just adds some logging compared to the CherryPy signal handler."""
|
|
78
|
+
cherrypy.log("INFO: restarting server gracefully from signal %d" % signum, severity=logging.INFO)
|
|
79
|
+
cherrypy.engine.graceful()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class ProfiledApp(Application):
|
|
83
|
+
"""Wrapper CherryPy Application object which generates aggregated
|
|
84
|
+
profiles for the component on each call. Note that there needs to
|
|
85
|
+
be an instance of this for each mount point to be profiled."""
|
|
86
|
+
|
|
87
|
+
def __init__(self, app, path):
|
|
88
|
+
Application.__init__(self, app.root, app.script_name, app.config)
|
|
89
|
+
self.profiler = profiler.ProfileAggregator(path)
|
|
90
|
+
|
|
91
|
+
def __call__(self, env, handler):
|
|
92
|
+
def gather(): return Application.__call__(self, env, handler)
|
|
93
|
+
|
|
94
|
+
return self.profiler.run(gather)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class Logger(LogManager):
|
|
98
|
+
"""Custom logger to record information in format we prefer."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, *args, **kwargs):
|
|
101
|
+
self.host = socket.gethostname()
|
|
102
|
+
LogManager.__init__(self, *args, **kwargs)
|
|
103
|
+
|
|
104
|
+
def access(self):
|
|
105
|
+
"""Record one client access."""
|
|
106
|
+
request = cherrypy.request
|
|
107
|
+
remote = request.remote
|
|
108
|
+
response = cherrypy.response
|
|
109
|
+
inheaders = lowerCmsHeaders(request.headers)
|
|
110
|
+
outheaders = response.headers
|
|
111
|
+
wfile = request.wsgi_environ.get('cherrypy.wfile', None)
|
|
112
|
+
nout = (wfile and wfile.bytes_written) or outheaders.get('Content-Length', 0)
|
|
113
|
+
if hasattr(request, 'start_time'):
|
|
114
|
+
delta_time = (time.time() - request.start_time) * 1e6
|
|
115
|
+
else:
|
|
116
|
+
delta_time = 0
|
|
117
|
+
msg = ('%(t)s %(H)s %(h)s "%(r)s" %(s)s'
|
|
118
|
+
' [data: %(i)s in %(b)s out %(T).0f us ]'
|
|
119
|
+
' [auth: %(AS)s "%(AU)s" "%(AC)s" ]'
|
|
120
|
+
' [ref: "%(f)s" "%(a)s" ]') % \
|
|
121
|
+
{'t': self.time(),
|
|
122
|
+
'H': self.host,
|
|
123
|
+
'h': remote.name or remote.ip,
|
|
124
|
+
'r': request.request_line,
|
|
125
|
+
's': response.status,
|
|
126
|
+
# request.rfile.rfile.bytes_read is a custom CMS web
|
|
127
|
+
# cherrypy patch not always available, hence the test
|
|
128
|
+
'i': (getattr(request.rfile, 'rfile', None)
|
|
129
|
+
and getattr(request.rfile.rfile, "bytes_read", None)
|
|
130
|
+
and request.rfile.rfile.bytes_read) or "-",
|
|
131
|
+
'b': nout or "-",
|
|
132
|
+
'T': delta_time,
|
|
133
|
+
'AS': inheaders.get("cms-auth-status", "-"),
|
|
134
|
+
'AU': inheaders.get("cms-auth-cert", inheaders.get("cms-auth-host", "")),
|
|
135
|
+
'AC': getattr(request.cookie.get("cms-auth", None), "value", ""),
|
|
136
|
+
'f': inheaders.get("Referer", ""),
|
|
137
|
+
'a': inheaders.get("User-Agent", "")}
|
|
138
|
+
self.access_log.log(logging.INFO, msg)
|
|
139
|
+
self.access_log.propagate = False # to avoid duplicate records on the log
|
|
140
|
+
self.error_log.propagate = False # to avoid duplicate records on the log
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class RESTMain(object):
|
|
144
|
+
"""Base class for the core CherryPy main application object.
|
|
145
|
+
|
|
146
|
+
The :class:`~.RESTMain` implements basic functionality of a CherryPy-based
|
|
147
|
+
server. Most users will want the fully functional :class:`~.RESTDaemon`
|
|
148
|
+
instead; but in some cases such as tests and other single-shot jobs which
|
|
149
|
+
don't require a daemon process this class is useful in its own right.
|
|
150
|
+
|
|
151
|
+
The class implements the methods required to configure, but not run, a
|
|
152
|
+
CherryPy server set up with an application configuration.
|
|
153
|
+
|
|
154
|
+
The main application object takes the server configuration and state
|
|
155
|
+
directory as parametres. It provides methods to create full CherryPy
|
|
156
|
+
serer and configure the application based on configuration description."""
|
|
157
|
+
|
|
158
|
+
def __init__(self, config, statedir):
|
|
159
|
+
"""Prepare the server.
|
|
160
|
+
|
|
161
|
+
:arg config: server configuration
|
|
162
|
+
:arg str statedir: server state directory."""
|
|
163
|
+
self.config = config
|
|
164
|
+
self.appname = config.main.application.lower()
|
|
165
|
+
self.appconfig = config.section_(self.appname)
|
|
166
|
+
self.srvconfig = config.section_("main")
|
|
167
|
+
self.statedir = statedir
|
|
168
|
+
self.hostname = socket.getfqdn().lower()
|
|
169
|
+
self.silent = False
|
|
170
|
+
self.extensions = {}
|
|
171
|
+
self.views = {}
|
|
172
|
+
|
|
173
|
+
def validate_config(self):
|
|
174
|
+
"""Check the server configuration has the required sections."""
|
|
175
|
+
for key in ('admin', 'description', 'title'):
|
|
176
|
+
if not hasattr(self.appconfig, key):
|
|
177
|
+
raise RuntimeError("'%s' required in application configuration" % key)
|
|
178
|
+
|
|
179
|
+
def setup_server(self):
|
|
180
|
+
"""Configure CherryPy server from application configuration.
|
|
181
|
+
|
|
182
|
+
Traverses the server configuration portion and applies parameters
|
|
183
|
+
known to be for CherryPy to the CherryPy server configuration.
|
|
184
|
+
These are: engine, hooks, log, request, respose, server, tools,
|
|
185
|
+
wsgi, checker.
|
|
186
|
+
|
|
187
|
+
Also applies pseudo-parameters ``thread_stack_size`` (default: 128kB)
|
|
188
|
+
and ``sys_check_interval`` (default: 10000). The former sets the
|
|
189
|
+
default stack size to desired value, to avoid excessively large
|
|
190
|
+
thread stacks -- typical operating system default is 8 MB, which
|
|
191
|
+
adds up rather a lot for lots of server threads. The latter sets
|
|
192
|
+
python's ``sys.setcheckinterval``; the default is to increase this
|
|
193
|
+
to avoid unnecessarily frequent checks for python's GIL, global
|
|
194
|
+
interpreter lock. In general we want each thread to complete as
|
|
195
|
+
quickly as possible without making unnecessary checks."""
|
|
196
|
+
cpconfig = cherrypy.config
|
|
197
|
+
|
|
198
|
+
# Determine server local base.
|
|
199
|
+
port = getattr(self.srvconfig, 'port', 8080)
|
|
200
|
+
local_base = getattr(self.srvconfig, 'local_base', socket.gethostname())
|
|
201
|
+
if local_base.find(':') == -1:
|
|
202
|
+
local_base = '%s:%d' % (local_base, port)
|
|
203
|
+
|
|
204
|
+
# Set default server configuration.
|
|
205
|
+
cherrypy.log = Logger()
|
|
206
|
+
|
|
207
|
+
cpconfig.update({'server.max_request_body_size': 0})
|
|
208
|
+
cpconfig.update({'server.environment': 'production'})
|
|
209
|
+
cpconfig.update({'server.socket_host': '0.0.0.0'})
|
|
210
|
+
cpconfig.update({'server.socket_port': port})
|
|
211
|
+
cpconfig.update({'server.socket_queue_size': 100})
|
|
212
|
+
cpconfig.update({'server.thread_pool': 100})
|
|
213
|
+
cpconfig.update({'tools.proxy.on': True})
|
|
214
|
+
cpconfig.update({'tools.proxy.base': local_base})
|
|
215
|
+
cpconfig.update({'tools.time.on': True})
|
|
216
|
+
cpconfig.update({'engine.autoreload.on': False})
|
|
217
|
+
cpconfig.update({'request.show_tracebacks': False})
|
|
218
|
+
cpconfig.update({'request.methods_with_bodies': ("POST", "PUT", "DELETE")})
|
|
219
|
+
_thread.stack_size(getattr(self.srvconfig, 'thread_stack_size', 128 * 1024))
|
|
220
|
+
# read sys_check_interval from server config which sets in instructions
|
|
221
|
+
# as we previously used sys.setcheckinterval
|
|
222
|
+
interval = getattr(self.srvconfig, 'sys_check_interval', 10000)
|
|
223
|
+
# set check interval in seconds for sys.setswitchinterval
|
|
224
|
+
sys.setswitchinterval(interval/1000)
|
|
225
|
+
self.silent = getattr(self.srvconfig, 'silent', False)
|
|
226
|
+
|
|
227
|
+
# Apply any override options from app config file.
|
|
228
|
+
for section in ('engine', 'hooks', 'log', 'request', 'response',
|
|
229
|
+
'server', 'tools', 'wsgi', 'checker'):
|
|
230
|
+
if not hasattr(self.srvconfig, section):
|
|
231
|
+
continue
|
|
232
|
+
for opt, value in viewitems(getattr(self.srvconfig, section).dictionary_()):
|
|
233
|
+
if isinstance(value, ConfigSection):
|
|
234
|
+
for xopt, xvalue in viewitems(value.dictionary_()):
|
|
235
|
+
cpconfig.update({"%s.%s.%s" % (section, opt, xopt): xvalue})
|
|
236
|
+
elif isinstance(value, str) or isinstance(value, int):
|
|
237
|
+
cpconfig.update({"%s.%s" % (section, opt): value})
|
|
238
|
+
else:
|
|
239
|
+
raise RuntimeError("%s.%s should be string or int, got %s"
|
|
240
|
+
% (section, opt, type(value)))
|
|
241
|
+
|
|
242
|
+
# Apply security customisation.
|
|
243
|
+
if hasattr(self.srvconfig, 'authz_defaults'):
|
|
244
|
+
defsec = self.srvconfig.authz_defaults
|
|
245
|
+
cpconfig.update({'tools.cms_auth.on': True})
|
|
246
|
+
cpconfig.update({'tools.cms_auth.role': defsec['role']})
|
|
247
|
+
cpconfig.update({'tools.cms_auth.group': defsec['group']})
|
|
248
|
+
cpconfig.update({'tools.cms_auth.site': defsec['site']})
|
|
249
|
+
|
|
250
|
+
if hasattr(self.srvconfig, 'authz_policy'):
|
|
251
|
+
cpconfig.update({'tools.cms_auth.policy': self.srvconfig.authz_policy})
|
|
252
|
+
cherrypy.log("INFO: final CherryPy configuration: %s" % pformat(cpconfig))
|
|
253
|
+
|
|
254
|
+
def install_application(self):
|
|
255
|
+
"""Install application and its components from the configuration."""
|
|
256
|
+
index = self.srvconfig.index
|
|
257
|
+
|
|
258
|
+
# First instantiate non-view extensions.
|
|
259
|
+
if getattr(self.config, 'extensions', None):
|
|
260
|
+
for ext in self.config.extensions:
|
|
261
|
+
name = ext._internal_name
|
|
262
|
+
if not self.silent:
|
|
263
|
+
cherrypy.log("INFO: instantiating extension %s" % name)
|
|
264
|
+
module_name, class_name = ext.object.rsplit(".", 1)
|
|
265
|
+
module = __import__(module_name, globals(), locals(), [class_name])
|
|
266
|
+
obj = getattr(module, class_name)(self, ext)
|
|
267
|
+
self.extensions[name] = obj
|
|
268
|
+
|
|
269
|
+
# Then instantiate views and mount them to cherrypy. If the view is
|
|
270
|
+
# designated as the index, create it as an application, profiled one
|
|
271
|
+
# if server profiling was requested. Otherwise just mount it as a
|
|
272
|
+
# normal server content object. Force tracebacks off for everything.
|
|
273
|
+
for view in self.config.views:
|
|
274
|
+
name = view._internal_name
|
|
275
|
+
path = "/%s" % self.appname + ((name != index and "/%s" % name) or "")
|
|
276
|
+
if not self.silent:
|
|
277
|
+
cherrypy.log("INFO: loading %s into %s" % (name, path))
|
|
278
|
+
module_name, class_name = view.object.rsplit(".", 1)
|
|
279
|
+
module = __import__(module_name, globals(), locals(), [class_name])
|
|
280
|
+
obj = getattr(module, class_name)(self, view, path)
|
|
281
|
+
app = Application(obj, path, {"/": {"request.show_tracebacks": False}})
|
|
282
|
+
if getattr(self.srvconfig, 'profile', False):
|
|
283
|
+
profdir = "%s/profile" % self.statedir
|
|
284
|
+
if not os.path.exists(profdir):
|
|
285
|
+
os.makedirs(profdir)
|
|
286
|
+
app = ProfiledApp(app, profdir)
|
|
287
|
+
cherrypy.tree.mount(app)
|
|
288
|
+
self.views[name] = obj
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
class RESTDaemon(RESTMain):
|
|
292
|
+
"""Web server object.
|
|
293
|
+
|
|
294
|
+
The `RESTDaemon` represents the web server daemon. It provides all
|
|
295
|
+
services for starting, stopping and checking the status of the daemon,
|
|
296
|
+
as well as running the main loop.
|
|
297
|
+
|
|
298
|
+
The class implements all the methods required for proper unix
|
|
299
|
+
daemonisation, including maintaing a PID file for the process group
|
|
300
|
+
and correct progressively more aggressive signals sent to terminate
|
|
301
|
+
the daemon. Starting multiple daemons in same directory is refused.
|
|
302
|
+
|
|
303
|
+
The daemon takes the server configuration as a parametre. When the
|
|
304
|
+
server is started, it creates a CherryPy server and configuration
|
|
305
|
+
from the application config contents."""
|
|
306
|
+
|
|
307
|
+
def __init__(self, config, statedir):
|
|
308
|
+
"""Initialise the daemon.
|
|
309
|
+
|
|
310
|
+
:arg config: server configuration
|
|
311
|
+
:arg str statedir: server state directory."""
|
|
312
|
+
RESTMain.__init__(self, config, statedir)
|
|
313
|
+
self.pidfile = "%s/%s.pid" % (self.statedir, self.appname)
|
|
314
|
+
self.logfile = ["rotatelogs", "%s/%s-%%Y%%m%%d.log" % (self.statedir, self.appname), "86400"]
|
|
315
|
+
|
|
316
|
+
def daemon_pid(self):
|
|
317
|
+
"""Check if there is a daemon running, and if so return its pid.
|
|
318
|
+
|
|
319
|
+
Reads the pid file from the daemon work directory, if any. If a
|
|
320
|
+
non-empty pid file exists, checks if a process by that PGID exists.
|
|
321
|
+
If yes, reports the daemon as running, otherwise reports either a
|
|
322
|
+
stale daemon no longer running or no deamon running at all.
|
|
323
|
+
|
|
324
|
+
:returns: A tuple (running, pgid). The first value will be True if a
|
|
325
|
+
running daemon was found, in which case pid will be its PGID. The
|
|
326
|
+
first value will be false otherwise, and pgid will be either None
|
|
327
|
+
if no pid file was found, or an integer if there was a stale file."""
|
|
328
|
+
pid = None
|
|
329
|
+
try:
|
|
330
|
+
with open(self.pidfile) as fd:
|
|
331
|
+
pid = int(fd.readline())
|
|
332
|
+
os.killpg(pid, 0)
|
|
333
|
+
return (True, pid)
|
|
334
|
+
except:
|
|
335
|
+
return (False, pid)
|
|
336
|
+
|
|
337
|
+
def kill_daemon(self, silent=False):
|
|
338
|
+
"""Check if the daemon is running, and if so kill it.
|
|
339
|
+
|
|
340
|
+
If there is no daemon running and no pid file, does nothing. If there
|
|
341
|
+
is a pid file but no such process running, removes the stale pid file.
|
|
342
|
+
Otherwise sends progressively more lethal signals at intervals to the
|
|
343
|
+
daemon process until it quits.
|
|
344
|
+
|
|
345
|
+
The signals are always sent to the entire process group, and signals
|
|
346
|
+
will keep on getting sent as long as at least one process from the
|
|
347
|
+
daemon process group is still alive. If for some reason the group
|
|
348
|
+
cannot be killed otherwise, sends SIGKILL to the group in the end.
|
|
349
|
+
|
|
350
|
+
The message about removing a stale pid file cannot be silenced. All
|
|
351
|
+
other messages are squelched if `silent` is True.
|
|
352
|
+
|
|
353
|
+
:arg bool silent: do not print any messages if True."""
|
|
354
|
+
running, pid = self.daemon_pid()
|
|
355
|
+
if not running:
|
|
356
|
+
if pid != None:
|
|
357
|
+
print("Removing stale pid file %s" % self.pidfile)
|
|
358
|
+
os.remove(self.pidfile)
|
|
359
|
+
else:
|
|
360
|
+
if not silent:
|
|
361
|
+
print("%s not running, not killing" % self.appname)
|
|
362
|
+
else:
|
|
363
|
+
if not silent:
|
|
364
|
+
sys.stdout.write("Killing %s pgid %d " % (self.appname, pid))
|
|
365
|
+
sys.stdout.flush()
|
|
366
|
+
|
|
367
|
+
dead = False
|
|
368
|
+
for sig, grace in ((signal.SIGINT, .5), (signal.SIGINT, 1),
|
|
369
|
+
(signal.SIGINT, 3), (signal.SIGINT, 5),
|
|
370
|
+
(signal.SIGKILL, 0)):
|
|
371
|
+
try:
|
|
372
|
+
if not silent:
|
|
373
|
+
sys.stdout.write(".")
|
|
374
|
+
sys.stdout.flush()
|
|
375
|
+
os.killpg(pid, sig)
|
|
376
|
+
time.sleep(grace)
|
|
377
|
+
os.killpg(pid, 0)
|
|
378
|
+
except OSError as e:
|
|
379
|
+
if e.errno == errno.ESRCH:
|
|
380
|
+
dead = True
|
|
381
|
+
break
|
|
382
|
+
raise
|
|
383
|
+
|
|
384
|
+
if not dead:
|
|
385
|
+
try:
|
|
386
|
+
os.killpg(pid, signal.SIGKILL)
|
|
387
|
+
except:
|
|
388
|
+
pass
|
|
389
|
+
|
|
390
|
+
if not silent:
|
|
391
|
+
sys.stdout.write("\n")
|
|
392
|
+
|
|
393
|
+
def start_daemon(self):
|
|
394
|
+
"""Start the deamon."""
|
|
395
|
+
# This REST server can be run in the frontend i.e. interactively (e.g. to use pdb)
|
|
396
|
+
# by setting an env. var. via: export DONT_DAEMONIZE_REST=True
|
|
397
|
+
# if the variable is missing or set to any other value, code starts normally as a daemon
|
|
398
|
+
daemonize = os.getenv('DONT_DAEMONIZE_REST', 'False') != 'True'
|
|
399
|
+
|
|
400
|
+
os.chdir(self.statedir)
|
|
401
|
+
|
|
402
|
+
if daemonize:
|
|
403
|
+
# Redirect all output to the logging daemon.
|
|
404
|
+
devnull = open(os.devnull, "w")
|
|
405
|
+
if isinstance(self.logfile, list):
|
|
406
|
+
subproc = Popen(self.logfile, stdin=PIPE, stdout=devnull, stderr=devnull,
|
|
407
|
+
bufsize=0, close_fds=True, shell=False)
|
|
408
|
+
logger = subproc.stdin
|
|
409
|
+
elif isinstance(self.logfile, str):
|
|
410
|
+
# if a unix pipe is set as the logfile, it must be opened to append to the end of the file
|
|
411
|
+
# if file/pipe does not exist, create it
|
|
412
|
+
logger = open(self.logfile, "a")
|
|
413
|
+
else:
|
|
414
|
+
raise TypeError("'logfile' must be a string or array")
|
|
415
|
+
os.dup2(logger.fileno(), sys.stdout.fileno())
|
|
416
|
+
os.dup2(logger.fileno(), sys.stderr.fileno())
|
|
417
|
+
os.dup2(devnull.fileno(), sys.stdin.fileno())
|
|
418
|
+
logger.close()
|
|
419
|
+
devnull.close()
|
|
420
|
+
|
|
421
|
+
# First fork. Discard the parent.
|
|
422
|
+
pid = os.fork()
|
|
423
|
+
if pid > 0:
|
|
424
|
+
os._exit(0)
|
|
425
|
+
|
|
426
|
+
# Establish as a daemon, set process group / session id.
|
|
427
|
+
os.setsid()
|
|
428
|
+
|
|
429
|
+
# Second fork. The child does the work, discard the second parent.
|
|
430
|
+
pid = os.fork()
|
|
431
|
+
if pid > 0:
|
|
432
|
+
os._exit(0)
|
|
433
|
+
|
|
434
|
+
# Save process group id to pid file, then run real worker.
|
|
435
|
+
with open(self.pidfile, "w") as pidObj:
|
|
436
|
+
pidObj.write("%d\n" % os.getpgid(0))
|
|
437
|
+
|
|
438
|
+
# following code is executed both in daemon and not daemon mode
|
|
439
|
+
error = False
|
|
440
|
+
try:
|
|
441
|
+
self.run()
|
|
442
|
+
except Exception as e:
|
|
443
|
+
error = True
|
|
444
|
+
if PY2:
|
|
445
|
+
trace = BytesIO()
|
|
446
|
+
else:
|
|
447
|
+
trace = StringIO()
|
|
448
|
+
traceback.print_exc(file=trace)
|
|
449
|
+
cherrypy.log("ERROR: terminating due to error: %s" % trace.getvalue())
|
|
450
|
+
|
|
451
|
+
# Remove pid file once we are done.
|
|
452
|
+
try:
|
|
453
|
+
os.remove(self.pidfile)
|
|
454
|
+
except:
|
|
455
|
+
pass
|
|
456
|
+
|
|
457
|
+
# Exit
|
|
458
|
+
sys.exit((error and 1) or 0)
|
|
459
|
+
|
|
460
|
+
def run(self):
|
|
461
|
+
"""Run the server daemon main loop."""
|
|
462
|
+
# Fork. The child always exits the loop and executes the code below
|
|
463
|
+
# to run the server proper. The parent monitors the child, and if
|
|
464
|
+
# it exits abnormally, restarts it, otherwise exits completely with
|
|
465
|
+
# the child's exit code.
|
|
466
|
+
daemonize = os.getenv('DONT_DAEMONIZE_REST', 'False') != 'True'
|
|
467
|
+
if daemonize:
|
|
468
|
+
cherrypy.log("WATCHDOG: starting server daemon (pid %d)" % os.getpid())
|
|
469
|
+
while True:
|
|
470
|
+
serverpid = os.fork()
|
|
471
|
+
if not serverpid: break
|
|
472
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
473
|
+
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
474
|
+
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
|
|
475
|
+
(xpid, exitrc) = os.waitpid(serverpid, 0)
|
|
476
|
+
(exitcode, exitsigno, exitcore) = (exitrc >> 8, exitrc & 127, exitrc & 128)
|
|
477
|
+
retval = (exitsigno and ("signal %d" % exitsigno)) or str(exitcode)
|
|
478
|
+
retmsg = retval + ((exitcore and " (core dumped)") or "")
|
|
479
|
+
restart = (exitsigno > 0 and exitsigno not in (2, 3, 15))
|
|
480
|
+
cherrypy.log("WATCHDOG: server exited with exit code %s%s"
|
|
481
|
+
% (retmsg, (restart and "... restarting") or ""))
|
|
482
|
+
|
|
483
|
+
if not restart:
|
|
484
|
+
sys.exit((exitsigno and 1) or exitcode)
|
|
485
|
+
|
|
486
|
+
for pidfile in glob("%s/*/*pid" % self.statedir):
|
|
487
|
+
if os.path.exists(pidfile):
|
|
488
|
+
with open(pidfile) as fd:
|
|
489
|
+
pid = int(fd.readline())
|
|
490
|
+
os.remove(pidfile)
|
|
491
|
+
cherrypy.log("WATCHDOG: killing slave server %d" % pid)
|
|
492
|
+
try:
|
|
493
|
+
os.kill(pid, 9)
|
|
494
|
+
except:
|
|
495
|
+
pass
|
|
496
|
+
|
|
497
|
+
# Run. Override signal handlers after CherryPy has itself started and
|
|
498
|
+
# installed its own handlers. To achieve this we need to start the
|
|
499
|
+
# server in non-blocking mode, fiddle with, than ask server to block.
|
|
500
|
+
self.validate_config()
|
|
501
|
+
self.setup_server()
|
|
502
|
+
self.install_application()
|
|
503
|
+
cherrypy.log("INFO: starting server in %s" % self.statedir)
|
|
504
|
+
cherrypy.config.update({'log.screen': bool(getattr(self.srvconfig, "log_screen", True))})
|
|
505
|
+
cherrypy.engine.start()
|
|
506
|
+
signal.signal(signal.SIGHUP, sig_reload)
|
|
507
|
+
signal.signal(signal.SIGUSR1, sig_graceful)
|
|
508
|
+
signal.signal(signal.SIGTERM, sig_terminate)
|
|
509
|
+
signal.signal(signal.SIGQUIT, sig_terminate)
|
|
510
|
+
signal.signal(signal.SIGINT, sig_terminate)
|
|
511
|
+
cherrypy.engine.block()
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def main():
|
|
515
|
+
# Re-exec if we don't have unbuffered i/o. This is essential to get server
|
|
516
|
+
# to output its logs synchronous to its operation, such that log output does
|
|
517
|
+
# not remain buffered in the python server. This is particularly important
|
|
518
|
+
# when infrequently accessed server redirects output to 'rotatelogs'.
|
|
519
|
+
if 'PYTHONUNBUFFERED' not in os.environ:
|
|
520
|
+
os.environ['PYTHONUNBUFFERED'] = "1"
|
|
521
|
+
if PY2:
|
|
522
|
+
os.execvp("python", ["python"] + sys.argv)
|
|
523
|
+
else:
|
|
524
|
+
os.execvp("python3", ["python3"] + sys.argv)
|
|
525
|
+
|
|
526
|
+
opt = ArgumentParser(usage=__doc__)
|
|
527
|
+
opt.add_argument("-q", "--quiet", action="store_true", dest="quiet", default=False,
|
|
528
|
+
help="be quiet, don't print unnecessary output")
|
|
529
|
+
opt.add_argument("-v", "--verify", action="store_true", dest="verify", default=False,
|
|
530
|
+
help="verify daemon is running, restart if not")
|
|
531
|
+
opt.add_argument("-s", "--status", action="store_true", dest="status", default=False,
|
|
532
|
+
help="check if the server monitor daemon is running")
|
|
533
|
+
opt.add_argument("-k", "--kill", action="store_true", dest="kill", default=False,
|
|
534
|
+
help="kill any existing already running daemon")
|
|
535
|
+
opt.add_argument("-r", "--restart", action="store_true", dest="restart", default=False,
|
|
536
|
+
help="restart, kill any existing running daemon first")
|
|
537
|
+
opt.add_argument("-d", "--dir", dest="statedir", metavar="DIR", default=os.getcwd(),
|
|
538
|
+
help="server state directory (default: current working directory)")
|
|
539
|
+
opt.add_argument("-l", "--log", dest="logfile", metavar="DEST", default=None,
|
|
540
|
+
help="log to DEST, via pipe if DEST begins with '|', otherwise a file")
|
|
541
|
+
opts, args = opt.parse_known_args()
|
|
542
|
+
|
|
543
|
+
if len(args) != 1:
|
|
544
|
+
print("%s: exactly one configuration file required" % sys.argv[0], file=sys.stderr)
|
|
545
|
+
sys.exit(1)
|
|
546
|
+
|
|
547
|
+
if not os.path.isfile(args[0]) or not os.access(args[0], os.R_OK):
|
|
548
|
+
print("%s: %s: invalid configuration file" % (sys.argv[0], args[0]), file=sys.stderr)
|
|
549
|
+
sys.exit(1)
|
|
550
|
+
|
|
551
|
+
if not opts.statedir or \
|
|
552
|
+
not os.path.isdir(opts.statedir) or \
|
|
553
|
+
not os.access(opts.statedir, os.W_OK):
|
|
554
|
+
print("%s: %s: invalid state directory" % (sys.argv[0], opts.statedir), file=sys.stderr)
|
|
555
|
+
sys.exit(1)
|
|
556
|
+
|
|
557
|
+
# Create server object.
|
|
558
|
+
cfg = loadConfigurationFile(args[0])
|
|
559
|
+
app = cfg.main.application.lower()
|
|
560
|
+
server = RESTDaemon(cfg, opts.statedir)
|
|
561
|
+
|
|
562
|
+
# Now actually execute the task.
|
|
563
|
+
if opts.status:
|
|
564
|
+
# Show status of running daemon, including exit code matching the
|
|
565
|
+
# daemon status: 0 = running, 1 = not running, 2 = not running but
|
|
566
|
+
# there is a stale pid file. If silent don't print out anything
|
|
567
|
+
# but still return the right exit code.
|
|
568
|
+
running, pid = server.daemon_pid()
|
|
569
|
+
if running:
|
|
570
|
+
if not opts.quiet:
|
|
571
|
+
print("%s is %sRUNNING%s, PID %d" \
|
|
572
|
+
% (app, COLOR_OK, COLOR_NORMAL, pid))
|
|
573
|
+
sys.exit(0)
|
|
574
|
+
elif pid != None:
|
|
575
|
+
if not opts.quiet:
|
|
576
|
+
print("%s is %sNOT RUNNING%s, stale PID %d" \
|
|
577
|
+
% (app, COLOR_WARN, COLOR_NORMAL, pid))
|
|
578
|
+
sys.exit(2)
|
|
579
|
+
else:
|
|
580
|
+
if not opts.quiet:
|
|
581
|
+
print("%s is %sNOT RUNNING%s" \
|
|
582
|
+
% (app, COLOR_WARN, COLOR_NORMAL))
|
|
583
|
+
sys.exit(1)
|
|
584
|
+
|
|
585
|
+
elif opts.kill:
|
|
586
|
+
# Stop any previously running daemon. If quiet squelch messages,
|
|
587
|
+
# except removal of stale pid file cannot be silenced.
|
|
588
|
+
server.kill_daemon(silent=opts.quiet)
|
|
589
|
+
|
|
590
|
+
else:
|
|
591
|
+
# We are handling a server start, in one of many possible ways:
|
|
592
|
+
# normal start, restart (= kill any previous daemon), or verify
|
|
593
|
+
# (= if daemon is running leave it alone, otherwise start).
|
|
594
|
+
|
|
595
|
+
# Convert 'verify' to 'restart' if the server isn't running.
|
|
596
|
+
if opts.verify:
|
|
597
|
+
opts.restart = True
|
|
598
|
+
if server.daemon_pid()[0]:
|
|
599
|
+
sys.exit(0)
|
|
600
|
+
|
|
601
|
+
# If restarting, kill any previous daemon, otherwise complain if
|
|
602
|
+
# there is a daemon already running here. Starting overlapping
|
|
603
|
+
# daemons is not supported because pid file would be overwritten
|
|
604
|
+
# and we'd lose track of the previous daemon.
|
|
605
|
+
if opts.restart:
|
|
606
|
+
server.kill_daemon(silent=opts.quiet)
|
|
607
|
+
else:
|
|
608
|
+
running, pid = server.daemon_pid()
|
|
609
|
+
if running:
|
|
610
|
+
print("Refusing to start over an already running daemon, pid %d" % pid, file=sys.stderr)
|
|
611
|
+
sys.exit(1)
|
|
612
|
+
|
|
613
|
+
# If we are (re)starting and were given a log file option, convert
|
|
614
|
+
# the logfile option to a list if it looks like a pipe request, i.e.
|
|
615
|
+
# starts with "|", such as "|rotatelogs foo/bar-%Y%m%d.log".
|
|
616
|
+
if opts.logfile:
|
|
617
|
+
if opts.logfile.startswith("|"):
|
|
618
|
+
server.logfile = re.split(r"\s+", opts.logfile[1:])
|
|
619
|
+
else:
|
|
620
|
+
server.logfile = opts.logfile
|
|
621
|
+
|
|
622
|
+
# Actually start the daemon now.
|
|
623
|
+
server.start_daemon()
|