wmglobalqueue 2.4.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Utils/CPMetrics.py +270 -0
- Utils/CertTools.py +100 -0
- Utils/EmailAlert.py +50 -0
- Utils/ExtendedUnitTestCase.py +62 -0
- Utils/FileTools.py +182 -0
- Utils/IteratorTools.py +80 -0
- Utils/MathUtils.py +31 -0
- Utils/MemoryCache.py +119 -0
- Utils/Patterns.py +24 -0
- Utils/Pipeline.py +137 -0
- Utils/PortForward.py +97 -0
- Utils/ProcFS.py +112 -0
- Utils/ProcessStats.py +194 -0
- Utils/PythonVersion.py +17 -0
- Utils/Signals.py +36 -0
- Utils/TemporaryEnvironment.py +27 -0
- Utils/Throttled.py +227 -0
- Utils/Timers.py +130 -0
- Utils/Timestamps.py +86 -0
- Utils/TokenManager.py +143 -0
- Utils/Tracing.py +60 -0
- Utils/TwPrint.py +98 -0
- Utils/Utilities.py +318 -0
- Utils/__init__.py +11 -0
- Utils/wmcoreDTools.py +707 -0
- WMCore/ACDC/Collection.py +57 -0
- WMCore/ACDC/CollectionTypes.py +12 -0
- WMCore/ACDC/CouchCollection.py +67 -0
- WMCore/ACDC/CouchFileset.py +238 -0
- WMCore/ACDC/CouchService.py +73 -0
- WMCore/ACDC/DataCollectionService.py +485 -0
- WMCore/ACDC/Fileset.py +94 -0
- WMCore/ACDC/__init__.py +11 -0
- WMCore/Algorithms/Alarm.py +39 -0
- WMCore/Algorithms/MathAlgos.py +274 -0
- WMCore/Algorithms/MiscAlgos.py +67 -0
- WMCore/Algorithms/ParseXMLFile.py +115 -0
- WMCore/Algorithms/Permissions.py +27 -0
- WMCore/Algorithms/Singleton.py +58 -0
- WMCore/Algorithms/SubprocessAlgos.py +129 -0
- WMCore/Algorithms/__init__.py +7 -0
- WMCore/Cache/GenericDataCache.py +98 -0
- WMCore/Cache/WMConfigCache.py +572 -0
- WMCore/Cache/__init__.py +0 -0
- WMCore/Configuration.py +659 -0
- WMCore/DAOFactory.py +47 -0
- WMCore/DataStructs/File.py +177 -0
- WMCore/DataStructs/Fileset.py +140 -0
- WMCore/DataStructs/Job.py +182 -0
- WMCore/DataStructs/JobGroup.py +142 -0
- WMCore/DataStructs/JobPackage.py +49 -0
- WMCore/DataStructs/LumiList.py +734 -0
- WMCore/DataStructs/Mask.py +219 -0
- WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py +197 -0
- WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py +92 -0
- WMCore/DataStructs/MathStructs/SummaryHistogram.py +117 -0
- WMCore/DataStructs/MathStructs/__init__.py +0 -0
- WMCore/DataStructs/Pickleable.py +24 -0
- WMCore/DataStructs/Run.py +256 -0
- WMCore/DataStructs/Subscription.py +175 -0
- WMCore/DataStructs/WMObject.py +47 -0
- WMCore/DataStructs/WorkUnit.py +112 -0
- WMCore/DataStructs/Workflow.py +60 -0
- WMCore/DataStructs/__init__.py +8 -0
- WMCore/Database/CMSCouch.py +1430 -0
- WMCore/Database/ConfigDBMap.py +29 -0
- WMCore/Database/CouchMonitoring.py +450 -0
- WMCore/Database/CouchUtils.py +118 -0
- WMCore/Database/DBCore.py +198 -0
- WMCore/Database/DBCreator.py +113 -0
- WMCore/Database/DBExceptionHandler.py +59 -0
- WMCore/Database/DBFactory.py +117 -0
- WMCore/Database/DBFormatter.py +177 -0
- WMCore/Database/Dialects.py +13 -0
- WMCore/Database/ExecuteDAO.py +327 -0
- WMCore/Database/MongoDB.py +241 -0
- WMCore/Database/MySQL/Destroy.py +42 -0
- WMCore/Database/MySQL/ListUserContent.py +20 -0
- WMCore/Database/MySQL/__init__.py +9 -0
- WMCore/Database/MySQLCore.py +132 -0
- WMCore/Database/Oracle/Destroy.py +56 -0
- WMCore/Database/Oracle/ListUserContent.py +19 -0
- WMCore/Database/Oracle/__init__.py +9 -0
- WMCore/Database/ResultSet.py +44 -0
- WMCore/Database/Transaction.py +91 -0
- WMCore/Database/__init__.py +9 -0
- WMCore/Database/ipy_profile_couch.py +438 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/CleanUpTask.py +29 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/HeartbeatMonitor.py +105 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/LocationUpdateTask.py +28 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/ReqMgrInteractionTask.py +35 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/__init__.py +0 -0
- WMCore/GlobalWorkQueue/__init__.py +0 -0
- WMCore/GroupUser/CouchObject.py +127 -0
- WMCore/GroupUser/Decorators.py +51 -0
- WMCore/GroupUser/Group.py +33 -0
- WMCore/GroupUser/Interface.py +73 -0
- WMCore/GroupUser/User.py +96 -0
- WMCore/GroupUser/__init__.py +11 -0
- WMCore/Lexicon.py +836 -0
- WMCore/REST/Auth.py +202 -0
- WMCore/REST/CherryPyPeriodicTask.py +166 -0
- WMCore/REST/Error.py +333 -0
- WMCore/REST/Format.py +642 -0
- WMCore/REST/HeartbeatMonitorBase.py +90 -0
- WMCore/REST/Main.py +636 -0
- WMCore/REST/Server.py +2435 -0
- WMCore/REST/Services.py +24 -0
- WMCore/REST/Test.py +120 -0
- WMCore/REST/Tools.py +38 -0
- WMCore/REST/Validation.py +250 -0
- WMCore/REST/__init__.py +1 -0
- WMCore/ReqMgr/DataStructs/RequestStatus.py +209 -0
- WMCore/ReqMgr/DataStructs/RequestType.py +13 -0
- WMCore/ReqMgr/DataStructs/__init__.py +0 -0
- WMCore/ReqMgr/__init__.py +1 -0
- WMCore/Services/AlertManager/AlertManagerAPI.py +111 -0
- WMCore/Services/AlertManager/__init__.py +0 -0
- WMCore/Services/CRIC/CRIC.py +238 -0
- WMCore/Services/CRIC/__init__.py +0 -0
- WMCore/Services/DBS/DBS3Reader.py +1044 -0
- WMCore/Services/DBS/DBSConcurrency.py +44 -0
- WMCore/Services/DBS/DBSErrors.py +112 -0
- WMCore/Services/DBS/DBSReader.py +23 -0
- WMCore/Services/DBS/DBSUtils.py +166 -0
- WMCore/Services/DBS/DBSWriterObjects.py +381 -0
- WMCore/Services/DBS/ProdException.py +133 -0
- WMCore/Services/DBS/__init__.py +8 -0
- WMCore/Services/FWJRDB/FWJRDBAPI.py +118 -0
- WMCore/Services/FWJRDB/__init__.py +0 -0
- WMCore/Services/HTTPS/HTTPSAuthHandler.py +66 -0
- WMCore/Services/HTTPS/__init__.py +0 -0
- WMCore/Services/LogDB/LogDB.py +201 -0
- WMCore/Services/LogDB/LogDBBackend.py +191 -0
- WMCore/Services/LogDB/LogDBExceptions.py +11 -0
- WMCore/Services/LogDB/LogDBReport.py +85 -0
- WMCore/Services/LogDB/__init__.py +0 -0
- WMCore/Services/MSPileup/__init__.py +0 -0
- WMCore/Services/MSUtils/MSUtils.py +54 -0
- WMCore/Services/MSUtils/__init__.py +0 -0
- WMCore/Services/McM/McM.py +173 -0
- WMCore/Services/McM/__init__.py +8 -0
- WMCore/Services/MonIT/Grafana.py +133 -0
- WMCore/Services/MonIT/__init__.py +0 -0
- WMCore/Services/PyCondor/PyCondorAPI.py +154 -0
- WMCore/Services/PyCondor/__init__.py +0 -0
- WMCore/Services/ReqMgr/ReqMgr.py +261 -0
- WMCore/Services/ReqMgr/__init__.py +0 -0
- WMCore/Services/ReqMgrAux/ReqMgrAux.py +419 -0
- WMCore/Services/ReqMgrAux/__init__.py +0 -0
- WMCore/Services/RequestDB/RequestDBReader.py +267 -0
- WMCore/Services/RequestDB/RequestDBWriter.py +39 -0
- WMCore/Services/RequestDB/__init__.py +0 -0
- WMCore/Services/Requests.py +624 -0
- WMCore/Services/Rucio/Rucio.py +1290 -0
- WMCore/Services/Rucio/RucioUtils.py +74 -0
- WMCore/Services/Rucio/__init__.py +0 -0
- WMCore/Services/RucioConMon/RucioConMon.py +121 -0
- WMCore/Services/RucioConMon/__init__.py +0 -0
- WMCore/Services/Service.py +400 -0
- WMCore/Services/StompAMQ/__init__.py +0 -0
- WMCore/Services/TagCollector/TagCollector.py +155 -0
- WMCore/Services/TagCollector/XMLUtils.py +98 -0
- WMCore/Services/TagCollector/__init__.py +0 -0
- WMCore/Services/UUIDLib.py +13 -0
- WMCore/Services/UserFileCache/UserFileCache.py +160 -0
- WMCore/Services/UserFileCache/__init__.py +8 -0
- WMCore/Services/WMAgent/WMAgent.py +63 -0
- WMCore/Services/WMAgent/__init__.py +0 -0
- WMCore/Services/WMArchive/CMSSWMetrics.py +526 -0
- WMCore/Services/WMArchive/DataMap.py +463 -0
- WMCore/Services/WMArchive/WMArchive.py +33 -0
- WMCore/Services/WMArchive/__init__.py +0 -0
- WMCore/Services/WMBS/WMBS.py +97 -0
- WMCore/Services/WMBS/__init__.py +0 -0
- WMCore/Services/WMStats/DataStruct/RequestInfoCollection.py +300 -0
- WMCore/Services/WMStats/DataStruct/__init__.py +0 -0
- WMCore/Services/WMStats/WMStatsPycurl.py +145 -0
- WMCore/Services/WMStats/WMStatsReader.py +445 -0
- WMCore/Services/WMStats/WMStatsWriter.py +273 -0
- WMCore/Services/WMStats/__init__.py +0 -0
- WMCore/Services/WMStatsServer/WMStatsServer.py +134 -0
- WMCore/Services/WMStatsServer/__init__.py +0 -0
- WMCore/Services/WorkQueue/WorkQueue.py +492 -0
- WMCore/Services/WorkQueue/__init__.py +0 -0
- WMCore/Services/__init__.py +8 -0
- WMCore/Services/pycurl_manager.py +574 -0
- WMCore/WMBase.py +50 -0
- WMCore/WMConnectionBase.py +164 -0
- WMCore/WMException.py +183 -0
- WMCore/WMExceptions.py +269 -0
- WMCore/WMFactory.py +76 -0
- WMCore/WMInit.py +377 -0
- WMCore/WMLogging.py +104 -0
- WMCore/WMSpec/ConfigSectionTree.py +442 -0
- WMCore/WMSpec/Persistency.py +135 -0
- WMCore/WMSpec/Steps/BuildMaster.py +87 -0
- WMCore/WMSpec/Steps/BuildTools.py +201 -0
- WMCore/WMSpec/Steps/Builder.py +97 -0
- WMCore/WMSpec/Steps/Diagnostic.py +89 -0
- WMCore/WMSpec/Steps/Emulator.py +62 -0
- WMCore/WMSpec/Steps/ExecuteMaster.py +208 -0
- WMCore/WMSpec/Steps/Executor.py +210 -0
- WMCore/WMSpec/Steps/StepFactory.py +213 -0
- WMCore/WMSpec/Steps/TaskEmulator.py +75 -0
- WMCore/WMSpec/Steps/Template.py +204 -0
- WMCore/WMSpec/Steps/Templates/AlcaHarvest.py +76 -0
- WMCore/WMSpec/Steps/Templates/CMSSW.py +613 -0
- WMCore/WMSpec/Steps/Templates/DQMUpload.py +59 -0
- WMCore/WMSpec/Steps/Templates/DeleteFiles.py +70 -0
- WMCore/WMSpec/Steps/Templates/LogArchive.py +84 -0
- WMCore/WMSpec/Steps/Templates/LogCollect.py +105 -0
- WMCore/WMSpec/Steps/Templates/StageOut.py +105 -0
- WMCore/WMSpec/Steps/Templates/__init__.py +10 -0
- WMCore/WMSpec/Steps/WMExecutionFailure.py +21 -0
- WMCore/WMSpec/Steps/__init__.py +8 -0
- WMCore/WMSpec/Utilities.py +63 -0
- WMCore/WMSpec/WMSpecErrors.py +12 -0
- WMCore/WMSpec/WMStep.py +347 -0
- WMCore/WMSpec/WMTask.py +1997 -0
- WMCore/WMSpec/WMWorkload.py +2288 -0
- WMCore/WMSpec/WMWorkloadTools.py +382 -0
- WMCore/WMSpec/__init__.py +9 -0
- WMCore/WorkQueue/DataLocationMapper.py +273 -0
- WMCore/WorkQueue/DataStructs/ACDCBlock.py +47 -0
- WMCore/WorkQueue/DataStructs/Block.py +48 -0
- WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py +148 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElement.py +274 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py +152 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementsSummary.py +185 -0
- WMCore/WorkQueue/DataStructs/__init__.py +0 -0
- WMCore/WorkQueue/Policy/End/EndPolicyInterface.py +44 -0
- WMCore/WorkQueue/Policy/End/SingleShot.py +22 -0
- WMCore/WorkQueue/Policy/End/__init__.py +32 -0
- WMCore/WorkQueue/Policy/PolicyInterface.py +17 -0
- WMCore/WorkQueue/Policy/Start/Block.py +258 -0
- WMCore/WorkQueue/Policy/Start/Dataset.py +180 -0
- WMCore/WorkQueue/Policy/Start/MonteCarlo.py +131 -0
- WMCore/WorkQueue/Policy/Start/ResubmitBlock.py +171 -0
- WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +316 -0
- WMCore/WorkQueue/Policy/Start/__init__.py +34 -0
- WMCore/WorkQueue/Policy/__init__.py +57 -0
- WMCore/WorkQueue/WMBSHelper.py +772 -0
- WMCore/WorkQueue/WorkQueue.py +1237 -0
- WMCore/WorkQueue/WorkQueueBackend.py +750 -0
- WMCore/WorkQueue/WorkQueueBase.py +39 -0
- WMCore/WorkQueue/WorkQueueExceptions.py +44 -0
- WMCore/WorkQueue/WorkQueueReqMgrInterface.py +278 -0
- WMCore/WorkQueue/WorkQueueUtils.py +130 -0
- WMCore/WorkQueue/__init__.py +13 -0
- WMCore/Wrappers/JsonWrapper/JSONThunker.py +342 -0
- WMCore/Wrappers/JsonWrapper/__init__.py +7 -0
- WMCore/Wrappers/__init__.py +6 -0
- WMCore/__init__.py +10 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-patch +15 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-unpatch +8 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-httpd +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/.couchapprc +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/README.md +40 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/index.html +264 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/ElementInfoByWorkflow.js +96 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/StuckElementInfo.js +57 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/WorkloadInfoTable.js +80 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/dataTable.js +70 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/namespace.js +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/style/main.css +75 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/couchapp.json +4 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/childQueueFilter.js +13 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/filterDeletedDocs.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/queueFilter.js +11 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/language +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/mustache.js +333 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/validate.js +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/workqueue_utils.js +61 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/elementsDetail.js +28 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/filter.js +86 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/stuckElements.js +38 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workRestrictions.js +153 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workflowSummary.js +28 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/rewrites.json +73 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/redirect.js +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/status.js +40 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/ElementSummaryByWorkflow.html +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/StuckElementSummary.html +26 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/TaskStatus.html +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/WorkflowSummary.html +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/workqueue-common-lib.html +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib-remote.html +16 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib.html +18 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/updates/in-place.js +50 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/validate_doc_update.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.couch.app.js +235 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.pathbinder.js +173 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/map.js +11 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/availableByPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/conflicts/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elements/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParent/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParentData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByPileupData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByStatus/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsBySubscription/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/reduce.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsDetailByWorkflowAndStatus/map.js +26 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/map.js +10 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/openRequests/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/recent-items/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/specsByWorkflow/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/stuckElements/map.js +38 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/map.js +12 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/reduce.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/map.js +9 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/reduce.js +10 -0
- wmglobalqueue-2.4.5.1.dist-info/METADATA +26 -0
- wmglobalqueue-2.4.5.1.dist-info/RECORD +347 -0
- wmglobalqueue-2.4.5.1.dist-info/WHEEL +5 -0
- wmglobalqueue-2.4.5.1.dist-info/licenses/LICENSE +202 -0
- wmglobalqueue-2.4.5.1.dist-info/licenses/NOTICE +16 -0
- wmglobalqueue-2.4.5.1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
from builtins import str, bytes
|
|
2
|
+
from future.utils import viewitems
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from WMCore.Services.Service import Service
|
|
8
|
+
from WMCore.Cache.GenericDataCache import MemoryCacheStruct
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ReqMgr(Service):
|
|
12
|
+
"""
|
|
13
|
+
API for dealing with retrieving information from RequestManager dataservice
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, url, header=None, logger=None):
|
|
18
|
+
"""
|
|
19
|
+
responseType will be either xml or json
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
httpDict = {}
|
|
23
|
+
header = header or {}
|
|
24
|
+
# url is end point
|
|
25
|
+
httpDict['endpoint'] = "%s/data" % url
|
|
26
|
+
httpDict['logger'] = logger if logger else logging.getLogger()
|
|
27
|
+
|
|
28
|
+
# cherrypy converts request.body to params when content type is set
|
|
29
|
+
# application/x-www-form-urlencodeds
|
|
30
|
+
httpDict.setdefault("content_type", 'application/json')
|
|
31
|
+
httpDict.setdefault('cacheduration', 0)
|
|
32
|
+
httpDict.setdefault("accept_type", "application/json")
|
|
33
|
+
httpDict.update(header)
|
|
34
|
+
self.encoder = json.dumps
|
|
35
|
+
Service.__init__(self, httpDict)
|
|
36
|
+
# This is only for the unittest: never set it true unless it is unittest
|
|
37
|
+
self._noStale = False
|
|
38
|
+
|
|
39
|
+
def _getResult(self, callname, clearCache=True, args=None, verb="GET",
|
|
40
|
+
encoder=json.loads, decoder=json.loads, contentType=None):
|
|
41
|
+
"""
|
|
42
|
+
_getResult_
|
|
43
|
+
"""
|
|
44
|
+
result = ''
|
|
45
|
+
cfile = callname.replace("/", "_")
|
|
46
|
+
if clearCache:
|
|
47
|
+
self.clearCache(cfile, args, verb)
|
|
48
|
+
|
|
49
|
+
f = self.refreshCache(cfile, callname, args, encoder=encoder,
|
|
50
|
+
verb=verb, contentType=contentType)
|
|
51
|
+
result = f.read()
|
|
52
|
+
f.close()
|
|
53
|
+
|
|
54
|
+
if result and decoder:
|
|
55
|
+
result = decoder(result)
|
|
56
|
+
|
|
57
|
+
return result['result']
|
|
58
|
+
|
|
59
|
+
def _createQuery(self, queryDict):
|
|
60
|
+
"""
|
|
61
|
+
_createQuery
|
|
62
|
+
:param queryDict: is the format of {name: values} fair. value can be sting, int or list
|
|
63
|
+
:type queryDict: dict
|
|
64
|
+
:returns: url query string
|
|
65
|
+
"""
|
|
66
|
+
if self._noStale:
|
|
67
|
+
args = "_nostale=true&"
|
|
68
|
+
else:
|
|
69
|
+
args = ""
|
|
70
|
+
for name, values in viewitems(queryDict):
|
|
71
|
+
if isinstance(values, (str, bytes, int)):
|
|
72
|
+
values = [values]
|
|
73
|
+
for val in values:
|
|
74
|
+
args += '%s=%s&' % (name, val)
|
|
75
|
+
|
|
76
|
+
return args.rstrip('&')
|
|
77
|
+
|
|
78
|
+
def getParentLocks(self):
|
|
79
|
+
"""
|
|
80
|
+
_getParentLocks_
|
|
81
|
+
|
|
82
|
+
A public method to return the parent locks from ReqMgr.
|
|
83
|
+
:returns: A list of datasets
|
|
84
|
+
|
|
85
|
+
"""
|
|
86
|
+
callname = 'parentlocks'
|
|
87
|
+
result = self._getResult(callname, clearCache=True, verb="GET")
|
|
88
|
+
return result[0][callname]
|
|
89
|
+
|
|
90
|
+
def getRequestByNames(self, names):
|
|
91
|
+
|
|
92
|
+
"""
|
|
93
|
+
_getRequestByNames_
|
|
94
|
+
|
|
95
|
+
:param names: list or sting of request name(s)
|
|
96
|
+
:type statusList: list, str
|
|
97
|
+
:returns: list of dict or list of request names depending on the detail value
|
|
98
|
+
-- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
|
|
99
|
+
'BlockWhitelist': [],
|
|
100
|
+
'CMSSWVersion': 'CMSSW_4_4_2_patch2',
|
|
101
|
+
....
|
|
102
|
+
'_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
|
|
103
|
+
'inputMode': 'couchDB'}}]
|
|
104
|
+
TODO: need proper error handling if status is not 200 from orignal reporting.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
query = self._createQuery({'name': names})
|
|
108
|
+
callname = 'request?%s' % query
|
|
109
|
+
return self._getResult(callname, verb="GET")
|
|
110
|
+
|
|
111
|
+
def getRequestByStatus(self, statusList, detail=True):
|
|
112
|
+
"""
|
|
113
|
+
_getRequestByStatus_
|
|
114
|
+
|
|
115
|
+
:param statusList: list of status
|
|
116
|
+
:type statusList: list
|
|
117
|
+
:param detail: boolean of request list.
|
|
118
|
+
:type detail: boolean
|
|
119
|
+
:returns: list of dict or list of request names depending on the detail value
|
|
120
|
+
-- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
|
|
121
|
+
'BlockWhitelist': [],
|
|
122
|
+
'CMSSWVersion': 'CMSSW_4_4_2_patch2',
|
|
123
|
+
....
|
|
124
|
+
'_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
|
|
125
|
+
'inputMode': 'couchDB'}}]
|
|
126
|
+
TODO: need proper error handling if status is not 200 from orignal reporting.
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
query = self._createQuery({'status': statusList, 'detail': detail})
|
|
130
|
+
callname = 'request?%s' % query
|
|
131
|
+
return self._getResult(callname, verb="GET")
|
|
132
|
+
|
|
133
|
+
def getRequestTasks(self, name):
|
|
134
|
+
|
|
135
|
+
"""
|
|
136
|
+
_getRequestTasks_
|
|
137
|
+
|
|
138
|
+
:param name: request name
|
|
139
|
+
:type string: str
|
|
140
|
+
:returns: list of dict or list of request names depending on the detail value
|
|
141
|
+
-- {result:[{requestTask}, {requestTask}]}
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
callname = 'splitting/%s' % name
|
|
145
|
+
return self._getResult(callname, verb="GET")
|
|
146
|
+
|
|
147
|
+
def getConfig(self, name):
|
|
148
|
+
|
|
149
|
+
"""
|
|
150
|
+
_getConfig_
|
|
151
|
+
|
|
152
|
+
:param name: request name
|
|
153
|
+
:type string: str
|
|
154
|
+
:returns: list of dict or list of request names depending on the detail value
|
|
155
|
+
-- {result:[config_string]}
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
callname = 'workload_config/%s' % name
|
|
159
|
+
return self._getResult(callname, verb="GET")
|
|
160
|
+
|
|
161
|
+
def insertRequests(self, requestDict):
|
|
162
|
+
"""
|
|
163
|
+
_insertRequests_
|
|
164
|
+
|
|
165
|
+
:param requestDict: request argument dictionary
|
|
166
|
+
:type requestDict: dict
|
|
167
|
+
:returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
|
|
168
|
+
'BlockWhitelist': [],
|
|
169
|
+
'CMSSWVersion': 'CMSSW_4_4_2_patch2',
|
|
170
|
+
....
|
|
171
|
+
'_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
|
|
172
|
+
'inputMode': 'couchDB'}}]
|
|
173
|
+
TODO: need proper error handling if status is not 200 from orignal reporting.
|
|
174
|
+
"""
|
|
175
|
+
return self["requests"].post('request', requestDict)[0]['result']
|
|
176
|
+
|
|
177
|
+
def updateRequestStatus(self, request, status):
|
|
178
|
+
"""
|
|
179
|
+
_updateRequestStatus_
|
|
180
|
+
|
|
181
|
+
:param request: request(workflow name)
|
|
182
|
+
:type reqeust: str
|
|
183
|
+
:param status: status of workflow to update (i.e. 'assigned')
|
|
184
|
+
:type status: str
|
|
185
|
+
:returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
|
|
186
|
+
'BlockWhitelist': [],
|
|
187
|
+
'CMSSWVersion': 'CMSSW_4_4_2_patch2',
|
|
188
|
+
....
|
|
189
|
+
'_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
|
|
190
|
+
'inputMode': 'couchDB'}}]
|
|
191
|
+
TODO: need proper error handling if status is not 200 from orignal reporting.
|
|
192
|
+
"""
|
|
193
|
+
|
|
194
|
+
status = {"RequestStatus": status}
|
|
195
|
+
status["RequestName"] = request
|
|
196
|
+
return self["requests"].put('request', status)[0]['result']
|
|
197
|
+
|
|
198
|
+
def updateRequestStats(self, request, stats):
|
|
199
|
+
"""
|
|
200
|
+
put initial stats for request
|
|
201
|
+
param: stats: dict of {'total_jobs': 100, 'input_lumis': 100,
|
|
202
|
+
'input_events': 100, 'input_num_files': 100}
|
|
203
|
+
specific to ReqMgr app. not implemented for T0
|
|
204
|
+
"""
|
|
205
|
+
# this stats dict will be validated on the server side
|
|
206
|
+
self.updateRequestProperty(request, stats)
|
|
207
|
+
|
|
208
|
+
def updateRequestProperty(self, request, propDict):
|
|
209
|
+
"""
|
|
210
|
+
_updateRequestProperty_
|
|
211
|
+
:param request: request(workflow name)
|
|
212
|
+
:type reqeust: str
|
|
213
|
+
:param propDict: request property with key value -- {"SiteWhitelist": ["ABC"], "SiteBlacklist": ["A"], "RequestStatus": "assigned"}
|
|
214
|
+
:type propDict: dict
|
|
215
|
+
:returns: list of dictionary -- [{'test_RequestString-OVERRIDE-ME_141125_142331_4966': {'BlockBlacklist': [],
|
|
216
|
+
'BlockWhitelist': [],
|
|
217
|
+
'CMSSWVersion': 'CMSSW_4_4_2_patch2',
|
|
218
|
+
....
|
|
219
|
+
'_id': 'test_RequestString-OVERRIDE-ME_141125_142331_4966',
|
|
220
|
+
'inputMode': 'couchDB'}}]
|
|
221
|
+
"""
|
|
222
|
+
propDict["RequestName"] = request
|
|
223
|
+
return self["requests"].put('request/%s' % request, propDict)[0]['result']
|
|
224
|
+
|
|
225
|
+
def getAbortedAndForceCompleteRequestsFromMemoryCache(self, expire=0):
|
|
226
|
+
"""
|
|
227
|
+
_getAbortedAndForceCompleteRequestsFromMemoryCache_
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
maskStates = ["aborted", "aborted-completed", "force-complete"]
|
|
231
|
+
return self.getRequestByStatusFromMemoryCache(maskStates, expire)
|
|
232
|
+
|
|
233
|
+
def getRequestByStatusFromMemoryCache(self, statusList, expire=0):
|
|
234
|
+
|
|
235
|
+
return MemoryCacheStruct(expire=expire, func=self.getRequestByStatus, initCacheValue=[],
|
|
236
|
+
logger=self['logger'], kwargs={'statusList': statusList, "detail": False})
|
|
237
|
+
|
|
238
|
+
def cloneRequest(self, requestName, overwrittenParams=None):
|
|
239
|
+
"""
|
|
240
|
+
_cloneRequest_
|
|
241
|
+
|
|
242
|
+
:param requestName: request name which need to be cloned
|
|
243
|
+
:type reqeust: str
|
|
244
|
+
:param overwrittenParams: status of workflow to update (i.e. 'assigned')
|
|
245
|
+
:type overwrittenParams: dict (dictionary format of parameters need to be overwritten)
|
|
246
|
+
:returns list of dict [{'request': cloned_request_name}]
|
|
247
|
+
"""
|
|
248
|
+
params = overwrittenParams or {}
|
|
249
|
+
uri = 'request/clone/%s' % requestName
|
|
250
|
+
return self["requests"].post(uri, params)[0]['result']
|
|
251
|
+
|
|
252
|
+
def getGenericRequestInfo(self, queryDict):
|
|
253
|
+
"""
|
|
254
|
+
_getGenericRequestInfo_
|
|
255
|
+
:param queryDict: query dictionary which will be converted to url query string.
|
|
256
|
+
:type queryDict: dict
|
|
257
|
+
:returns list of dict [{'request': { any info request by mask}]
|
|
258
|
+
"""
|
|
259
|
+
query = self._createQuery(queryDict)
|
|
260
|
+
callname = 'request?%s' % query
|
|
261
|
+
return self._getResult(callname, verb="GET")
|
|
File without changes
|
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
from __future__ import division, print_function
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from Utils.Utilities import diskUse
|
|
7
|
+
from WMCore.Cache.GenericDataCache import MemoryCacheStruct
|
|
8
|
+
from WMCore.Services.Service import Service
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ReqMgrAux(Service):
|
|
12
|
+
"""
|
|
13
|
+
API for dealing with retrieving information from RequestManager dataservice
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, url, httpDict=None, logger=None):
|
|
18
|
+
"""
|
|
19
|
+
responseType will be either xml or json
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
httpDict = httpDict or {}
|
|
23
|
+
# url is end point
|
|
24
|
+
httpDict['endpoint'] = "%s/data" % url
|
|
25
|
+
httpDict['logger'] = logger if logger else logging.getLogger()
|
|
26
|
+
|
|
27
|
+
# cherrypy converts request.body to params when content type is set
|
|
28
|
+
# application/x-www-form-urlencodeds
|
|
29
|
+
httpDict.setdefault("content_type", 'application/json')
|
|
30
|
+
httpDict.setdefault('cacheduration', 0)
|
|
31
|
+
self.cacheExpire = httpDict['cacheduration']
|
|
32
|
+
httpDict.setdefault("accept_type", "application/json")
|
|
33
|
+
self.encoder = json.dumps
|
|
34
|
+
Service.__init__(self, httpDict)
|
|
35
|
+
# This is only for the unittest: never set it true unless it is unittest
|
|
36
|
+
self._noStale = False
|
|
37
|
+
|
|
38
|
+
def _getResult(self, callname, clearCache=False, args=None, verb="GET",
|
|
39
|
+
encoder=json.loads, decoder=json.loads, contentType=None):
|
|
40
|
+
"""
|
|
41
|
+
_getResult_
|
|
42
|
+
"""
|
|
43
|
+
cfile = callname.replace("/", "_")
|
|
44
|
+
if clearCache:
|
|
45
|
+
self.clearCache(cfile, args, verb)
|
|
46
|
+
|
|
47
|
+
f = self.refreshCache(cfile, callname, args, encoder=encoder,
|
|
48
|
+
verb=verb, contentType=contentType)
|
|
49
|
+
result = f.read()
|
|
50
|
+
f.close()
|
|
51
|
+
|
|
52
|
+
if result and decoder:
|
|
53
|
+
result = decoder(result)
|
|
54
|
+
|
|
55
|
+
return result['result']
|
|
56
|
+
|
|
57
|
+
def _getDataFromMemoryCache(self, callname):
|
|
58
|
+
cache = MemoryCacheStruct(expire=self.cacheExpire, func=self._getResult, initCacheValue={},
|
|
59
|
+
logger=self['logger'], kwargs={'callname': callname, "verb": "GET"})
|
|
60
|
+
return cache.getData()
|
|
61
|
+
|
|
62
|
+
def getCMSSWVersion(self):
|
|
63
|
+
"""
|
|
64
|
+
get dictionary format of architecture and cmssw versions
|
|
65
|
+
i.e.
|
|
66
|
+
{"slc5_amd64_gcc462": [
|
|
67
|
+
"CMSSW_5_3_4_patch1",
|
|
68
|
+
"CMSSW_5_2_9",
|
|
69
|
+
"CMSSW_5_3_4_patch2",
|
|
70
|
+
"CMSSW_5_3_3_cand1_patch1"],
|
|
71
|
+
...
|
|
72
|
+
}
|
|
73
|
+
"""
|
|
74
|
+
return self._getDataFromMemoryCache('cmsswversions')
|
|
75
|
+
|
|
76
|
+
def _updateRecords(self, callName, resource, kwparams):
|
|
77
|
+
"""
|
|
78
|
+
Fetches the original document, locally update it according to
|
|
79
|
+
the key/value pairs provided and update the document.
|
|
80
|
+
:param callName: resource to be requested
|
|
81
|
+
:param resource: name of the resource/document to be updated
|
|
82
|
+
:param kwparams: a dictionary with the content to be updated
|
|
83
|
+
:return: a dictionary with the CouchDB response
|
|
84
|
+
"""
|
|
85
|
+
apiMap = {'wmagentconfig': self.getWMAgentConfig,
|
|
86
|
+
'campaignconfig': self.getCampaignConfig,
|
|
87
|
+
'transferinfo': self.getTransferInfo}
|
|
88
|
+
|
|
89
|
+
thisDoc = apiMap[callName](resource)
|
|
90
|
+
# getWMAgentConfig method returns directly the document, while the others
|
|
91
|
+
# return a list of document(s)
|
|
92
|
+
if isinstance(thisDoc, (list, set)):
|
|
93
|
+
thisDoc = thisDoc[0]
|
|
94
|
+
thisDoc.update(kwparams)
|
|
95
|
+
return self["requests"].put("%s/%s" % (callName, resource), thisDoc)[0]['result']
|
|
96
|
+
|
|
97
|
+
def populateCMSSWVersion(self, tcUrl, **kwargs):
|
|
98
|
+
"""
|
|
99
|
+
Query TagCollector and update the CMSSW versions document in Couch
|
|
100
|
+
:return: a boolean with the result of the operation
|
|
101
|
+
"""
|
|
102
|
+
from WMCore.Services.TagCollector.TagCollector import TagCollector
|
|
103
|
+
cmsswVersions = TagCollector(tcUrl, **kwargs).releases_by_architecture()
|
|
104
|
+
resp = self["requests"].put('cmsswversions', cmsswVersions)[0]['result']
|
|
105
|
+
|
|
106
|
+
if resp and resp[0].get("ok", False):
|
|
107
|
+
self["logger"].info("CMSSW document successfully updated.")
|
|
108
|
+
return True
|
|
109
|
+
|
|
110
|
+
msg = "Failed to update CMSSW document. Response: %s" % resp
|
|
111
|
+
self["logger"].warning(msg)
|
|
112
|
+
return False
|
|
113
|
+
|
|
114
|
+
def getWMAgentConfig(self, agentName):
|
|
115
|
+
"""
|
|
116
|
+
retrieve agent configuration reqmgr aux db.
|
|
117
|
+
"""
|
|
118
|
+
agentConfig = self._getDataFromMemoryCache('wmagentconfig/%s' % agentName)
|
|
119
|
+
|
|
120
|
+
if len(agentConfig) != 1:
|
|
121
|
+
# something wrong with database record. returns default value for both case.
|
|
122
|
+
self["logger"].warning("agent config is not correct: %s" % agentConfig)
|
|
123
|
+
return {}
|
|
124
|
+
|
|
125
|
+
return agentConfig[0]
|
|
126
|
+
|
|
127
|
+
def postWMAgentConfig(self, agentName, agentConfig):
|
|
128
|
+
"""
|
|
129
|
+
Create a new WMAgent configuration file in ReqMgrAux.
|
|
130
|
+
If document already exists, nothing happens.
|
|
131
|
+
"""
|
|
132
|
+
return self["requests"].post('wmagentconfig/%s' % agentName, agentConfig)[0]['result']
|
|
133
|
+
|
|
134
|
+
def updateWMAgentConfig(self, agentName, content, inPlace=False):
|
|
135
|
+
"""
|
|
136
|
+
Update the corresponding agent configuration with the content
|
|
137
|
+
provided, replacing the old document.
|
|
138
|
+
If inPlace is set to True, then a modification of the current
|
|
139
|
+
document is performed, according to the key/value pairs provided.
|
|
140
|
+
:param agentName: name of the agent/document in couch
|
|
141
|
+
:param content: a dictionary with the data to be updated
|
|
142
|
+
:param inPlace: a boolean defining whether to perform a replace
|
|
143
|
+
or a update modification
|
|
144
|
+
:return: a boolean with the result of the operation
|
|
145
|
+
"""
|
|
146
|
+
api = 'wmagentconfig'
|
|
147
|
+
if inPlace:
|
|
148
|
+
resp = self._updateRecords(api, agentName, content)
|
|
149
|
+
else:
|
|
150
|
+
resp = self["requests"].put("%s/%s" % (api, agentName), content)[0]['result']
|
|
151
|
+
|
|
152
|
+
if resp and resp[0].get("ok", False):
|
|
153
|
+
self["logger"].info("Update in-place: %s for agent: %s was successful.", inPlace, agentName)
|
|
154
|
+
return True
|
|
155
|
+
|
|
156
|
+
msg = "Failed to update agent: %s in-place: %s. Response: %s" % (agentName, inPlace, resp)
|
|
157
|
+
self["logger"].warning(msg)
|
|
158
|
+
return False
|
|
159
|
+
|
|
160
|
+
def getCampaignConfig(self, campaignName):
|
|
161
|
+
"""
|
|
162
|
+
get campaign config for transferor function in unified ReqMgr2MS.
|
|
163
|
+
"""
|
|
164
|
+
return self._getDataFromMemoryCache('campaignconfig/%s' % campaignName)
|
|
165
|
+
|
|
166
|
+
def postCampaignConfig(self, campaignName, campaignConfig):
|
|
167
|
+
"""
|
|
168
|
+
Create a new campaign configuration document
|
|
169
|
+
|
|
170
|
+
:param campaignName
|
|
171
|
+
:type basestringg
|
|
172
|
+
:param campaignConfig:
|
|
173
|
+
:type dict - only can replace whole campaign no partial parameters
|
|
174
|
+
|
|
175
|
+
campaignName: "HIRun2015":
|
|
176
|
+
campaignConfig: {
|
|
177
|
+
"go": true,
|
|
178
|
+
"labels" : ["02May2016","25Aug2016"],
|
|
179
|
+
"overflow" : {"PRIM" : {}},
|
|
180
|
+
"DDMcopies": {
|
|
181
|
+
"all" : { "N" : 2 }
|
|
182
|
+
},
|
|
183
|
+
"custodial_override" : ["DQMIO"],
|
|
184
|
+
"fractionpass": 1.0,
|
|
185
|
+
"lumisize" : -1,
|
|
186
|
+
"maxcopies" : 1,
|
|
187
|
+
"custodial": "T1_FR_CCIN2P3_MSS",
|
|
188
|
+
"parameters" :{
|
|
189
|
+
"NonCustodialSites" : ["T2_US_Vanderbilt"],
|
|
190
|
+
"SiteBlacklist": [
|
|
191
|
+
"T1_US_FNAL",
|
|
192
|
+
"T2_US_Purdue",
|
|
193
|
+
"T2_US_Caltech",
|
|
194
|
+
"T2_US_Florida",
|
|
195
|
+
"T2_US_Nebraska",
|
|
196
|
+
"T2_US_UCSD",
|
|
197
|
+
"T2_US_Wisconsin"
|
|
198
|
+
]
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
:return: CouchDB response dictionary
|
|
203
|
+
"""
|
|
204
|
+
return self["requests"].post('campaignconfig/%s' % campaignName, campaignConfig)[0]['result']
|
|
205
|
+
|
|
206
|
+
def updateCampaignConfig(self, campaignName, content, inPlace=False):
|
|
207
|
+
"""
|
|
208
|
+
Update the corresponding campaign configuration with the content
|
|
209
|
+
provided, replacing the old document.
|
|
210
|
+
If inPlace is set to True, then a modification of the current
|
|
211
|
+
document is performed, according to the key/value pairs provided.
|
|
212
|
+
:param campaignName: name of the campaign document in couch
|
|
213
|
+
:param content: a dictionary with the data to be updated
|
|
214
|
+
:param inPlace: a boolean defining whether to perform a replace
|
|
215
|
+
or a update modification
|
|
216
|
+
:return: a boolean with the result of the operation
|
|
217
|
+
"""
|
|
218
|
+
api = 'campaignconfig'
|
|
219
|
+
if inPlace:
|
|
220
|
+
resp = self._updateRecords(api, campaignName, content)
|
|
221
|
+
else:
|
|
222
|
+
resp = self["requests"].put("%s/%s" % (api, campaignName), content)[0]['result']
|
|
223
|
+
|
|
224
|
+
if resp and resp[0].get("ok", False):
|
|
225
|
+
self["logger"].info("Update in-place: %s for campaign: %s was successful.", inPlace, campaignName)
|
|
226
|
+
return True
|
|
227
|
+
msg = "Failed to update campaign: %s in-place: %s. Response: %s" % (campaignName, inPlace, resp)
|
|
228
|
+
self["logger"].warning(msg)
|
|
229
|
+
return False
|
|
230
|
+
|
|
231
|
+
def getUnifiedConfig(self, docName=None):
|
|
232
|
+
"""
|
|
233
|
+
Retrieve a Unified document type from the auxiliary db.
|
|
234
|
+
If docName is not provided, then fetch the default UNIFIED_CONFIG doc
|
|
235
|
+
"""
|
|
236
|
+
if docName:
|
|
237
|
+
unifiedConfig = self._getDataFromMemoryCache('unifiedconfig/%s' % docName)
|
|
238
|
+
else:
|
|
239
|
+
unifiedConfig = self._getDataFromMemoryCache('unifiedconfig')
|
|
240
|
+
|
|
241
|
+
if not unifiedConfig:
|
|
242
|
+
self["logger"].warning("Unified configuration document not found. Result: %s", unifiedConfig)
|
|
243
|
+
|
|
244
|
+
return unifiedConfig
|
|
245
|
+
|
|
246
|
+
def postUnifiedConfig(self, unifiedConfig, docName=None):
|
|
247
|
+
"""
|
|
248
|
+
Create a new unified configuration document
|
|
249
|
+
"""
|
|
250
|
+
if docName:
|
|
251
|
+
return self["requests"].post('unifiedconfig/%s' % docName, unifiedConfig)[0]['result']
|
|
252
|
+
return self["requests"].post('unifiedconfig', unifiedConfig)[0]['result']
|
|
253
|
+
|
|
254
|
+
def updateUnifiedConfig(self, content, docName=None):
|
|
255
|
+
"""
|
|
256
|
+
Update the unified configuration with the content provided, replacing
|
|
257
|
+
the old document.
|
|
258
|
+
:param content: a dictionary with the data to be updated
|
|
259
|
+
:return: a boolean with the result of the operation
|
|
260
|
+
"""
|
|
261
|
+
api = 'unifiedconfig'
|
|
262
|
+
if docName:
|
|
263
|
+
resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
|
|
264
|
+
else:
|
|
265
|
+
resp = self["requests"].put("%s" % api, content)[0]['result']
|
|
266
|
+
|
|
267
|
+
if resp and resp[0].get("ok", False):
|
|
268
|
+
self["logger"].info("Unified configuration successfully updated.")
|
|
269
|
+
return True
|
|
270
|
+
|
|
271
|
+
self["logger"].warning("Failed to update the unified configuration. Response: %s", resp)
|
|
272
|
+
return False
|
|
273
|
+
|
|
274
|
+
def getTransferInfo(self, docName):
|
|
275
|
+
"""
|
|
276
|
+
get a workflow transfer document, to be used by unified ReqMgr2MS.
|
|
277
|
+
"""
|
|
278
|
+
return self._getDataFromMemoryCache('transferinfo/%s' % docName)
|
|
279
|
+
|
|
280
|
+
def postTransferInfo(self, docName, transferInfo):
|
|
281
|
+
"""
|
|
282
|
+
Create a new workflow transfer document
|
|
283
|
+
|
|
284
|
+
:param docName: the name of the document to be created
|
|
285
|
+
:param transferInfo: a dictionary with the document content
|
|
286
|
+
:return: CouchDB response dictionary
|
|
287
|
+
"""
|
|
288
|
+
return self["requests"].post('transferinfo/%s' % docName, transferInfo)[0]['result']
|
|
289
|
+
|
|
290
|
+
def updateTransferInfo(self, docName, content, inPlace=False):
|
|
291
|
+
"""
|
|
292
|
+
Update the corresponding workflow transfer document with the content
|
|
293
|
+
provided, replacing the old one.
|
|
294
|
+
If inPlace is set to True, then a modification of the current
|
|
295
|
+
document is performed, according to the key/value pairs provided.
|
|
296
|
+
:param docName: name of the transfer document in couch
|
|
297
|
+
:param content: a dictionary with the data to be updated
|
|
298
|
+
:param inPlace: a boolean defining whether to perform a replace
|
|
299
|
+
or a update modification
|
|
300
|
+
:return: a boolean with the result of the operation
|
|
301
|
+
"""
|
|
302
|
+
api = 'transferinfo'
|
|
303
|
+
if inPlace:
|
|
304
|
+
resp = self._updateRecords(api, docName, content)
|
|
305
|
+
else:
|
|
306
|
+
resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
|
|
307
|
+
|
|
308
|
+
if resp and resp[0].get("ok", False):
|
|
309
|
+
self["logger"].info("Update in-place: %s for transfer doc: %s was successful.", inPlace, docName)
|
|
310
|
+
return True
|
|
311
|
+
msg = "Failed to update transfer doc: %s in-place: %s. Response: %s" % (docName, inPlace, resp)
|
|
312
|
+
self["logger"].warning(msg)
|
|
313
|
+
return False
|
|
314
|
+
|
|
315
|
+
def updateParentLocks(self, content, docName=None):
|
|
316
|
+
"""
|
|
317
|
+
Update the list of locked parent datasets with the content provided, replacing
|
|
318
|
+
the old document.
|
|
319
|
+
:param content: a dictionary with the data to be updated
|
|
320
|
+
:return: a boolean with the result of the operation
|
|
321
|
+
"""
|
|
322
|
+
api = 'parentlocks'
|
|
323
|
+
if docName:
|
|
324
|
+
resp = self["requests"].put("%s/%s" % (api, docName), content)[0]['result']
|
|
325
|
+
else:
|
|
326
|
+
resp = self["requests"].put("%s" % api, content)[0]['result']
|
|
327
|
+
|
|
328
|
+
if resp and resp[0].get("ok", False):
|
|
329
|
+
self["logger"].info("Parent dataset locks successfully updated.")
|
|
330
|
+
return True
|
|
331
|
+
|
|
332
|
+
self["logger"].warning("Failed to update parent dataset locks. Response: %s", resp)
|
|
333
|
+
return False
|
|
334
|
+
|
|
335
|
+
def getParentLocks(self):
|
|
336
|
+
"""
|
|
337
|
+
get the list of parent locks
|
|
338
|
+
"""
|
|
339
|
+
return self._getDataFromMemoryCache('parentlocks')
|
|
340
|
+
|
|
341
|
+
def deleteConfigDoc(self, docType, docName):
|
|
342
|
+
"""
|
|
343
|
+
Given a document type and a document name, delete it from the
|
|
344
|
+
Couch auxiliary DB.
|
|
345
|
+
:param docType: a string with the document type to be deleted, which
|
|
346
|
+
corresponds to the same API name
|
|
347
|
+
:param docName: the name of the document to be deleted
|
|
348
|
+
:return: response dictionary from the couch operation
|
|
349
|
+
"""
|
|
350
|
+
allowedValues = ('wmagentconfig', 'campaignconfig', 'unifiedconfig', 'transferinfo')
|
|
351
|
+
if docType not in allowedValues:
|
|
352
|
+
msg = "Document type: '%s' not allowed for deletion." % docType
|
|
353
|
+
msg += " Supported documents are: %s" % allowedValues
|
|
354
|
+
self["logger"].warning(msg)
|
|
355
|
+
else:
|
|
356
|
+
return self["requests"].delete('%s/%s' % (docType, docName))[0]['result']
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
AUXDB_AGENT_CONFIG_CACHE = {}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
# function to check whether agent is should be in draining status.
|
|
363
|
+
def isDrainMode(config):
|
|
364
|
+
"""
|
|
365
|
+
config is loaded from WMAgentConfig in local config
|
|
366
|
+
agentDrainMode is boolean value. (if it is passed update the WMAgentConfig in couchdb)
|
|
367
|
+
"""
|
|
368
|
+
if hasattr(config, "Tier0Feeder"):
|
|
369
|
+
return False
|
|
370
|
+
|
|
371
|
+
global AUXDB_AGENT_CONFIG_CACHE
|
|
372
|
+
|
|
373
|
+
reqMgrAux = ReqMgrAux(config.General.ReqMgr2ServiceURL)
|
|
374
|
+
agentConfig = reqMgrAux.getWMAgentConfig(config.Agent.hostName)
|
|
375
|
+
if "UserDrainMode" in agentConfig and "AgentDrainMode" in agentConfig:
|
|
376
|
+
AUXDB_AGENT_CONFIG_CACHE = agentConfig
|
|
377
|
+
return agentConfig["UserDrainMode"] or agentConfig["AgentDrainMode"]
|
|
378
|
+
# if the cache is empty this will raise Key not exist exception.
|
|
379
|
+
return AUXDB_AGENT_CONFIG_CACHE["UserDrainMode"] or AUXDB_AGENT_CONFIG_CACHE["AgentDrainMode"]
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def listDiskUsageOverThreshold(config, updateDB):
|
|
383
|
+
"""
|
|
384
|
+
check whether disk usage is over threshold,
|
|
385
|
+
return list of disk paths
|
|
386
|
+
if updateDB is True update the aux couch db value.
|
|
387
|
+
This function contains both check an update to avoide multiple db calls.
|
|
388
|
+
"""
|
|
389
|
+
defaultDiskThreshold = 85
|
|
390
|
+
defaultIgnoredDisks = []
|
|
391
|
+
if hasattr(config, "Tier0Feeder"):
|
|
392
|
+
# get the value from config.
|
|
393
|
+
ignoredDisks = getattr(config.AgentStatusWatcher, "ignoreDisks", defaultIgnoredDisks)
|
|
394
|
+
diskUseThreshold = getattr(config.AgentStatusWatcher, "diskUseThreshold", defaultDiskThreshold)
|
|
395
|
+
t0Flag = True
|
|
396
|
+
else:
|
|
397
|
+
reqMgrAux = ReqMgrAux(config.General.ReqMgr2ServiceURL)
|
|
398
|
+
agentConfig = reqMgrAux.getWMAgentConfig(config.Agent.hostName)
|
|
399
|
+
diskUseThreshold = agentConfig.get("DiskUseThreshold", defaultDiskThreshold)
|
|
400
|
+
ignoredDisks = agentConfig.get("IgnoreDisks", defaultIgnoredDisks)
|
|
401
|
+
t0Flag = False
|
|
402
|
+
|
|
403
|
+
# Disk space warning
|
|
404
|
+
diskUseList = diskUse()
|
|
405
|
+
overThresholdDisks = []
|
|
406
|
+
for disk in diskUseList:
|
|
407
|
+
if (float(disk['percent'].strip('%')) >= diskUseThreshold and
|
|
408
|
+
disk['mounted'] not in ignoredDisks):
|
|
409
|
+
overThresholdDisks.append(disk)
|
|
410
|
+
|
|
411
|
+
if updateDB and not t0Flag:
|
|
412
|
+
agentDrainMode = bool(len(overThresholdDisks))
|
|
413
|
+
if agentConfig and (agentDrainMode != agentConfig.get("AgentDrainMode")):
|
|
414
|
+
if agentConfig.get("UserDrainMode") is True:
|
|
415
|
+
logging.info("Agent is already in UserDrainMode: True mode. No need to update.")
|
|
416
|
+
else:
|
|
417
|
+
reqMgrAux.updateWMAgentConfig(config.Agent.hostName, {"AgentDrainMode": agentDrainMode}, inPlace=True)
|
|
418
|
+
|
|
419
|
+
return overThresholdDisks
|
|
File without changes
|