wmglobalqueue 2.4.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Utils/CPMetrics.py +270 -0
- Utils/CertTools.py +100 -0
- Utils/EmailAlert.py +50 -0
- Utils/ExtendedUnitTestCase.py +62 -0
- Utils/FileTools.py +182 -0
- Utils/IteratorTools.py +80 -0
- Utils/MathUtils.py +31 -0
- Utils/MemoryCache.py +119 -0
- Utils/Patterns.py +24 -0
- Utils/Pipeline.py +137 -0
- Utils/PortForward.py +97 -0
- Utils/ProcFS.py +112 -0
- Utils/ProcessStats.py +194 -0
- Utils/PythonVersion.py +17 -0
- Utils/Signals.py +36 -0
- Utils/TemporaryEnvironment.py +27 -0
- Utils/Throttled.py +227 -0
- Utils/Timers.py +130 -0
- Utils/Timestamps.py +86 -0
- Utils/TokenManager.py +143 -0
- Utils/Tracing.py +60 -0
- Utils/TwPrint.py +98 -0
- Utils/Utilities.py +318 -0
- Utils/__init__.py +11 -0
- Utils/wmcoreDTools.py +707 -0
- WMCore/ACDC/Collection.py +57 -0
- WMCore/ACDC/CollectionTypes.py +12 -0
- WMCore/ACDC/CouchCollection.py +67 -0
- WMCore/ACDC/CouchFileset.py +238 -0
- WMCore/ACDC/CouchService.py +73 -0
- WMCore/ACDC/DataCollectionService.py +485 -0
- WMCore/ACDC/Fileset.py +94 -0
- WMCore/ACDC/__init__.py +11 -0
- WMCore/Algorithms/Alarm.py +39 -0
- WMCore/Algorithms/MathAlgos.py +274 -0
- WMCore/Algorithms/MiscAlgos.py +67 -0
- WMCore/Algorithms/ParseXMLFile.py +115 -0
- WMCore/Algorithms/Permissions.py +27 -0
- WMCore/Algorithms/Singleton.py +58 -0
- WMCore/Algorithms/SubprocessAlgos.py +129 -0
- WMCore/Algorithms/__init__.py +7 -0
- WMCore/Cache/GenericDataCache.py +98 -0
- WMCore/Cache/WMConfigCache.py +572 -0
- WMCore/Cache/__init__.py +0 -0
- WMCore/Configuration.py +659 -0
- WMCore/DAOFactory.py +47 -0
- WMCore/DataStructs/File.py +177 -0
- WMCore/DataStructs/Fileset.py +140 -0
- WMCore/DataStructs/Job.py +182 -0
- WMCore/DataStructs/JobGroup.py +142 -0
- WMCore/DataStructs/JobPackage.py +49 -0
- WMCore/DataStructs/LumiList.py +734 -0
- WMCore/DataStructs/Mask.py +219 -0
- WMCore/DataStructs/MathStructs/ContinuousSummaryHistogram.py +197 -0
- WMCore/DataStructs/MathStructs/DiscreteSummaryHistogram.py +92 -0
- WMCore/DataStructs/MathStructs/SummaryHistogram.py +117 -0
- WMCore/DataStructs/MathStructs/__init__.py +0 -0
- WMCore/DataStructs/Pickleable.py +24 -0
- WMCore/DataStructs/Run.py +256 -0
- WMCore/DataStructs/Subscription.py +175 -0
- WMCore/DataStructs/WMObject.py +47 -0
- WMCore/DataStructs/WorkUnit.py +112 -0
- WMCore/DataStructs/Workflow.py +60 -0
- WMCore/DataStructs/__init__.py +8 -0
- WMCore/Database/CMSCouch.py +1430 -0
- WMCore/Database/ConfigDBMap.py +29 -0
- WMCore/Database/CouchMonitoring.py +450 -0
- WMCore/Database/CouchUtils.py +118 -0
- WMCore/Database/DBCore.py +198 -0
- WMCore/Database/DBCreator.py +113 -0
- WMCore/Database/DBExceptionHandler.py +59 -0
- WMCore/Database/DBFactory.py +117 -0
- WMCore/Database/DBFormatter.py +177 -0
- WMCore/Database/Dialects.py +13 -0
- WMCore/Database/ExecuteDAO.py +327 -0
- WMCore/Database/MongoDB.py +241 -0
- WMCore/Database/MySQL/Destroy.py +42 -0
- WMCore/Database/MySQL/ListUserContent.py +20 -0
- WMCore/Database/MySQL/__init__.py +9 -0
- WMCore/Database/MySQLCore.py +132 -0
- WMCore/Database/Oracle/Destroy.py +56 -0
- WMCore/Database/Oracle/ListUserContent.py +19 -0
- WMCore/Database/Oracle/__init__.py +9 -0
- WMCore/Database/ResultSet.py +44 -0
- WMCore/Database/Transaction.py +91 -0
- WMCore/Database/__init__.py +9 -0
- WMCore/Database/ipy_profile_couch.py +438 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/CleanUpTask.py +29 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/HeartbeatMonitor.py +105 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/LocationUpdateTask.py +28 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/ReqMgrInteractionTask.py +35 -0
- WMCore/GlobalWorkQueue/CherryPyThreads/__init__.py +0 -0
- WMCore/GlobalWorkQueue/__init__.py +0 -0
- WMCore/GroupUser/CouchObject.py +127 -0
- WMCore/GroupUser/Decorators.py +51 -0
- WMCore/GroupUser/Group.py +33 -0
- WMCore/GroupUser/Interface.py +73 -0
- WMCore/GroupUser/User.py +96 -0
- WMCore/GroupUser/__init__.py +11 -0
- WMCore/Lexicon.py +836 -0
- WMCore/REST/Auth.py +202 -0
- WMCore/REST/CherryPyPeriodicTask.py +166 -0
- WMCore/REST/Error.py +333 -0
- WMCore/REST/Format.py +642 -0
- WMCore/REST/HeartbeatMonitorBase.py +90 -0
- WMCore/REST/Main.py +636 -0
- WMCore/REST/Server.py +2435 -0
- WMCore/REST/Services.py +24 -0
- WMCore/REST/Test.py +120 -0
- WMCore/REST/Tools.py +38 -0
- WMCore/REST/Validation.py +250 -0
- WMCore/REST/__init__.py +1 -0
- WMCore/ReqMgr/DataStructs/RequestStatus.py +209 -0
- WMCore/ReqMgr/DataStructs/RequestType.py +13 -0
- WMCore/ReqMgr/DataStructs/__init__.py +0 -0
- WMCore/ReqMgr/__init__.py +1 -0
- WMCore/Services/AlertManager/AlertManagerAPI.py +111 -0
- WMCore/Services/AlertManager/__init__.py +0 -0
- WMCore/Services/CRIC/CRIC.py +238 -0
- WMCore/Services/CRIC/__init__.py +0 -0
- WMCore/Services/DBS/DBS3Reader.py +1044 -0
- WMCore/Services/DBS/DBSConcurrency.py +44 -0
- WMCore/Services/DBS/DBSErrors.py +112 -0
- WMCore/Services/DBS/DBSReader.py +23 -0
- WMCore/Services/DBS/DBSUtils.py +166 -0
- WMCore/Services/DBS/DBSWriterObjects.py +381 -0
- WMCore/Services/DBS/ProdException.py +133 -0
- WMCore/Services/DBS/__init__.py +8 -0
- WMCore/Services/FWJRDB/FWJRDBAPI.py +118 -0
- WMCore/Services/FWJRDB/__init__.py +0 -0
- WMCore/Services/HTTPS/HTTPSAuthHandler.py +66 -0
- WMCore/Services/HTTPS/__init__.py +0 -0
- WMCore/Services/LogDB/LogDB.py +201 -0
- WMCore/Services/LogDB/LogDBBackend.py +191 -0
- WMCore/Services/LogDB/LogDBExceptions.py +11 -0
- WMCore/Services/LogDB/LogDBReport.py +85 -0
- WMCore/Services/LogDB/__init__.py +0 -0
- WMCore/Services/MSPileup/__init__.py +0 -0
- WMCore/Services/MSUtils/MSUtils.py +54 -0
- WMCore/Services/MSUtils/__init__.py +0 -0
- WMCore/Services/McM/McM.py +173 -0
- WMCore/Services/McM/__init__.py +8 -0
- WMCore/Services/MonIT/Grafana.py +133 -0
- WMCore/Services/MonIT/__init__.py +0 -0
- WMCore/Services/PyCondor/PyCondorAPI.py +154 -0
- WMCore/Services/PyCondor/__init__.py +0 -0
- WMCore/Services/ReqMgr/ReqMgr.py +261 -0
- WMCore/Services/ReqMgr/__init__.py +0 -0
- WMCore/Services/ReqMgrAux/ReqMgrAux.py +419 -0
- WMCore/Services/ReqMgrAux/__init__.py +0 -0
- WMCore/Services/RequestDB/RequestDBReader.py +267 -0
- WMCore/Services/RequestDB/RequestDBWriter.py +39 -0
- WMCore/Services/RequestDB/__init__.py +0 -0
- WMCore/Services/Requests.py +624 -0
- WMCore/Services/Rucio/Rucio.py +1290 -0
- WMCore/Services/Rucio/RucioUtils.py +74 -0
- WMCore/Services/Rucio/__init__.py +0 -0
- WMCore/Services/RucioConMon/RucioConMon.py +121 -0
- WMCore/Services/RucioConMon/__init__.py +0 -0
- WMCore/Services/Service.py +400 -0
- WMCore/Services/StompAMQ/__init__.py +0 -0
- WMCore/Services/TagCollector/TagCollector.py +155 -0
- WMCore/Services/TagCollector/XMLUtils.py +98 -0
- WMCore/Services/TagCollector/__init__.py +0 -0
- WMCore/Services/UUIDLib.py +13 -0
- WMCore/Services/UserFileCache/UserFileCache.py +160 -0
- WMCore/Services/UserFileCache/__init__.py +8 -0
- WMCore/Services/WMAgent/WMAgent.py +63 -0
- WMCore/Services/WMAgent/__init__.py +0 -0
- WMCore/Services/WMArchive/CMSSWMetrics.py +526 -0
- WMCore/Services/WMArchive/DataMap.py +463 -0
- WMCore/Services/WMArchive/WMArchive.py +33 -0
- WMCore/Services/WMArchive/__init__.py +0 -0
- WMCore/Services/WMBS/WMBS.py +97 -0
- WMCore/Services/WMBS/__init__.py +0 -0
- WMCore/Services/WMStats/DataStruct/RequestInfoCollection.py +300 -0
- WMCore/Services/WMStats/DataStruct/__init__.py +0 -0
- WMCore/Services/WMStats/WMStatsPycurl.py +145 -0
- WMCore/Services/WMStats/WMStatsReader.py +445 -0
- WMCore/Services/WMStats/WMStatsWriter.py +273 -0
- WMCore/Services/WMStats/__init__.py +0 -0
- WMCore/Services/WMStatsServer/WMStatsServer.py +134 -0
- WMCore/Services/WMStatsServer/__init__.py +0 -0
- WMCore/Services/WorkQueue/WorkQueue.py +492 -0
- WMCore/Services/WorkQueue/__init__.py +0 -0
- WMCore/Services/__init__.py +8 -0
- WMCore/Services/pycurl_manager.py +574 -0
- WMCore/WMBase.py +50 -0
- WMCore/WMConnectionBase.py +164 -0
- WMCore/WMException.py +183 -0
- WMCore/WMExceptions.py +269 -0
- WMCore/WMFactory.py +76 -0
- WMCore/WMInit.py +377 -0
- WMCore/WMLogging.py +104 -0
- WMCore/WMSpec/ConfigSectionTree.py +442 -0
- WMCore/WMSpec/Persistency.py +135 -0
- WMCore/WMSpec/Steps/BuildMaster.py +87 -0
- WMCore/WMSpec/Steps/BuildTools.py +201 -0
- WMCore/WMSpec/Steps/Builder.py +97 -0
- WMCore/WMSpec/Steps/Diagnostic.py +89 -0
- WMCore/WMSpec/Steps/Emulator.py +62 -0
- WMCore/WMSpec/Steps/ExecuteMaster.py +208 -0
- WMCore/WMSpec/Steps/Executor.py +210 -0
- WMCore/WMSpec/Steps/StepFactory.py +213 -0
- WMCore/WMSpec/Steps/TaskEmulator.py +75 -0
- WMCore/WMSpec/Steps/Template.py +204 -0
- WMCore/WMSpec/Steps/Templates/AlcaHarvest.py +76 -0
- WMCore/WMSpec/Steps/Templates/CMSSW.py +613 -0
- WMCore/WMSpec/Steps/Templates/DQMUpload.py +59 -0
- WMCore/WMSpec/Steps/Templates/DeleteFiles.py +70 -0
- WMCore/WMSpec/Steps/Templates/LogArchive.py +84 -0
- WMCore/WMSpec/Steps/Templates/LogCollect.py +105 -0
- WMCore/WMSpec/Steps/Templates/StageOut.py +105 -0
- WMCore/WMSpec/Steps/Templates/__init__.py +10 -0
- WMCore/WMSpec/Steps/WMExecutionFailure.py +21 -0
- WMCore/WMSpec/Steps/__init__.py +8 -0
- WMCore/WMSpec/Utilities.py +63 -0
- WMCore/WMSpec/WMSpecErrors.py +12 -0
- WMCore/WMSpec/WMStep.py +347 -0
- WMCore/WMSpec/WMTask.py +1997 -0
- WMCore/WMSpec/WMWorkload.py +2288 -0
- WMCore/WMSpec/WMWorkloadTools.py +382 -0
- WMCore/WMSpec/__init__.py +9 -0
- WMCore/WorkQueue/DataLocationMapper.py +273 -0
- WMCore/WorkQueue/DataStructs/ACDCBlock.py +47 -0
- WMCore/WorkQueue/DataStructs/Block.py +48 -0
- WMCore/WorkQueue/DataStructs/CouchWorkQueueElement.py +148 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElement.py +274 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementResult.py +152 -0
- WMCore/WorkQueue/DataStructs/WorkQueueElementsSummary.py +185 -0
- WMCore/WorkQueue/DataStructs/__init__.py +0 -0
- WMCore/WorkQueue/Policy/End/EndPolicyInterface.py +44 -0
- WMCore/WorkQueue/Policy/End/SingleShot.py +22 -0
- WMCore/WorkQueue/Policy/End/__init__.py +32 -0
- WMCore/WorkQueue/Policy/PolicyInterface.py +17 -0
- WMCore/WorkQueue/Policy/Start/Block.py +258 -0
- WMCore/WorkQueue/Policy/Start/Dataset.py +180 -0
- WMCore/WorkQueue/Policy/Start/MonteCarlo.py +131 -0
- WMCore/WorkQueue/Policy/Start/ResubmitBlock.py +171 -0
- WMCore/WorkQueue/Policy/Start/StartPolicyInterface.py +316 -0
- WMCore/WorkQueue/Policy/Start/__init__.py +34 -0
- WMCore/WorkQueue/Policy/__init__.py +57 -0
- WMCore/WorkQueue/WMBSHelper.py +772 -0
- WMCore/WorkQueue/WorkQueue.py +1237 -0
- WMCore/WorkQueue/WorkQueueBackend.py +750 -0
- WMCore/WorkQueue/WorkQueueBase.py +39 -0
- WMCore/WorkQueue/WorkQueueExceptions.py +44 -0
- WMCore/WorkQueue/WorkQueueReqMgrInterface.py +278 -0
- WMCore/WorkQueue/WorkQueueUtils.py +130 -0
- WMCore/WorkQueue/__init__.py +13 -0
- WMCore/Wrappers/JsonWrapper/JSONThunker.py +342 -0
- WMCore/Wrappers/JsonWrapper/__init__.py +7 -0
- WMCore/Wrappers/__init__.py +6 -0
- WMCore/__init__.py +10 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-patch +15 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-dist-unpatch +8 -0
- wmglobalqueue-2.4.5.1.data/data/bin/wmc-httpd +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/.couchapprc +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/README.md +40 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/index.html +264 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/ElementInfoByWorkflow.js +96 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/StuckElementInfo.js +57 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/WorkloadInfoTable.js +80 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/dataTable.js +70 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/js/namespace.js +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/_attachments/style/main.css +75 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/couchapp.json +4 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/childQueueFilter.js +13 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/filterDeletedDocs.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/filters/queueFilter.js +11 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/language +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/mustache.js +333 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/validate.js +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lib/workqueue_utils.js +61 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/elementsDetail.js +28 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/filter.js +86 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/stuckElements.js +38 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workRestrictions.js +153 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/lists/workflowSummary.js +28 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/rewrites.json +73 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/redirect.js +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/shows/status.js +40 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/ElementSummaryByWorkflow.html +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/StuckElementSummary.html +26 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/TaskStatus.html +23 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/WorkflowSummary.html +27 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/workqueue-common-lib.html +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib-remote.html +16 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/templates/partials/yui-lib.html +18 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/updates/in-place.js +50 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/validate_doc_update.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.couch.app.js +235 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/vendor/couchapp/_attachments/jquery.pathbinder.js +173 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activeParentData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/activePileupData/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/map.js +11 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/analyticsData/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/availableByPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/conflicts/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elements/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParent/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByParentData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByPileupData/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByStatus/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsBySubscription/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/map.js +8 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsByWorkflow/reduce.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/elementsDetailByWorkflowAndStatus/map.js +26 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/map.js +10 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobInjectStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobStatusByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndPriority/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByChildQueueAndStatus/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatus/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/jobsByStatusAndPriority/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/openRequests/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/recent-items/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/siteWhitelistByRequest/reduce.js +1 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/specsByWorkflow/map.js +5 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/stuckElements/map.js +38 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/map.js +12 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsInjectStatusByRequest/reduce.js +3 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrl/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/map.js +6 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/wmbsUrlByRequest/reduce.js +2 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/map.js +9 -0
- wmglobalqueue-2.4.5.1.data/data/data/couchapps/WorkQueue/views/workflowSummary/reduce.js +10 -0
- wmglobalqueue-2.4.5.1.dist-info/METADATA +26 -0
- wmglobalqueue-2.4.5.1.dist-info/RECORD +347 -0
- wmglobalqueue-2.4.5.1.dist-info/WHEEL +5 -0
- wmglobalqueue-2.4.5.1.dist-info/licenses/LICENSE +202 -0
- wmglobalqueue-2.4.5.1.dist-info/licenses/NOTICE +16 -0
- wmglobalqueue-2.4.5.1.dist-info/top_level.txt +2 -0
WMCore/REST/Services.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import print_function, division
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from Utils.ProcessStats import processStatusDict
|
|
5
|
+
from WMCore.REST.Format import JSONFormat, PrettyJSONFormat
|
|
6
|
+
from WMCore.REST.Server import RESTEntity, restcall, rows
|
|
7
|
+
|
|
8
|
+
class ProcessMatrix(RESTEntity):
|
|
9
|
+
"""
|
|
10
|
+
Process and thread matrix in reqmgr2
|
|
11
|
+
"""
|
|
12
|
+
def __init__(self, app, api, config, mount):
|
|
13
|
+
# CouchDB auxiliary database name
|
|
14
|
+
RESTEntity.__init__(self, app, api, config, mount)
|
|
15
|
+
self.time0 = time.time()
|
|
16
|
+
|
|
17
|
+
def validate(self, apiobj, method, api, param, safe):
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
@restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())])
|
|
21
|
+
def get(self):
|
|
22
|
+
sdict = {'server': processStatusDict()}
|
|
23
|
+
sdict['server'].update({'uptime': time.time() - self.time0})
|
|
24
|
+
return rows([sdict])
|
WMCore/REST/Test.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
|
|
2
|
+
from Utils.Utilities import encodeUnicodeToBytes
|
|
3
|
+
from future.utils import viewitems, viewvalues, listitems
|
|
4
|
+
|
|
5
|
+
import os, hmac, hashlib, cherrypy
|
|
6
|
+
from tempfile import NamedTemporaryFile
|
|
7
|
+
|
|
8
|
+
from Utils.PythonVersion import PY3
|
|
9
|
+
|
|
10
|
+
from WMCore.REST.Main import RESTMain
|
|
11
|
+
from WMCore.REST.Auth import authz_canonical
|
|
12
|
+
from WMCore.Configuration import Configuration
|
|
13
|
+
|
|
14
|
+
def fake_authz_headers(hmac_key, method = 'HNLogin',
|
|
15
|
+
login='testuser', name='Test User',
|
|
16
|
+
dn="/test/dn", roles={}, format="list"):
|
|
17
|
+
"""Create fake authentication and authorisation headers compatible
|
|
18
|
+
with the CMSWEB front-ends. Assumes you have the HMAC signing key
|
|
19
|
+
the back-end will use to validate the headers.
|
|
20
|
+
|
|
21
|
+
:arg str hmac_key: binary key data for signing headers.
|
|
22
|
+
:arg str method: authentication method, one of X509Cert, X509Proxy,
|
|
23
|
+
HNLogin, HostIP, AUCookie or None.
|
|
24
|
+
:arg str login: account login name.
|
|
25
|
+
:arg str name: account user name.
|
|
26
|
+
:arg str dn: account X509 subject.
|
|
27
|
+
:arg dict roles: role dictionary, each role with 'site' and 'group' lists.
|
|
28
|
+
:returns: list of header name, value tuples to add to a HTTP request."""
|
|
29
|
+
headers = { 'cms-auth-status': 'OK', 'cms-authn-method': method }
|
|
30
|
+
|
|
31
|
+
if login:
|
|
32
|
+
headers['cms-authn-login'] = login
|
|
33
|
+
|
|
34
|
+
if name:
|
|
35
|
+
headers['cms-authn-name'] = name
|
|
36
|
+
|
|
37
|
+
if dn:
|
|
38
|
+
headers['cms-authn-dn'] = dn
|
|
39
|
+
|
|
40
|
+
for name, role in viewitems(roles):
|
|
41
|
+
name = 'cms-authz-' + authz_canonical(name)
|
|
42
|
+
headers[name] = []
|
|
43
|
+
for r in 'site', 'group':
|
|
44
|
+
if r in role:
|
|
45
|
+
headers[name].extend(["%s:%s" % (r, authz_canonical(v)) for v in role[r]])
|
|
46
|
+
headers[name] = " ".join(headers[name])
|
|
47
|
+
|
|
48
|
+
prefix = suffix = ""
|
|
49
|
+
hkeys = list(headers)
|
|
50
|
+
for hk in sorted(hkeys):
|
|
51
|
+
if hk != 'cms-auth-status':
|
|
52
|
+
prefix += "h%xv%x" % (len(hk), len(headers[hk]))
|
|
53
|
+
suffix += "%s%s" % (hk, headers[hk])
|
|
54
|
+
|
|
55
|
+
msg = prefix + "#" + suffix
|
|
56
|
+
if PY3:
|
|
57
|
+
hmac_key = encodeUnicodeToBytes(hmac_key)
|
|
58
|
+
msg = encodeUnicodeToBytes(msg)
|
|
59
|
+
cksum = hmac.new(hmac_key, msg, hashlib.sha1).hexdigest()
|
|
60
|
+
headers['cms-authn-hmac'] = cksum
|
|
61
|
+
if format == "list":
|
|
62
|
+
return listitems(headers)
|
|
63
|
+
else:
|
|
64
|
+
return headers
|
|
65
|
+
|
|
66
|
+
def fake_authz_key_file(delete=True):
|
|
67
|
+
"""Create temporary file for fake authorisation hmac signing key.
|
|
68
|
+
|
|
69
|
+
:returns: Instance of :class:`~.NamedTemporaryFile`, whose *data*
|
|
70
|
+
attribute contains the HMAC signing binary key."""
|
|
71
|
+
t = NamedTemporaryFile(delete=delete)
|
|
72
|
+
with open("/dev/urandom", "rb") as fd:
|
|
73
|
+
t.data = fd.read(20)
|
|
74
|
+
t.write(t.data)
|
|
75
|
+
t.seek(0)
|
|
76
|
+
return t
|
|
77
|
+
|
|
78
|
+
def setup_dummy_server(module_name, class_name, app_name = None, authz_key_file=None, port=8888):
|
|
79
|
+
"""Helper function to set up a :class:`~.RESTMain` server from given
|
|
80
|
+
module and class. Creates a fake server configuration and instantiates
|
|
81
|
+
the server application from it.
|
|
82
|
+
|
|
83
|
+
:arg str module_name: module from which to import test class.
|
|
84
|
+
:arg str class_type: name of the server test class.
|
|
85
|
+
:arg str app_name: optional test application name, 'test' by default.
|
|
86
|
+
:returns: tuple with the server object and authz hmac signing key."""
|
|
87
|
+
if authz_key_file:
|
|
88
|
+
test_authz_key = authz_key_file
|
|
89
|
+
else:
|
|
90
|
+
test_authz_key = fake_authz_key_file()
|
|
91
|
+
|
|
92
|
+
cfg = Configuration()
|
|
93
|
+
main = cfg.section_('main')
|
|
94
|
+
main.application = app_name or 'test'
|
|
95
|
+
main.silent = True
|
|
96
|
+
main.index = 'top'
|
|
97
|
+
main.authz_defaults = { 'role': None, 'group': None, 'site': None }
|
|
98
|
+
main.section_('tools').section_('cms_auth').key_file = test_authz_key.name
|
|
99
|
+
|
|
100
|
+
app = cfg.section_(app_name or 'test')
|
|
101
|
+
app.admin = 'dada@example.org'
|
|
102
|
+
app.description = app.title = 'Test'
|
|
103
|
+
|
|
104
|
+
views = cfg.section_('views')
|
|
105
|
+
top = views.section_('top')
|
|
106
|
+
top.object = module_name + "." + class_name
|
|
107
|
+
|
|
108
|
+
server = RESTMain(cfg, os.getcwd())
|
|
109
|
+
server.validate_config()
|
|
110
|
+
server.setup_server()
|
|
111
|
+
server.install_application()
|
|
112
|
+
cherrypy.config.update({'server.socket_port': port})
|
|
113
|
+
cherrypy.config.update({'server.socket_host': '127.0.0.1'})
|
|
114
|
+
cherrypy.config.update({'request.show_tracebacks': True})
|
|
115
|
+
cherrypy.config.update({'environment': 'test_suite'})
|
|
116
|
+
for app in viewvalues(cherrypy.tree.apps):
|
|
117
|
+
if '/' in app.config:
|
|
118
|
+
app.config["/"]["request.show_tracebacks"] = True
|
|
119
|
+
|
|
120
|
+
return server, test_authz_key
|
WMCore/REST/Tools.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
import cherrypy
|
|
4
|
+
from cherrypy import Tool, request, tools
|
|
5
|
+
|
|
6
|
+
from WMCore.REST.Auth import RESTAuth
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def set_request_time():
|
|
10
|
+
"""Utility to time stamp the start of HTTP request handling."""
|
|
11
|
+
request.start_time = time.time()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def set_proxy_base(base=None):
|
|
15
|
+
"""Utility to correctly handle requests behind a proxy."""
|
|
16
|
+
scheme = request.headers.get('X-Forwarded-Proto', request.base[:request.base.find("://")])
|
|
17
|
+
base = request.headers.get('X-Forwarded-Host', base)
|
|
18
|
+
if not base:
|
|
19
|
+
port = cherrypy.request.local.port
|
|
20
|
+
if port == 80:
|
|
21
|
+
base = 'localhost'
|
|
22
|
+
else:
|
|
23
|
+
base = 'localhost:%s' % port
|
|
24
|
+
|
|
25
|
+
base = base.split(',')[0].strip()
|
|
26
|
+
if base.find("://") == -1:
|
|
27
|
+
base = scheme + "://" + base
|
|
28
|
+
request.base = base
|
|
29
|
+
|
|
30
|
+
xff = request.headers.get('X-Forwarded-For')
|
|
31
|
+
if xff:
|
|
32
|
+
xff = xff.split(',')[0].strip()
|
|
33
|
+
request.remote.ip = xff
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
tools.cms_auth = RESTAuth()
|
|
37
|
+
tools.time = Tool('on_start_resource', set_request_time)
|
|
38
|
+
tools.proxy = Tool('before_request_body', set_proxy_base, priority=30)
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
from builtins import str as newstr, bytes as newbytes
|
|
2
|
+
|
|
3
|
+
from WMCore.REST.Error import *
|
|
4
|
+
import math
|
|
5
|
+
import re
|
|
6
|
+
import numbers
|
|
7
|
+
|
|
8
|
+
from Utils.Utilities import decodeBytesToUnicodeConditional, encodeUnicodeToBytesConditional
|
|
9
|
+
from Utils.PythonVersion import PY3, PY2
|
|
10
|
+
|
|
11
|
+
def return_message(main_err, custom_err):
|
|
12
|
+
if custom_err:
|
|
13
|
+
return custom_err
|
|
14
|
+
return main_err
|
|
15
|
+
|
|
16
|
+
def _arglist(argname, kwargs):
|
|
17
|
+
val = kwargs.get(argname, None)
|
|
18
|
+
if val == None:
|
|
19
|
+
return []
|
|
20
|
+
elif not isinstance(val, list):
|
|
21
|
+
return [ val ]
|
|
22
|
+
else:
|
|
23
|
+
return val
|
|
24
|
+
|
|
25
|
+
def _check_rx(argname, val, custom_err = None):
|
|
26
|
+
if not isinstance(val, (newstr, newbytes)):
|
|
27
|
+
raise InvalidParameter(return_message("Incorrect '%s' parameter" % argname, custom_err))
|
|
28
|
+
try:
|
|
29
|
+
return re.compile(val)
|
|
30
|
+
except:
|
|
31
|
+
raise InvalidParameter(return_message("Incorrect '%s' parameter" % argname, custom_err))
|
|
32
|
+
|
|
33
|
+
def _check_str(argname, val, rx, custom_err = None):
|
|
34
|
+
"""
|
|
35
|
+
This is not really check val is ASCII.
|
|
36
|
+
2021 09: we are now using version 17.4.0 -> we do not need to convert to
|
|
37
|
+
bytes here anymore, we are using a recent verison of cherrypy.
|
|
38
|
+
We merged the funcionality of _check_str and _check_ustr into a single function
|
|
39
|
+
|
|
40
|
+
:type val: str or bytes (only utf8 encoded string) in py3, unicode or str in py2
|
|
41
|
+
:type rx: regex, compiled from native str (unicode in py3, bytes in py2)
|
|
42
|
+
"""
|
|
43
|
+
val = decodeBytesToUnicodeConditional(val, condition=PY3)
|
|
44
|
+
val = encodeUnicodeToBytesConditional(val, condition=PY2)
|
|
45
|
+
# `val` should now be a "native str" (unicode in py3, bytes in py2)
|
|
46
|
+
# here str has not been redefined. it is default `str` in both py2 and py3.
|
|
47
|
+
if not isinstance(val, str) or not rx.match(val):
|
|
48
|
+
raise InvalidParameter(return_message("Incorrect '%s' parameter %s %s" % (argname, type(val), val), custom_err))
|
|
49
|
+
return val
|
|
50
|
+
|
|
51
|
+
def _check_num(argname, val, bare, minval, maxval, custom_err = None):
|
|
52
|
+
if not isinstance(val, numbers.Integral) and (not isinstance(val, (newstr, newbytes)) or (bare and not val.isdigit())):
|
|
53
|
+
raise InvalidParameter(return_message("Incorrect '%s' parameter" % argname, custom_err))
|
|
54
|
+
try:
|
|
55
|
+
n = int(val)
|
|
56
|
+
if (minval != None and n < minval) or (maxval != None and n > maxval):
|
|
57
|
+
raise InvalidParameter(return_message("Parameter '%s' value out of bounds" % argname, custom_err))
|
|
58
|
+
return n
|
|
59
|
+
except InvalidParameter:
|
|
60
|
+
raise
|
|
61
|
+
except:
|
|
62
|
+
raise InvalidParameter(return_message("Invalid '%s' parameter" % argname, custom_err))
|
|
63
|
+
|
|
64
|
+
def _check_real(argname, val, special, minval, maxval, custom_err = None):
|
|
65
|
+
if not isinstance(val, numbers.Number) and not isinstance(val, (newstr, newbytes)):
|
|
66
|
+
raise InvalidParameter(return_message("Incorrect '%s' parameter" % argname, custom_err))
|
|
67
|
+
try:
|
|
68
|
+
n = float(val)
|
|
69
|
+
if not special and (math.isnan(n) or math.isinf(n)):
|
|
70
|
+
raise InvalidParameter(return_message("Parameter '%s' improper value" % argname, custom_err))
|
|
71
|
+
if (minval != None and n < minval) or (maxval != None and n > maxval):
|
|
72
|
+
raise InvalidParameter(return_message("Parameter '%s' value out of bounds" % argname, custom_err))
|
|
73
|
+
return n
|
|
74
|
+
except InvalidParameter:
|
|
75
|
+
raise
|
|
76
|
+
except:
|
|
77
|
+
raise InvalidParameter(return_message("Invalid '%s' parameter" % argname, custom_err))
|
|
78
|
+
|
|
79
|
+
def _validate_one(argname, param, safe, checker, optional, *args):
|
|
80
|
+
val = param.kwargs.get(argname, None)
|
|
81
|
+
if optional and val == None:
|
|
82
|
+
safe.kwargs[argname] = None
|
|
83
|
+
else:
|
|
84
|
+
safe.kwargs[argname] = checker(argname, val, *args)
|
|
85
|
+
del param.kwargs[argname]
|
|
86
|
+
|
|
87
|
+
def _validate_all(argname, param, safe, checker, *args):
|
|
88
|
+
safe.kwargs[argname] = [checker(argname, v, *args) for v in _arglist(argname, param.kwargs)]
|
|
89
|
+
if argname in param.kwargs:
|
|
90
|
+
del param.kwargs[argname]
|
|
91
|
+
|
|
92
|
+
def validate_rx(argname, param, safe, optional = False, custom_err = None):
|
|
93
|
+
"""Validates that an argument is a valid regexp.
|
|
94
|
+
|
|
95
|
+
Checks that an argument named `argname` exists in `param.kwargs`,
|
|
96
|
+
and it a string which compiles into a python regular expression.
|
|
97
|
+
If successful, the regexp object (not the string) is copied into
|
|
98
|
+
`safe.kwargs` and the string value is removed from `param.kwargs`.
|
|
99
|
+
|
|
100
|
+
If `optional` is True, the argument is not required to exist in
|
|
101
|
+
`param.kwargs`; None is then inserted into `safe.kwargs`. Otherwise
|
|
102
|
+
a missing value raises an exception."""
|
|
103
|
+
_validate_one(argname, param, safe, _check_rx, optional, custom_err)
|
|
104
|
+
|
|
105
|
+
def validate_str(argname, param, safe, rx, optional = False, custom_err = None):
|
|
106
|
+
"""Validates that an argument is a string and matches a regexp.
|
|
107
|
+
|
|
108
|
+
Checks that an argument named `argname` exists in `param.kwargs`
|
|
109
|
+
and it is a string which matches regular expression `rx`. If
|
|
110
|
+
successful the string is copied into `safe.kwargs` and the value
|
|
111
|
+
is removed from `param.kwargs`.
|
|
112
|
+
|
|
113
|
+
Accepts both unicode strings and utf8-encoded bytes strings as argument
|
|
114
|
+
string.
|
|
115
|
+
Accepts regex compiled only with "native strings", which means str in both
|
|
116
|
+
py2 and py3 (unicode in py3, bytes of utf8-encoded strings in py2)
|
|
117
|
+
|
|
118
|
+
If `optional` is True, the argument is not required to exist in
|
|
119
|
+
`param.kwargs`; None is then inserted into `safe.kwargs`. Otherwise
|
|
120
|
+
a missing value raises an exception."""
|
|
121
|
+
_validate_one(argname, param, safe, _check_str, optional, rx, custom_err)
|
|
122
|
+
|
|
123
|
+
def validate_num(argname, param, safe, optional = False,
|
|
124
|
+
bare = False, minval = None, maxval = None, custom_err = None):
|
|
125
|
+
"""Validates that an argument is a valid integer number.
|
|
126
|
+
|
|
127
|
+
Checks that an argument named `argname` exists in `param.kwargs`,
|
|
128
|
+
and it is an int or a string convertible to a valid number. If successful
|
|
129
|
+
the integer value (not the string) is copied into `safe.kwargs`
|
|
130
|
+
and the original int/string value is removed from `param.kwargs`.
|
|
131
|
+
|
|
132
|
+
If `optional` is True, the argument is not required to exist in
|
|
133
|
+
`param.kwargs`; None is then inserted into `safe.kwargs`. Otherwise
|
|
134
|
+
a missing value raises an exception.
|
|
135
|
+
|
|
136
|
+
If `bare` is True, the number is required to be a pure digit sequence if it is a string.
|
|
137
|
+
Otherwise anything accepted by `int(val)` is acceted, including for
|
|
138
|
+
example leading white space or sign. Note that either way arbitrarily
|
|
139
|
+
large values are accepted; if you want to prevent abuse against big
|
|
140
|
+
integers, use the `minval` and `maxval` thresholds described below,
|
|
141
|
+
or check the length the of the string against some limit first.
|
|
142
|
+
|
|
143
|
+
If `minval` or `maxval` are given, values less than or greater than,
|
|
144
|
+
respectively, the threshold are rejected."""
|
|
145
|
+
_validate_one(argname, param, safe, _check_num, optional, bare, minval, maxval, custom_err)
|
|
146
|
+
|
|
147
|
+
def validate_real(argname, param, safe, optional = False,
|
|
148
|
+
special = False, minval = None, maxval = None, custom_err = None):
|
|
149
|
+
"""Validates that an argument is a valid real number.
|
|
150
|
+
|
|
151
|
+
Checks that an argument named `argname` exists in `param.kwargs`,
|
|
152
|
+
and it is float number or a string convertible to a valid number. If successful
|
|
153
|
+
the float value (not the string) is copied into `safe.kwargs`
|
|
154
|
+
and the original float/string value is removed from `param.kwargs`.
|
|
155
|
+
|
|
156
|
+
If `optional` is True, the argument is not required to exist in
|
|
157
|
+
`param.kwargs`; None is then inserted into `safe.kwargs`. Otherwise
|
|
158
|
+
a missing value raises an exception.
|
|
159
|
+
|
|
160
|
+
Anything accepted by `float(val)` is accepted, including for example
|
|
161
|
+
leading white space, sign and exponent. However NaN and +/- Inf are
|
|
162
|
+
rejected unless `special` is True.
|
|
163
|
+
|
|
164
|
+
If `minval` or `maxval` are given, values less than or greater than,
|
|
165
|
+
respectively, the threshold are rejected."""
|
|
166
|
+
_validate_one(argname, param, safe, _check_real, optional, special, minval, maxval, custom_err)
|
|
167
|
+
|
|
168
|
+
def validate_rxlist(argname, param, safe, custom_err = None):
|
|
169
|
+
"""Validates that an argument is an array of strings, each of which
|
|
170
|
+
can be compiled into a python regexp object.
|
|
171
|
+
|
|
172
|
+
Checks that an argument named `argname` is either a single string or
|
|
173
|
+
an array of strings, each of which compiles into a regular expression.
|
|
174
|
+
If successful the array is copied into `safe.kwargs` and the value is
|
|
175
|
+
removed from `param.kwargs`. The value always becomes an array in
|
|
176
|
+
`safe.kwargs`, even if no or only one argument was provided.
|
|
177
|
+
|
|
178
|
+
Note that an array of zero length is accepted, meaning there were no
|
|
179
|
+
`argname` parameters at all in `param.kwargs`."""
|
|
180
|
+
_validate_all(argname, param, safe, _check_rx, custom_err)
|
|
181
|
+
|
|
182
|
+
def validate_strlist(argname, param, safe, rx, custom_err = None):
|
|
183
|
+
"""Validates that an argument is an array of strings, each of which
|
|
184
|
+
matches a regexp.
|
|
185
|
+
|
|
186
|
+
Checks that an argument named `argname` is either a single string or
|
|
187
|
+
an array of strings, each of which matches the regular expression
|
|
188
|
+
`rx`. If successful the array is copied into `safe.kwargs` and the
|
|
189
|
+
value is removed from `param.kwargs`. The value always becomes an
|
|
190
|
+
array in `safe.kwargs`, even if no or only one argument was provided.
|
|
191
|
+
|
|
192
|
+
Use this method only for inputs which are meant to be bare strings.
|
|
193
|
+
|
|
194
|
+
Note that an array of zero length is accepted, meaning there were no
|
|
195
|
+
`argname` parameters at all in `param.kwargs`."""
|
|
196
|
+
_validate_all(argname, param, safe, _check_str, rx, custom_err)
|
|
197
|
+
|
|
198
|
+
def validate_numlist(argname, param, safe, bare=False, minval=None, maxval=None, custom_err = None):
|
|
199
|
+
"""Validates that an argument is an array of integers, as checked by
|
|
200
|
+
`validate_num()`.
|
|
201
|
+
|
|
202
|
+
Checks that an argument named `argname` is either a single string/int or
|
|
203
|
+
an array of strings/int, each of which validates with `validate_num` and
|
|
204
|
+
`bare`, `minval` and `maxval` arguments. If successful the array is
|
|
205
|
+
copied into `safe.kwargs` and the value is removed from `param.kwargs`.
|
|
206
|
+
The value always becomes an array in `kwsafe`, even if no or only one
|
|
207
|
+
argument was provided.
|
|
208
|
+
|
|
209
|
+
Note that an array of zero length is accepted, meaning there were no
|
|
210
|
+
`argname` parameters at all in `param.kwargs`."""
|
|
211
|
+
_validate_all(argname, param, safe, _check_num, bare, minval, maxval, custom_err)
|
|
212
|
+
|
|
213
|
+
def validate_reallist(argname, param, safe, special=False, minval=None, maxval=None, custom_err = None):
|
|
214
|
+
"""Validates that an argument is an array of integers, as checked by
|
|
215
|
+
`validate_real()`.
|
|
216
|
+
|
|
217
|
+
Checks that an argument named `argname` is either a single string/float or
|
|
218
|
+
an array of strings/floats, each of which validates with `validate_real` and
|
|
219
|
+
`special`, `minval` and `maxval` arguments. If successful the array is
|
|
220
|
+
copied into `safe.kwargs` and the value is removed from `param.kwargs`.
|
|
221
|
+
The value always becomes an array in `safe.kwargs`, even if no or only
|
|
222
|
+
one argument was provided.
|
|
223
|
+
|
|
224
|
+
Note that an array of zero length is accepted, meaning there were no
|
|
225
|
+
`argname` parameters at all in `param.kwargs`."""
|
|
226
|
+
_validate_all(argname, param, safe, _check_real, special, minval, maxval, custom_err)
|
|
227
|
+
|
|
228
|
+
def validate_no_more_input(param):
|
|
229
|
+
"""Verifies no more input is left in `param.args` or `param.kwargs`."""
|
|
230
|
+
if param.args:
|
|
231
|
+
raise InvalidParameter("Excess path arguments, not validated args='%s'" % param.args)
|
|
232
|
+
if param.kwargs:
|
|
233
|
+
raise InvalidParameter("Excess keyword arguments, not validated kwargs='%s'" % param.kwargs)
|
|
234
|
+
|
|
235
|
+
def validate_lengths(safe, *names):
|
|
236
|
+
"""Verifies that all `names` exist in `safe.kwargs`, are lists, and
|
|
237
|
+
all the lists have the same length. This is convenience function for
|
|
238
|
+
checking that an API accepting multiple values receives equal number
|
|
239
|
+
of values for all of its parameters."""
|
|
240
|
+
refname = names[0]
|
|
241
|
+
if refname not in safe.kwargs or not isinstance(safe.kwargs[refname], list):
|
|
242
|
+
raise InvalidParameter("Incorrect '%s' parameter" % refname)
|
|
243
|
+
|
|
244
|
+
reflen = len(safe.kwargs[refname])
|
|
245
|
+
for other in names[1:]:
|
|
246
|
+
if other not in safe.kwargs or not isinstance(safe.kwargs[other], list):
|
|
247
|
+
raise InvalidParameter("Incorrect '%s' parameter" % other)
|
|
248
|
+
elif len(safe.kwargs[other]) != reflen:
|
|
249
|
+
raise InvalidParameter("Mismatched number of arguments: %d %s vs. %d %s"
|
|
250
|
+
% (reflen, refname, len(safe.kwargs[other]), other))
|
WMCore/REST/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""WMCore.REST module"""
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Definition of valid status values for a request and valid status transitions.
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from future.utils import viewitems
|
|
7
|
+
|
|
8
|
+
# make this list to ensure insertion order here
|
|
9
|
+
REQUEST_START_STATE = "new"
|
|
10
|
+
REQUEST_STATE_TRANSITION = {
|
|
11
|
+
REQUEST_START_STATE: [REQUEST_START_STATE,
|
|
12
|
+
"assignment-approved",
|
|
13
|
+
"rejected"],
|
|
14
|
+
|
|
15
|
+
"assignment-approved": ["assigned", # manual transition
|
|
16
|
+
"rejected"], # manual transition
|
|
17
|
+
|
|
18
|
+
"assigned": ["staging",
|
|
19
|
+
"aborted"], # manual transition
|
|
20
|
+
"staging": ["staged",
|
|
21
|
+
"aborted"], # manual transition
|
|
22
|
+
"staged": ["acquired",
|
|
23
|
+
"aborted", # manual transition
|
|
24
|
+
"failed"],
|
|
25
|
+
"acquired": ["running-open",
|
|
26
|
+
"aborted", # manual transition
|
|
27
|
+
"failed"],
|
|
28
|
+
"running-open": ["running-closed",
|
|
29
|
+
"force-complete", # manual transition
|
|
30
|
+
"aborted"], # manual transition
|
|
31
|
+
|
|
32
|
+
"running-closed": ["force-complete", # manual transition
|
|
33
|
+
"completed",
|
|
34
|
+
"aborted"], # manual transition
|
|
35
|
+
|
|
36
|
+
"force-complete": ["completed"],
|
|
37
|
+
|
|
38
|
+
"failed": ["rejected", # manual transition
|
|
39
|
+
"assigned"], # manual transition
|
|
40
|
+
|
|
41
|
+
"completed": ["closed-out",
|
|
42
|
+
"rejected"], # manual transition
|
|
43
|
+
|
|
44
|
+
"closed-out": ["announced", "rejected"], # manual transition
|
|
45
|
+
|
|
46
|
+
"announced": ["rejected", "normal-archived"],
|
|
47
|
+
|
|
48
|
+
"aborted": ["aborted-completed"],
|
|
49
|
+
|
|
50
|
+
"aborted-completed": ["aborted-archived"],
|
|
51
|
+
|
|
52
|
+
"rejected": ["rejected-archived"],
|
|
53
|
+
|
|
54
|
+
# final status
|
|
55
|
+
"normal-archived": ["rejected-archived"],
|
|
56
|
+
|
|
57
|
+
"aborted-archived": [],
|
|
58
|
+
|
|
59
|
+
"rejected-archived": []
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
# List of requests that a human user can change a request to
|
|
63
|
+
REQUEST_HUMAN_STATES = ["assignment-approved",
|
|
64
|
+
"assigned",
|
|
65
|
+
"force-complete",
|
|
66
|
+
"closed-out",
|
|
67
|
+
"announced",
|
|
68
|
+
"rejected",
|
|
69
|
+
"aborted"]
|
|
70
|
+
|
|
71
|
+
ACTIVE_STATUS = ["new",
|
|
72
|
+
"assignment-approved",
|
|
73
|
+
"assigned",
|
|
74
|
+
"staging",
|
|
75
|
+
"staged",
|
|
76
|
+
"acquired",
|
|
77
|
+
"running-open",
|
|
78
|
+
"running-closed",
|
|
79
|
+
"failed",
|
|
80
|
+
"force-complete",
|
|
81
|
+
"completed",
|
|
82
|
+
"closed-out",
|
|
83
|
+
"announced",
|
|
84
|
+
"aborted",
|
|
85
|
+
"aborted-completed",
|
|
86
|
+
"rejected"]
|
|
87
|
+
|
|
88
|
+
### WMSTATS_JOB_INFO + WMSTATS_NO_JOB_INFO is meant to be equal to ACTIVE_STATUS
|
|
89
|
+
WMSTATS_JOB_INFO = ["running-open",
|
|
90
|
+
"running-closed",
|
|
91
|
+
"force-complete",
|
|
92
|
+
"completed",
|
|
93
|
+
"closed-out"]
|
|
94
|
+
|
|
95
|
+
WMSTATS_NO_JOB_INFO = ["new",
|
|
96
|
+
"assignment-approved",
|
|
97
|
+
"assigned",
|
|
98
|
+
"staging",
|
|
99
|
+
"staged",
|
|
100
|
+
"acquired",
|
|
101
|
+
"failed",
|
|
102
|
+
"announced",
|
|
103
|
+
"aborted",
|
|
104
|
+
"aborted-completed",
|
|
105
|
+
"rejected"]
|
|
106
|
+
|
|
107
|
+
### Used for monitoring in T0-WMStats. See: Services/WMStats/WMStatsReader
|
|
108
|
+
T0_ACTIVE_STATUS = ["new",
|
|
109
|
+
"Closed",
|
|
110
|
+
"Merge",
|
|
111
|
+
"Harvesting",
|
|
112
|
+
"Processing Done",
|
|
113
|
+
"AlcaSkim",
|
|
114
|
+
"completed"]
|
|
115
|
+
|
|
116
|
+
# if the state is not defined here (new) allows all the property to get
|
|
117
|
+
# states in the key is the source states (need to define source states instead of destination states for GUI update)
|
|
118
|
+
ALLOWED_ACTIONS_FOR_STATUS = {
|
|
119
|
+
"new": ["RequestPriority"],
|
|
120
|
+
"assignment-approved": ["RequestPriority", "Team", "SiteWhitelist", "SiteBlacklist",
|
|
121
|
+
"AcquisitionEra", "ProcessingString", "ProcessingVersion",
|
|
122
|
+
"Dashboard", "MergedLFNBase", "TrustSitelists",
|
|
123
|
+
"UnmergedLFNBase", "MinMergeSize", "MaxMergeSize",
|
|
124
|
+
"MaxMergeEvents", "BlockCloseMaxWaitTime",
|
|
125
|
+
"BlockCloseMaxFiles", "BlockCloseMaxEvents", "BlockCloseMaxSize",
|
|
126
|
+
"SoftTimeout", "GracePeriod",
|
|
127
|
+
"TrustPUSitelists", "CustodialSites",
|
|
128
|
+
"NonCustodialSites", "Override",
|
|
129
|
+
"SubscriptionPriority"],
|
|
130
|
+
"assigned": ["RequestPriority"],
|
|
131
|
+
"staging": ["RequestPriority", "SiteWhitelist", "SiteBlacklist"],
|
|
132
|
+
"staged": ["RequestPriority"],
|
|
133
|
+
"acquired": ["RequestPriority", "SiteWhitelist", "SiteBlacklist"],
|
|
134
|
+
"running-open": ["RequestPriority", "SiteWhitelist", "SiteBlacklist"],
|
|
135
|
+
"running-closed": ["RequestPriority"],
|
|
136
|
+
"failed": [],
|
|
137
|
+
"force-complete": [],
|
|
138
|
+
"completed": [],
|
|
139
|
+
"closed-out": [],
|
|
140
|
+
"announced": [],
|
|
141
|
+
"aborted": [],
|
|
142
|
+
"aborted-completed": [],
|
|
143
|
+
"rejected": [],
|
|
144
|
+
"normal-archived": [],
|
|
145
|
+
"aborted-archived": [],
|
|
146
|
+
"rejected-archived": [],
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
# NOTE: We need to Explicitly add RequestStatus to reqArgsDiff, since it is
|
|
150
|
+
# missing from the ALLOWED_ACTIONS_FOR_STATUS mapping. The alternative
|
|
151
|
+
# would be to add it as allowed action to every state.
|
|
152
|
+
# The same applies to few more, such as all the keys needed for the
|
|
153
|
+
# workqueue_stat_validation() calls, but we do this only during
|
|
154
|
+
# request parameters validation.
|
|
155
|
+
ALLOWED_ACTIONS_ALL_STATUS = ["RequestStatus"]
|
|
156
|
+
|
|
157
|
+
# NOTE: We need to explicitly add all stat keys during validation
|
|
158
|
+
# They are needed for the workqueue_stat_validation() calls
|
|
159
|
+
ALLOWED_STAT_KEYS = ['total_jobs', 'input_lumis', 'input_events', 'input_num_files']
|
|
160
|
+
|
|
161
|
+
# Workflow state transition automatically controlled by ReqMgr2
|
|
162
|
+
### NOTE: order of this list matters and it's used for status transition
|
|
163
|
+
AUTO_TRANSITION = {"staged": ["acquired", "running-open", "running-closed", "completed"],
|
|
164
|
+
"acquired": ["running-open", "running-closed", "completed"],
|
|
165
|
+
"running-open": ["running-closed", "completed"],
|
|
166
|
+
"running-closed": ["completed"]}
|
|
167
|
+
|
|
168
|
+
# Workflow state transition automatically controlled by ReqMgr2
|
|
169
|
+
# Specific to workflows either aborted or force-completed
|
|
170
|
+
CANCEL_AUTO_TRANSITION = {"aborted": "aborted-completed",
|
|
171
|
+
"force-complete": "completed"}
|
|
172
|
+
|
|
173
|
+
# list of destination states which doesn't allow any additional argument update
|
|
174
|
+
STATES_ALLOW_ONLY_STATE_TRANSITION = [key for key, val in viewitems(ALLOWED_ACTIONS_FOR_STATUS) if len(val) == 0]
|
|
175
|
+
# each item from STATUS_TRANSITION is a dictionary with 1 item, the key
|
|
176
|
+
# is name of the status
|
|
177
|
+
REQUEST_STATE_LIST = list(REQUEST_STATE_TRANSITION)
|
|
178
|
+
|
|
179
|
+
ACTIVE_STATUS_FILTER = {"RequestStatus": ['assignment-approved', 'assigned', 'staging', 'staged',
|
|
180
|
+
'failed', 'acquired', 'running-open', 'running-closed',
|
|
181
|
+
'force-complete', 'completed', 'closed-out']}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def check_allowed_transition(preState, postState):
|
|
185
|
+
stateList = REQUEST_STATE_TRANSITION.get(preState, [])
|
|
186
|
+
if postState in stateList:
|
|
187
|
+
return True
|
|
188
|
+
else:
|
|
189
|
+
return False
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def get_modifiable_properties(status=None):
|
|
193
|
+
"""
|
|
194
|
+
returns mondifiable property list by status.
|
|
195
|
+
if status is not defined return dictionarly of all the status and property list
|
|
196
|
+
TODO: Currently gets the result from hardcoded list. change to get from configuration or db
|
|
197
|
+
"""
|
|
198
|
+
if status:
|
|
199
|
+
allowedKeys = ALLOWED_ACTIONS_FOR_STATUS.get(status, 'all_attributes')
|
|
200
|
+
return allowedKeys
|
|
201
|
+
else:
|
|
202
|
+
return ALLOWED_ACTIONS_FOR_STATUS
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def get_protected_properties():
|
|
206
|
+
"""
|
|
207
|
+
returns properties never be modified once request is created
|
|
208
|
+
"""
|
|
209
|
+
return ["RequestName", "_id"]
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|