wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/ACDC/Collection.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+Collection.py
+
+Created by Dave Evans on 2010-03-11.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+
+from WMCore.DataStructs.WMObject import WMObject
+import WMCore.ACDC.CollectionTypes as CollectionTypes
+
+
+class Collection(dict, WMObject):
+    def __init__(self, **options):
+        dict.__init__(self)
+        WMObject.__init__(self)
+        self.setdefault("name", None)
+        self.setdefault("type", CollectionTypes.GenericCollection)
+        self.setdefault("filesets", [])
+        self.update(options)
+
+    def create(self, unique=False):
+        """
+        _create_
+
+        Create this Collection in the back end
+
+        """
+        pass
+
+    def populate(self):
+        """
+        _populate_
+
+        Pull in all filesets & file entries
+
+        """
+        pass
+
+    def drop(self):
+        """
+        _drop_
+
+        Remove this collection.
+        """
+        pass
+
+    def addFileset(self, fileset):
+        """
+        _addFiles_
+
+        Add a fileset to the collection.
+        """
+        fileset.setCollection(self)
+        self["filesets"].append(fileset)
+        return

WMCore/ACDC/CollectionTypes.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+CollectionTypes.py
+
+Created by Dave Evans on 2010-03-05.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+
+
+GenericCollection = "ACDC.CollectionTypes.GenericCollection"
+DataCollection = "ACDC.CollectionTypes.DataCollection"

WMCore/ACDC/CouchCollection.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+CouchCollection.py
+
+Created by Dave Evans on 2010-03-14.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+
+from WMCore.ACDC.Collection import Collection
+from WMCore.ACDC.CouchFileset import CouchFileset
+from WMCore.Database.CouchUtils import connectToCouch
+
+
+class CouchCollection(Collection):
+    """
+    Collection that can be stored in CouchDB.
+
+    Required Args:
+      database - CouchDB database instance name
+      url - CouchDB Server URL
+      name - name of the collection
+    """
+
+    def __init__(self, **options):
+        Collection.__init__(self, **options)
+        self.url = options.get("url")
+        self.database = options.get("database")
+        self.name = options.get("name")
+        self.server = None
+        self.couchdb = None
+
+    @connectToCouch
+    def drop(self):
+        """
+        _drop_
+
+        Drop this collection and all files and filesets within it.
+        """
+        params = {"startkey": [self.name],
+                  "endkey": [self.name, {}],
+                  "reduce": False}
+        result = self.couchdb.loadView("ACDC", "coll_fileset_docs",
+                                       params)
+
+        for row in result["rows"]:
+            self.couchdb.delete_doc(row["id"])
+        return
+
+    @connectToCouch
+    def populate(self):
+        """
+        _populate_
+
+        The load the collection and all filesets and files out of couch.
+        """
+        params = {"startkey": [self.name],
+                  "endkey": [self.name, {}],
+                  "reduce": True, "group_level": 2}
+        result = self.couchdb.loadView("ACDC", "coll_fileset_docs",
+                                       params)
+        self["filesets"] = []
+        for row in result["rows"]:
+            fileset = CouchFileset(database=self.database, url=self.url,
+                                   name=row["key"][1])
+            self.addFileset(fileset)
+            fileset.populate()
+        return

WMCore/ACDC/CouchFileset.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+CouchFileset.py
+
+Created by Dave Evans on 2010-03-19.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+
+from future.utils import viewvalues
+
+import time
+
+import WMCore.Database.CMSCouch as CMSCouch
+from WMCore.ACDC.Fileset import Fileset
+from WMCore.Algorithms.ParseXMLFile import coroutine
+from WMCore.DataStructs.File import File
+from WMCore.DataStructs.Fileset import Fileset as DataStructsFileset
+from WMCore.DataStructs.Run import Run
+from WMCore.Database.CouchUtils import connectToCouch, requireFilesetName
+
+
+@coroutine
+def makeRun(targets):
+    """
+    _makeRun_
+
+    create a DataStruct Run instance form the Couch JSON dict
+    """
+    while True:
+        fileRef, runs = (yield)
+        for run in runs:
+            newRun = Run(run[u'Run'], *run[u'Lumis'])
+            fileRef.addRun(newRun)
+
+
+@coroutine
+def filePipeline(targets):
+    """
+    _conversionPipeline_
+
+    """
+    while True:
+        inputDict = (yield)
+        newFile = File(
+            lfn=str(inputDict[u'lfn']),
+            size=int(inputDict[u'size']),
+            events=int(inputDict[u'events'])
+        )
+        targets['run'].send((newFile, inputDict[u'runs']))
+        targets['fileset'].addFile(newFile)
+
+
+class CouchFileset(Fileset):
+    def __init__(self, **options):
+        Fileset.__init__(self, **options)
+        self.url = options.get('url')
+        self.database = options.get('database')
+        self['name'] = options.get('name')
+        self.server = None
+        self.couchdb = None
+
+    @connectToCouch
+    def drop(self):
+        """
+        _drop_
+
+        Remove this fileset
+
+        This is racy. someone can add to the fileset before we get done
+        deleting it. Actually, they can delete after we get done deleting
+        it too. Oh well.
+        """
+        for d in self.filelistDocuments():
+            self.couchdb.delete_doc(d)
+
+    @connectToCouch
+    @requireFilesetName
+    def filelistDocuments(self):
+        """
+        _filelistDocuments_
+
+        Get a list of document ids corresponding to filelists in this fileset
+        """
+        params = {"startkey": [self.collectionName, self["name"]],
+                  "endkey": [self.collectionName, self["name"]],
+                  "reduce": False}
+        result = self.couchdb.loadView("ACDC", "coll_fileset_docs",
+                                       params)
+
+        docs = [row["id"] for row in result["rows"]]
+        return docs
+
+    @connectToCouch
+    @requireFilesetName
+    def add(self, files, mask=None):
+        """
+        _add_
+
+        Add files to this fileset
+
+        Note: if job was lumi based splitted, then we do not have
+        reliable events information. If job was event based splitted,
+        then we do not have reliable lumi information.
+        """
+        filteredFiles = []
+        if mask:
+            for f in files:
+                # There might be no LastEvent for last job of a file
+                if mask['LastEvent'] and mask['FirstEvent']:
+                    f['events'] = mask['LastEvent'] - mask['FirstEvent'] + 1
+                    f['first_event'] = mask['FirstEvent']
+                elif mask['FirstEvent']:
+                    f['events'] = f['events'] - mask['FirstEvent'] + 1
+                    f['first_event'] = mask['FirstEvent']
+
+            maskLumis = mask.getRunAndLumis()
+            if maskLumis != {}:
+                # Then we actually have to do something
+                for f in files:
+                    newRuns = mask.filterRunLumisByMask(runs=f['runs'])
+                    if newRuns != set([]):
+                        f['runs'] = newRuns
+                        filteredFiles.append(f)
+            else:
+                # Likely real data with EventBased splitting
+                filteredFiles = files
+        else:
+            filteredFiles = files
+
+        jsonFiles = {}
+        for f in filteredFiles:
+            jsonFiles.__setitem__(f['lfn'], f.__to_json__(None))
+        fileList = self.makeFilelist(jsonFiles)
+        return fileList
+
+    @connectToCouch
+    @requireFilesetName
+    def makeFilelist(self, files=None):
+        """
+        _makeFilelist_
+
+        Create a new filelist document containing the id
+        """
+        files = files or {}
+        # add a version to each of these ACDC docs such that we can properly
+        # parse them and avoid issues between ACDC docs and agent base code
+        input = {"collection_name": self.collectionName,
+                 "collection_type": self.collectionType,
+                 "fileset_name": self["name"],
+                 "files": files,
+                 "acdc_version": 2,
+                 "timestamp": time.time()}
+
+        document = CMSCouch.Document(None, input)
+
+        commitInfo = self.couchdb.commitOne(document)
+        document['_id'] = commitInfo[0]['id']
+        if 'rev' in commitInfo[0]:
+            document['_rev'] = commitInfo[0]['rev']
+        else:
+            if commitInfo[0]['reason'].find('{exit_status,0}') != -1:
+                # TODO: in this case actually insert succeeded but return error
+                # due to the bug
+                # https://issues.apache.org/jira/browse/COUCHDB-893
+                # if rev is needed to proceed need to get by
+                # self.couchdb.documentExist(document['_id'])
+                # but that function need to be changed to return _rev
+                document['_rev'] = "NeedToGet"
+            else:
+                msg = "Unable to insert document: check acdc server doc id: %s" % document['_id']
+                raise RuntimeError(msg)
+        return document
+
+    @connectToCouch
+    def listFiles(self):
+        """
+        _listFiles_
+
+        return an iterator over the files contained in this fileset
+        """
+        for filelist in self.filelistDocuments():
+            try:
+                doc = self.couchdb.document(filelist)
+            except CMSCouch.CouchNotFoundError as ex:
+                msg = "Unable to retrieve Couch Document for fileset"
+                msg += str(msg)
+                raise RuntimeError(msg)
+
+            files = doc["files"]
+            for d in viewvalues(files):
+                yield d
+
+    @connectToCouch
+    def fileset(self):
+        """
+        _fileset_
+
+        Make a WMCore.DataStruct.Fileset instance containing the files in this fileset
+
+        """
+        result = DataStructsFileset(self['name'])
+        pipeline = filePipeline({'fileset': result, 'run': makeRun({})})
+        for f in self.listFiles():
+            pipeline.send(f)
+        return result
+
+    @connectToCouch
+    @requireFilesetName
+    def populate(self):
+        """
+        _populate_
+
+        Load all files out of couch.
+        """
+        params = {"startkey": [self.collectionName, self["name"]],
+                  "endkey": [self.collectionName, self["name"]],
+                  "include_docs": True, "reduce": False}
+        result = self.couchdb.loadView("ACDC", "coll_fileset_docs",
+                                       params)
+        self.files = {}
+        for row in result["rows"]:
+            self.files.update(row["doc"]["files"])
+            self["files"] = self.files
+        return
+
+    def fileCount(self):
+        """
+        _fileCount_
+
+        Determine how many files are in the fileset.
+        """
+        params = {"startkey": [self.collectionName, self["name"]],
+                  "endkey": [self.collectionName, self["name"]],
+                  "reduce": True, "group_level": 2}
+        result = self.couchdb.loadView("ACDC", "coll_fileset_count",
+                                       params)
+        return result["rows"][0]["value"]

WMCore/ACDC/CouchService.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# encoding: utf-8
+"""
+CouchService.py
+
+Created by Dave Evans on 2010-04-20.
+Copyright (c) 2010 Fermilab. All rights reserved.
+"""
+from builtins import object
+from time import time
+
+import WMCore.Database.CouchUtils as CouchUtils
+
+
+class CouchService(object):
+    def __init__(self, **options):
+        super(CouchService, self).__init__()
+        self.options = {}
+        self.options.update(options)
+        self.url = options.get('url', None)
+        self.database = options.get('database', None)
+        self.server = None
+        self.couchdb = None
+
+    def listFilesets(self, collectionInstance):
+        """
+        _listFilesets_
+
+        List filesets for the collection instance provided.
+        """
+        collectionInstance.populate()
+
+        for fileset in collectionInstance["filesets"]:
+            yield fileset
+
+    @CouchUtils.connectToCouch
+    def removeFilesetsByCollectionName(self, collectionName):
+        """
+        _removeFilesetsByCollectionName_
+
+        Remove all the collections matching certain collection
+        name.
+        """
+        result = self.couchdb.loadView("ACDC", "byCollectionName", options={"reduce": False}, keys=[collectionName])
+        for entry in result["rows"]:
+            self.couchdb.queueDelete(entry["value"])
+        return self.couchdb.commit()
+
+    @CouchUtils.connectToCouch
+    def removeOldFilesets(self, expirationDays):
+        """
+        _removeOldFilesets_
+
+        Remove filesets older than certain date defined
+        in expirationDays (in days).
+        """
+        cutoutPoint = time() - (expirationDays * 3600 * 24)
+        result = self.couchdb.loadView("ACDC", "byTimestamp", {"endkey": cutoutPoint})
+        count = 0
+        for entry in result["rows"]:
+            self.couchdb.queueDelete(entry["value"])
+            count += 1
+        self.couchdb.commit()
+        return count
+
+    @CouchUtils.connectToCouch
+    def listCollectionNames(self):
+        options = {'reduce': True, 'group_level': 1, 'stale': "update_after"}
+        result = self.couchdb.loadView("ACDC", "byCollectionName", options)
+        collectionNames = []
+        for row in result["rows"]:
+            collectionNames.append(row["key"])
+        return collectionNames