rcsb.exdb 1.31__py3-none-any.whl

rcsb/exdb/utils/ObjectExtractor.py

@@ -0,0 +1,286 @@
+##
+# File: ObjectExtractor.py
+# Date: 26-Jun-2019  jdw
+#
+# Utilities to extract document features from the document object server.
+#
+# Updates:
+# 27-Jun-2019  jdw add JSON path tracking utilities.
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import copy
+import logging
+import os
+
+from rcsb.db.mongo.Connection import Connection
+from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectExtractor(object):
+    """Utilities to extract document features from the document object server."""
+
+    def __init__(self, cfgOb, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__resourceName = "MONGO_DB"
+        self.__mU = MarshalUtil()
+        #
+        self.__objectD = self.__rebuildCache(**kwargs)
+        self.__objPathD = {}
+        self.__stringPathList = []
+        self.__objValD = {}
+        #
+
+    def getObjects(self):
+        return self.__objectD
+
+    def getPathList(self, filterList=True):
+        kL = []
+        if filterList:
+            tL = []
+            for ky in self.__objPathD:
+                if ky and (ky.find(".") != -1 or ky.startswith("_")) and ky not in ["_id"] and not ky.endswith("[]"):
+                    tL.append(ky)
+            for ky in tL:
+                for tky in tL:
+                    ok = True
+                    if ky in tky and ky != tky:
+                        ok = False
+                        break
+                if ok:
+                    kL.append(ky)
+        else:
+            kL = list(self.__objPathD.keys())
+        #
+        return sorted(kL)
+
+    def getValues(self):
+        return self.__objValD
+
+    def setPathList(self, stringPathList):
+        self.__objPathD = {k: True for k in stringPathList}
+        return True
+
+    def getCount(self):
+        return len(self.__objectD)
+
+    def __rebuildCache(self, **kwargs):
+        cacheFilePath = kwargs.get("cacheFilePath", None)
+        cacheKwargs = kwargs.get("cacheKwargs", {"fmt": "pickle"})
+        useCache = kwargs.get("useCache", True)
+        keyAttribute = kwargs.get("keyAttribute", "entry")
+        selectL = kwargs.get("selectionList", [])
+        #
+        cD = {keyAttribute: {}}
+        try:
+            if useCache and cacheFilePath and os.access(cacheFilePath, os.R_OK):
+                cD = self.__mU.doImport(cacheFilePath, **cacheKwargs)
+            else:
+                if selectL:
+                    objectD = self.__select(**kwargs)
+                else:
+                    objectD = self.__selectObjects(**kwargs)
+                cD[keyAttribute] = objectD
+                if cacheFilePath:
+                    pth, _ = os.path.split(cacheFilePath)
+                    ok = self.__mU.mkdir(pth)
+                    ok = self.__mU.doExport(cacheFilePath, cD, **cacheKwargs)
+                    logger.info("Saved object results (%d) status %r in %s", len(objectD), ok, cacheFilePath)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return cD[keyAttribute]
+
+    def __selectObjects(self, **kwargs):
+        """Return a dictionary of objects satisfying the input conditions (e.g. method, resolution limit)"""
+        databaseName = kwargs.get("databaseName", "pdbx_core")
+        collectionName = kwargs.get("collectionName", "pdbx_core_entry")
+        selectionQueryD = kwargs.get("selectionQuery", {})
+        #
+        uniqueAttributes = kwargs.get("uniqueAttributes", ["rcsb_id"])
+        #
+        tV = kwargs.get("objectLimit", None)
+        objLimit = int(tV) if tV is not None else None
+        stripObjectId = kwargs.get("stripObjectId", False)
+        logIncrement = kwargs.get("logIncrement", 10000)
+        #
+        objectD = {}
+        try:
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.info("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    qD = {}
+                    if selectionQueryD:
+                        qD.update(selectionQueryD)
+                    selectL = ["_id"]
+                    dL = mg.fetch(databaseName, collectionName, selectL, queryD=qD)
+                    numDoc = len(dL) if dL else 0
+                    logger.info("Selection %r fetch result count %d", selectL, numDoc)
+                    #
+                    for ii, dD in enumerate(dL, 1):
+                        if "_id" not in dD:
+                            continue
+                        rObj = mg.fetchOne(databaseName, collectionName, "_id", dD["_id"])
+                        if stripObjectId and rObj and "_id" in rObj:
+                            rObj.pop("_id")
+                        else:
+                            rObj["_id"] = str(rObj["_id"])
+                        #
+                        stKey = ".".join([rObj[ky] for ky in uniqueAttributes])
+                        objectD[stKey] = copy.copy(rObj)
+                        if objLimit and ii >= objLimit:
+                            break
+                        logger.debug("Saving %d %s", ii, stKey)
+                        if ii % logIncrement == 0 or ii == numDoc:
+                            logger.info("Extracting object (%d of %d)", ii, numDoc)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return objectD
+        #
+
+    def __select(self, **kwargs):
+        """Return a dictionary of object content satisfying the input conditions
+        (e.g. method, resolution limit) and selection options.
+        """
+        databaseName = kwargs.get("databaseName", "pdbx_core")
+        collectionName = kwargs.get("collectionName", "pdbx_core_entry")
+        selectionQueryD = kwargs.get("selectionQuery", {})
+        uniqueAttributes = kwargs.get("uniqueAttributes", ["rcsb_id"])
+        selectL = kwargs.get("selectionList", [])
+        stripObjectId = kwargs.get("stripObjectId", False)
+        #
+        tV = kwargs.get("objectLimit", None)
+        objLimit = int(tV) if tV is not None else None
+        #
+        objectD = {}
+        try:
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.info("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    qD = {}
+                    if selectionQueryD:
+                        qD.update(selectionQueryD)
+                    dL = mg.fetch(databaseName, collectionName, selectL, queryD=qD, suppressId=True)
+                    logger.info("Selection %r fetch result count %d", selectL, len(dL))
+                    #
+                    for ii, rObj in enumerate(dL, 1):
+                        stKey = ".".join([rObj[ky] for ky in uniqueAttributes])
+                        if stripObjectId and rObj and "_id" in rObj:
+                            rObj.pop("_id")
+                        objectD[stKey] = copy.copy(rObj)
+                        if objLimit and ii >= objLimit:
+                            break
+                        # logger.debug("Saving %d %s", ii, stKey)
+                        # logger.debug("Current objectD keys %r", list(objectD.keys()))
+
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return objectD
+        #
+
+    def __getKeyValues(self, dct, keyNames):
+        """Return the tuple of values of corresponding to the input dictionary key names expressed in dot notation.
+
+        Args:
+            dct (dict): source dictionary object (nested)
+            keyNames (list): list of dictionary keys in dot notation
+
+        Returns:
+            tuple: tuple of values corresponding to the input key names
+
+        """
+        rL = []
+        try:
+            for keyName in keyNames:
+                rL.append(self.__getKeyValue(dct, keyName))
+        except Exception as e:
+            logger.exception("Failing for key names %r with %s", keyNames, str(e))
+
+        return tuple(rL)
+
+    def __getKeyValue(self, dct, keyName):
+        """Return the value of the corresponding key expressed in dot notation in the input dictionary object (nested)."""
+        try:
+            kys = keyName.split(".")
+            for key in kys:
+                try:
+                    dct = dct[key]
+                except KeyError:
+                    return None
+            return dct
+        except Exception as e:
+            logger.exception("Failing for key %r with %s", keyName, str(e))
+
+        return None
+
+    def __toJsonPathString(self, path):
+        pL = [ky if ky else "[]" for ky in path]
+        sp = ".".join(pL)
+        sp = sp.replace(".[", "[")
+        return sp
+
+    def __pathCallBack(self, path, value):
+        sp = self.__toJsonPathString(path)
+        self.__objPathD[sp] = self.__objPathD[sp] + 1 if sp in self.__objPathD else 1
+        return value
+
+    def __saveCallBack(self, path, value):
+        sP = self.__toJsonPathString(path)
+        if sP in self.__objPathD:
+            ky = sP.replace("[]", "")
+            if sP.find("[") != -1:  # multivalued
+                if isinstance(value, list):
+                    self.__objValD.setdefault(ky, []).extend(value)
+                else:
+                    self.__objValD.setdefault(ky, []).append(value)
+            else:
+                self.__objValD[ky] = value
+        return value
+
+    def genPathList(self, dObj, path=None):
+        return self.__walk(dObj, jsonPath=path, funct=self.__pathCallBack)
+
+    def genValueList(self, dObj, path=None, clear=True):
+        self.__objValD = {} if clear else self.__objValD
+        return self.__walk(dObj, jsonPath=path, funct=self.__saveCallBack)
+
+    def __walk(self, jsonObj, jsonPath=None, funct=None):
+        """Walk JSON data types. An optional funct() is called to mutate
+        the value of each element. The jsonPath is updated at each element.
+        """
+        if jsonPath is None:
+            jsonPath = []
+
+        if isinstance(jsonObj, dict):
+            value = {k: self.__walk(v, jsonPath + [k], funct) for k, v in jsonObj.items()}
+        elif isinstance(jsonObj, list):
+            value = [self.__walk(elem, jsonPath + [[]], funct) for elem in jsonObj]
+        else:
+            value = jsonObj
+
+        if funct is None:
+            return value
+        else:
+            return funct(jsonPath, value)
+
+    def __toPath(self, path):
+        """Convert path strings into path lists."""
+        if isinstance(path, list):
+            return path  # already in list format
+
+        def _iterPath(path):
+            for parts in path.split("[]"):
+                for part in parts.strip(".").split("."):
+                    yield part
+                yield []
+
+        return list(_iterPath(path))[:-1]

rcsb/exdb/utils/ObjectTransformer.py

@@ -0,0 +1,124 @@
+##
+# File: ObjectTransformer.py
+# Date: 17-Oct-2019  jdw
+#
+# Utilities to extract and update object from the document object server.
+#
+# Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+
+from rcsb.db.mongo.Connection import Connection
+from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
+from rcsb.db.processors.DataExchangeStatus import DataExchangeStatus
+from rcsb.db.utils.TimeUtil import TimeUtil
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectTransformer(object):
+    """Utilities to extract and update object from the document object server."""
+
+    def __init__(self, cfgOb, objectAdapter=None, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__oAdapt = objectAdapter
+        self.__resourceName = "MONGO_DB"
+        _ = kwargs
+        self.__statusList = []
+
+    def doTransform(self, **kwargs):
+        desp = DataExchangeStatus()
+        statusStartTimestamp = desp.setStartTime()
+        #
+        databaseName = kwargs.get("databaseName", "pdbx_core")
+        collectionName = kwargs.get("collectionName", "pdbx_core_entry")
+        selectionQueryD = kwargs.get("selectionQuery", {})
+        fetchLimit = kwargs.get("fetchLimit", None)
+        tU = TimeUtil()
+        updateId = kwargs.get("updateId", tU.getCurrentWeekSignature())
+        #
+        docSelectList = self.__selectObjectIds(databaseName, collectionName, selectionQueryD)
+        docSelectList = docSelectList[:fetchLimit] if fetchLimit else docSelectList
+        ok = self.__transform(databaseName, collectionName, docSelectList)
+        #
+        okS = True
+        if updateId:
+            okS = self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
+        return ok and okS
+
+    def __selectObjectIds(self, databaseName, collectionName, selectionQueryD):
+        """Return a list of object identifiers for the input selection query."""
+        try:
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.info("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    qD = {}
+                    if selectionQueryD:
+                        qD.update(selectionQueryD)
+                    selectL = ["_id"]
+                    dL = mg.fetch(databaseName, collectionName, selectL, queryD=qD)
+                    logger.info("Selection %r fetch result count %d", selectL, len(dL))
+
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return dL
+        #
+
+    def __transform(self, databaseName, collectionName, docSelectList, logIncrement=10000):
+        """Return a list of object identifiers for the input selection query."""
+        #
+        ok = True
+        try:
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    numDoc = len(docSelectList)
+                    for ii, dD in enumerate(docSelectList, 1):
+                        if "_id" not in dD:
+                            continue
+                        rObj = mg.fetchOne(databaseName, collectionName, "_id", dD["_id"])
+                        del rObj["_id"]
+                        #
+                        fOk = True
+                        if self.__oAdapt:
+                            fOk, rObj = self.__oAdapt.filter(rObj)
+                        if fOk:
+                            rOk = mg.replace(databaseName, collectionName, rObj, dD)
+                            if rOk is None:
+                                tId = rObj["rcsb_id"] if "rcsb_id" in rObj else "anonymous"
+                                logger.error("%r %r (%r) failing", databaseName, collectionName, tId)
+                                logger.debug("rObj.keys() %r", list(rObj.keys()))
+                                logger.debug("rObj.items() %s", rObj.items())
+                                rOk = False
+                            ok = ok and rOk
+                        #
+                        if ii % logIncrement == 0 or ii == numDoc:
+                            logger.info("Replace status %r object (%d of %d)", ok, ii, numDoc)
+                        #
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok
+
+    def getLoadStatus(self):
+        return self.__statusList
+
+    def __updateStatus(self, updateId, databaseName, collectionName, status, startTimestamp):
+        try:
+            sFlag = "Y" if status else "N"
+            desp = DataExchangeStatus()
+            desp.setStartTime(tS=startTimestamp)
+            desp.setObject(databaseName, collectionName)
+            desp.setStatus(updateId=updateId, successFlag=sFlag)
+            desp.setEndTime()
+            self.__statusList.append(desp.getStatus())
+            return True
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return False

rcsb/exdb/utils/ObjectUpdater.py

@@ -0,0 +1,121 @@
+##
+# File: ObjectUpdater.py
+# Date: 9-Oct-2019  jdw
+#
+# Utilities to update document features from the document object server.
+#
+# Updates:
+#
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+
+from rcsb.db.mongo.Connection import Connection
+from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
+
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectUpdater(object):
+    """Utilities to update document features from the document object server."""
+
+    def __init__(self, cfgOb, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__resourceName = "MONGO_DB"
+        _ = kwargs
+        #
+
+    def update(self, databaseName, collectionName, updateDL):
+        """Update documents satisfying the selection details with the content of updateDL.
+
+        Args:
+            databaseName (str): Target database name
+            collectionName (str): Target collection name
+            updateDL = [{selectD: ..., updateD: ... }, ....]
+                selectD    = {'ky1': 'val1', 'ky2': 'val2',  ...}
+                updateD = {'key1.subkey1...': 'val1', 'key2.subkey2..': 'val2', ...}
+
+        """
+        try:
+            numUpdated = 0
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.debug("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    for updateD in updateDL:
+                        num = mg.update(databaseName, collectionName, updateD["updateD"], updateD["selectD"], upsertFlag=True)
+                        numUpdated += num
+
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return numUpdated
+
+    def count(self, databaseName, collectionName):
+        try:
+            numTotal = 0
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    numTotal = mg.count(databaseName, collectionName)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return numTotal
+
+    def createCollection(self, databaseName, collectionName, indexAttributeNames=None, indexName="primary", checkExists=False, bsonSchema=None):
+        """Create collection and optionally set index attributes for the named index and validation schema for a new collection.
+
+        Args:
+            databaseName (str): target database name
+            collectionName (str): target collection name
+            indexAttributeNames (list, optional): list of attribute names for the 'primary' index. Defaults to None.
+            checkExists (bool, optional): reuse an existing collection if True. Defaults to False.
+            bsonSchema (object, optional): BSON compatable validation schema. Defaults to None.
+
+        Returns:
+            (bool): True for success or False otherwise
+        """
+        try:
+            logger.debug("Create database %s collection %s", databaseName, collectionName)
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if checkExists and mg.databaseExists(databaseName) and mg.collectionExists(databaseName, collectionName):
+                    ok1 = True
+                else:
+                    ok1 = mg.createCollection(databaseName, collectionName, bsonSchema=bsonSchema)
+                ok2 = mg.databaseExists(databaseName)
+                ok3 = mg.collectionExists(databaseName, collectionName)
+                okI = True
+                if indexAttributeNames:
+                    okI = mg.createIndex(databaseName, collectionName, indexAttributeNames, indexName=indexName, indexType="DESCENDING", uniqueFlag=False)
+
+            return ok1 and ok2 and ok3 and okI
+            #
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return False
+
+    def delete(self, databaseName, collectionName, selectD):
+        """Remove documents satisfying the input selection details.
+
+        Args:
+            databaseName (str): Target database name
+            collectionName (str): Target collection name
+            selectD    = {'ky1': 'val1', 'ky2': 'val2',  ...}
+
+        """
+        try:
+            numDeleted = 0
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.info("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    numDeleted = mg.delete(databaseName, collectionName, selectD)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return numDeleted

rcsb/exdb/utils/ObjectValidator.py

@@ -0,0 +1,160 @@
+##
+# File: ObjectValidator.py
+# Date: 17-Oct-2019  jdw
+#
+# Utilities to extract and update object from the document object server including validation.
+#
+# Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+
+from jsonschema import Draft4Validator
+from jsonschema import FormatChecker
+
+from rcsb.db.mongo.Connection import Connection
+from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
+from rcsb.db.processors.DataExchangeStatus import DataExchangeStatus
+from rcsb.db.utils.SchemaProvider import SchemaProvider
+from rcsb.db.utils.TimeUtil import TimeUtil
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectValidator(object):
+    """Utilities to extract and update object from the document object server with validation."""
+
+    def __init__(self, cfgOb, objectAdapter=None, cachePath=".", useCache=True, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__oAdapt = objectAdapter
+        self.__resourceName = "MONGO_DB"
+        _ = kwargs
+        self.__statusList = []
+        self.__schP = SchemaProvider(self.__cfgOb, cachePath, useCache=useCache)
+        self.__valInst = None
+
+    def __getValidator(self, databaseName, collectionName, schemaLevel="full"):
+        _ = self.__schP.makeSchemaDef(databaseName, dataTyping="ANY", saveSchema=True)
+        cD = self.__schP.makeSchema(databaseName, collectionName, encodingType="JSON", level=schemaLevel, saveSchema=True)
+        # Raises exceptions for schema compliance.
+        Draft4Validator.check_schema(cD)
+        valInst = Draft4Validator(cD, format_checker=FormatChecker())
+        return valInst
+
+    def __validateObj(self, databaseName, collectionName, rObj, label=""):
+        try:
+            eCount = 0
+            tId = rObj["rcsb_id"] if rObj and "rcsb_id" in rObj else "anonymous"
+            for error in sorted(self.__valInst.iter_errors(rObj), key=str):
+                logger.info("Database %s collection %s (%s %r) path %s error: %s", databaseName, collectionName, label, tId, error.path, error.message)
+                logger.debug(">>> Failing object is %r", rObj)
+                eCount += 1
+        except Exception as e:
+            logger.exception("Validation failing %s", str(e))
+
+        return eCount
+
+    def doTransform(self, **kwargs):
+        desp = DataExchangeStatus()
+        statusStartTimestamp = desp.setStartTime()
+        #
+        databaseName = kwargs.get("databaseName", "pdbx_core")
+        collectionName = kwargs.get("collectionName", "pdbx_core_entry")
+        selectionQueryD = kwargs.get("selectionQuery", {})
+        fetchLimit = kwargs.get("fetchLimit", None)
+        #
+
+        #
+        tU = TimeUtil()
+        updateId = kwargs.get("updateId", tU.getCurrentWeekSignature())
+        #
+        docSelectList = self.__selectObjectIds(databaseName, collectionName, selectionQueryD)
+        docSelectList = docSelectList[:fetchLimit] if fetchLimit else docSelectList
+
+        ok = self.__transform(databaseName, collectionName, docSelectList)
+        #
+        okS = True
+        if updateId:
+            okS = self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
+        return ok and okS
+
+    def __selectObjectIds(self, databaseName, collectionName, selectionQueryD):
+        """Return a list of object identifiers for the input selection query."""
+        try:
+
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    logger.info("%s %s document count is %d", databaseName, collectionName, mg.count(databaseName, collectionName))
+                    qD = {}
+                    if selectionQueryD:
+                        qD.update(selectionQueryD)
+                    selectL = ["_id"]
+                    dL = mg.fetch(databaseName, collectionName, selectL, queryD=qD)
+                    logger.info("Selection %r fetch result count %d", selectL, len(dL))
+
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return dL
+        #
+
+    def __transform(self, databaseName, collectionName, docSelectList, logIncrement=100):
+        """Return a list of object identifiers for the input selection query."""
+        #
+        ok = True
+        try:
+            self.__valInst = self.__getValidator(databaseName, collectionName, schemaLevel="full")
+            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+                mg = MongoDbUtil(client)
+                if mg.collectionExists(databaseName, collectionName):
+                    numDoc = len(docSelectList)
+                    for ii, dD in enumerate(docSelectList, 1):
+                        if "_id" not in dD:
+                            continue
+                        rObj = mg.fetchOne(databaseName, collectionName, "_id", dD["_id"])
+                        del rObj["_id"]
+                        #
+                        fOk = True
+
+                        if self.__oAdapt:
+                            self.__validateObj(databaseName, collectionName, rObj, label="Original")
+                            fOk, rObj = self.__oAdapt.filter(rObj)
+                            self.__validateObj(databaseName, collectionName, rObj, label="Updated")
+                        if fOk:
+                            rOk = mg.replace(databaseName, collectionName, rObj, dD)
+                            if rOk is None:
+                                tId = rObj["rcsb_id"] if rObj and "rcsb_id" in rObj else "anonymous"
+                                logger.error("%r %r (%r) failing", databaseName, collectionName, tId)
+                                # logger.info("rObj.keys() %r", list(rObj.keys()))
+                                # logger.info("rObj.items() %s", rObj.items())
+                                rOk = False
+                            ok = ok and rOk
+                        #
+                        if ii % logIncrement == 0 or ii == numDoc:
+                            logger.info("Replace status %r object (%d of %d)", ok, ii, numDoc)
+                        #
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok
+
+    def getLoadStatus(self):
+        return self.__statusList
+
+    def __updateStatus(self, updateId, databaseName, collectionName, status, startTimestamp):
+        try:
+            sFlag = "Y" if status else "N"
+            desp = DataExchangeStatus()
+            desp.setStartTime(tS=startTimestamp)
+            desp.setObject(databaseName, collectionName)
+            desp.setStatus(updateId=updateId, successFlag=sFlag)
+            desp.setEndTime()
+            self.__statusList.append(desp.getStatus())
+            return True
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return False