rcsb.exdb 1.31__py3-none-any.whl

rcsb/exdb/seq/UniProtExtractor.py

@@ -0,0 +1,80 @@
+##
+# File: UniProtExtractor.py
+# Date: 5-Dec-2020  jdw
+#
+# Utilities to extract selected details from the UniProt exchange collections.
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+
+from rcsb.exdb.utils.ObjectExtractor import ObjectExtractor
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+logger = logging.getLogger(__name__)
+
+
+class UniProtExtractor(object):
+    """Utilities to extract selected details from the UniProt exchange collections."""
+
+    def __init__(self, cfgOb):
+        self.__cfgOb = cfgOb
+
+    def exportReferenceSequenceDetails(self, filePath, fmt="json"):
+        rD = self.getReferenceSequenceDetails()
+        mU = MarshalUtil()
+        ok = mU.doExport(filePath, rD, fmt=fmt, indent=3)
+        logger.info("Exporting (%d) UniProt reference sequences (status=%r)", len(rD), ok)
+        return ok
+
+    def getReferenceSequenceDetails(self):
+        """Get reference protein sequence essential details (sequence, taxonomy, name, gene, ...)"""
+        uD = None
+        try:
+            obEx = ObjectExtractor(
+                self.__cfgOb,
+                databaseName="uniprot_exdb",
+                collectionName="reference_entry",
+                useCache=False,
+                keyAttribute="uniprot",
+                uniqueAttributes=["rcsb_id"],
+                selectionQuery={},
+                selectionList=[
+                    "source_scientific",
+                    "taxonomy_id",
+                    "rcsb_id",
+                    "gene",
+                    "names",
+                    "sequence",
+                ],
+            )
+            #
+            eCount = obEx.getCount()
+            logger.info("Reference entry count is %d", eCount)
+            objD = obEx.getObjects()
+            rD = {}
+            for rId, uD in objD.items():
+                taxId = uD["taxonomy_id"]
+                sn = uD["source_scientific"]
+                sequence = uD["sequence"]
+                gn = None
+                pn = None
+                if "gene" in uD:
+                    for tD in uD["gene"]:
+                        if tD["type"] == "primary":
+                            gn = tD["name"]
+                            break
+                for tD in uD["names"]:
+                    if tD["nameType"] == "recommendedName":
+                        pn = tD["name"]
+                        break
+                rD[rId] = {"accession": rId, "taxId": taxId, "scientific_name": sn, "gene": gn, "name": pn, "sequence": sequence}
+
+        except Exception as e:
+            logger.exception("Failing uD %r with %s", uD, str(e))
+        #
+        return rD

rcsb/exdb/tests/TEST-EXDB-CLI-EXEC.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# File: TEST-EXDB-CL-EXEC.sh
+# Date: 3-Sep-2019 jdw
+#
+# Examples
+#
+# tree node list load
+#
+exdb_exec_cli --mock --full --etl_tree_node_lists --rebuild_cache --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGTREENODELIST
+#
+# Chemref load
+#
+exdb_exec_cli --mock --full --etl_chemref --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGCHEMREF
+#
+# Reference sequence update
+#
+exdb_exec_cli --mock --upd_ref_seq --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQ
+#
+#

rcsb/exdb/tests/TEST-EXDB-CLI-REFSEQ-EXEC.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# File: TEST-EXDB-CLI-REFSEQ-EXEC.sh
+# Date: 17-Oct-2019 jdw
+#
+# Reference sequence update --mock is required for example SIFTS files -
+#
+exdb_exec_cli --mock  --upd_ref_seq --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQ
+#
+exdb_exec_cli --test_req_seq_cache --mock  --upd_ref_seq --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUPDREFSEQTEST
+#
+exdb_exec_cli --mock --full --etl_uniprot --cache_path ../../../CACHE  --config_path ../../mock-data/config/dbload-setup-example.yml --config_name site_info_configuration >& ./test-output/LOGUNIPROT
+#

rcsb/exdb/tests/fixtureDictMethodResourceProvider.py

@@ -0,0 +1,104 @@
+# File:    DictMethodResourceProviderFixture.py
+# Author:  J. Westbrook
+# Date:    12-Aug-2019
+# Version: 0.001
+#
+# Update:
+
+##
+"""
+Fixture for setting up cached resources for dictionary method helpers
+
+"""
+
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+import os
+import platform
+import resource
+import time
+import unittest
+
+from rcsb.utils.config.ConfigUtil import ConfigUtil
+from rcsb.utils.dictionary.DictMethodResourceProvider import DictMethodResourceProvider
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+
+
+class DictMethodResourceProviderFixture(unittest.TestCase):
+    def setUp(self):
+        self.__cachePath = os.path.join(TOPDIR, "CACHE")
+        mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
+        configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
+        configName = "site_info_configuration"
+        self.__configName = configName
+        self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath)
+
+        self.__startTime = time.time()
+        logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
+
+    def tearDown(self):
+        unitS = "MB" if platform.system() == "Darwin" else "GB"
+        rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
+        endTime = time.time()
+        logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
+
+    def testBuildResourceCache(self):
+        """Fixture - generate and check selected resource caches"""
+        try:
+            resourceNameL = [
+                "AtcProvider instance",
+                "DrugBankProvider instance",
+                "PubChemProvider instance",
+                "CitationReferenceProvider instance",
+                "JournalTitleAbbreviationProvider instance",
+                "EnzymeDatabaseProvider instance",
+                "PfamProvider instance",
+                "SiftsSummaryProvider instance",
+                "CathProvider instance",
+                "ScopProvider instance",
+                "EcodProvider instance",
+                "Scop2Provider instance",
+                "TaxonomyProvider instance",
+            ]
+            rP = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath, restoreUseStash=False, restoreUseGit=True)
+            for resourceName in resourceNameL:
+                rP.getResource(resourceName, useCache=True, default=None, doRestore=True, doBackup=False)
+            #
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+            self.fail()
+
+    @unittest.skip("Troubleshooting test")
+    def testRecoverResourceCache(self):
+        """Fixture - generate and check resource caches"""
+        try:
+            rp = DictMethodResourceProvider(self.__cfgOb, configName=self.__configName, cachePath=self.__cachePath)
+            ret = rp.cacheResources(useCache=True)
+            self.assertTrue(ret)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+            self.fail()
+
+
+def dictMethodResourceProviderSuite():
+    suiteSelect = unittest.TestSuite()
+    suiteSelect.addTest(DictMethodResourceProviderFixture("testBuildResourceCache"))
+    # suiteSelect.addTest(DictMethodResourceProviderFixture("testRecoverResourceCache"))
+    return suiteSelect
+
+
+if __name__ == "__main__":
+
+    mySuite = dictMethodResourceProviderSuite()
+    unittest.TextTestRunner(verbosity=2).run(mySuite)

rcsb/exdb/tests/fixturePdbxLoader.py

@@ -0,0 +1,298 @@
+##
+# File:    PdbxLoaderFixture.py
+# Author:  J. Westbrook
+# Date:    4-Sep-2019
+# Version: 0.001
+#
+# Updates:
+#
+##
+"""
+Fixture for loading the chemical reference and pdbx_core collections in a loca mongo instance.
+
+"""
+
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+# import glob
+import logging
+import os
+import platform
+import resource
+import time
+import unittest
+
+from rcsb.db.mongo.DocumentLoader import DocumentLoader
+from rcsb.db.mongo.PdbxLoader import PdbxLoader
+from rcsb.utils.config.ConfigUtil import ConfigUtil
+# from rcsb.utils.io.FileUtil import FileUtil
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+
+
+class PdbxLoaderFixture(unittest.TestCase):
+
+    def __init__(self, methodName="runTest"):
+        super(PdbxLoaderFixture, self).__init__(methodName)
+        self.__verbose = True
+
+    def setUp(self):
+        #
+        #
+        self.__isMac = platform.system() == "Darwin"
+        self.__excludeTypeL = None if self.__isMac else ["optional"]
+        self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
+        configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
+        # configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example-local.yml")
+        # To Do: Investigate why GitUtil sometimes gives divergence error when using 'DISCOVERY_MODE: remote', but not with 'local':
+        #            stderr: 'fatal: Need to specify how to reconcile divergent branches.'
+        #        Behavior isn't entirely predictable, since it happens sometimes but not all the time.
+        #        To fully debug, will need to add more logging statements to GitUtil, StashableBase, & StashUtil (in rcsb.utils.io)
+        #        Or, can try to resolve error directly by specifying how to reconcile diverent branches in git.Repo class.
+        configName = "site_info_configuration"
+        self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
+        #
+        self.__resourceName = "MONGO_DB"
+        self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt")
+        self.__cachePath = os.path.join(TOPDIR, "CACHE")
+        self.__readBackCheck = True
+        self.__numProc = 1
+        self.__chunkSize = 2
+        self.__fileLimit = 38
+        self.__documentStyle = "rowwise_by_name_with_cardinality"
+        #
+        self.__birdChemCompCoreIdList = [
+            "PRD_000010",
+            "PRD_000060",
+            "PRD_000220",
+            "PRD_000882",
+            "PRD_000154",
+            "PRD_000877",
+            "PRD_000198",
+            "PRD_000009",
+            "PRD_000979",
+            "PRDCC_000010",
+            "PRDCC_000220",
+            "PRDCC_000882",
+            "PRDCC_000154",
+            "PRDCC_000198",
+            "PRDCC_000009",
+            "FAM_000010",
+            "FAM_000210",
+            "FAM_000220",
+            "FAM_000001",
+            "FAM_000391",
+            "FAM_000093",
+            "FAM_000084",
+            "FAM_000016",
+            "FAM_000336",
+            "1G1",
+            "2RT",
+            "2XL",
+            "2XN",
+            "ATP",
+            "BJA",
+            "BM3",
+            "CNC",
+            "DAL",
+            "DDZ",
+            "DHA",
+            "DSN",
+            "GTP",
+            "HKL",
+            "NAC",
+            "NAG",
+            "NND",
+            "PTR",
+            "SEP",
+            "SMJ",
+            "STL",
+            "UNK",
+            "UNX",
+            "UVL",
+        ]
+        #
+        self.__pdbIdList = [
+            "1AH1",
+            "1B5F",
+            "1BMV",
+            "1C58",
+            "1DSR",
+            "1DUL",
+            "1KQE",
+            "1O3Q",
+            "1SFO",
+            "2HW3",
+            "2HYV",
+            "2OSL",
+            "2VOO",
+            "2WMG",
+            "3AD7",
+            "3HYA",
+            "3IYD",
+            "3MBG",
+            "3RER",
+            "3VD8",
+            "3VFJ",
+            "3X11",
+            "3ZTJ",
+            "4E2O",
+            "4EN8",
+            "4MEY",
+            "5EU8",
+            "5KDS",
+            # "5TM0",
+            "5VH4",
+            # "5VP2",
+            # "6FSZ",
+            "6LU7",
+            "6NN7",
+            # "6Q20",
+            "6RFK",
+            "6RKU",
+            "6YRQ",
+        ]
+        self.__ldList = [
+            {
+                # "databaseName": "dw",
+                "collectionGroupName": "core_chem_comp",
+                "contentType": "bird_chem_comp_core",
+                "collectionNameList": None,
+                "loadType": "full",
+                "mergeContentTypes": None,
+                "validationLevel": "min",
+                "inputIdCodeList": self.__birdChemCompCoreIdList
+            },
+            {
+                # "databaseName": "pdbx_core",
+                "collectionGroupName": "pdbx_core",
+                "contentType": "pdbx_core",
+                "collectionNameList": None,
+                "loadType": "replace",
+                "mergeContentTypes": ["vrpt"],
+                "validationLevel": "min",
+                "inputIdCodeList": self.__pdbIdList
+            },
+            # {
+            #     "databaseName": "pdbx_comp_model_core",
+            #     "collectionGroupName": "pdbx_comp_model_core",
+            #     "contentType": "pdbx_comp_model_core",
+            #     "collectionNameList": None,
+            #     "loadType": "full",
+            #     "mergeContentTypes": None,
+            #     "validationLevel": "min",
+            #     "inputIdCodeList": None
+            # },
+        ]
+        #
+        # self.__modelFixture()
+        self.__startTime = time.time()
+        logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
+
+    def tearDown(self):
+        unitS = "MB" if platform.system() == "Darwin" else "GB"
+        rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
+        endTime = time.time()
+        logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
+
+    # def __modelFixture(self):
+    #     fU = FileUtil()
+    #     modelSourcePath = os.path.join(self.__mockTopPath, "AF")
+    #     for iPath in glob.iglob(os.path.join(modelSourcePath, "*.cif.gz")):
+    #         fn = os.path.basename(iPath)
+    #         uId = fn.split("-")[1]
+    #         h3 = uId[-2:]
+    #         h2 = uId[-4:-2]
+    #         h1 = uId[-6:-4]
+    #         oPath = os.path.join(self.__cachePath, "computed-models", h1, h2, h3, fn)
+    #         fU.put(iPath, oPath)
+
+    def testPdbxLoader(self):
+        #
+        for ld in self.__ldList:
+            ok = self.__pdbxLoaderWrapper(**ld)
+            self.assertTrue(ok)
+
+    def __pdbxLoaderWrapper(self, **kwargs):
+        """Wrapper for the PDBx loader module"""
+        ok = False
+        try:
+            logger.info("Loading %s", kwargs["collectionGroupName"])
+            mw = PdbxLoader(
+                self.__cfgOb,
+                cachePath=self.__cachePath,
+                resourceName=self.__resourceName,
+                numProc=self.__numProc,
+                chunkSize=self.__chunkSize,
+                fileLimit=kwargs.get("fileLimit", self.__fileLimit),
+                verbose=self.__verbose,
+                readBackCheck=self.__readBackCheck,
+                maxStepLength=1000,
+                useSchemaCache=True,
+                rebuildSchemaFlag=False,
+            )
+            ok = mw.load(
+                collectionGroupName=kwargs["collectionGroupName"],
+                collectionLoadList=kwargs["collectionNameList"],
+                contentType=kwargs["contentType"],
+                loadType=kwargs["loadType"],
+                inputPathList=None,
+                inputIdCodeList=kwargs["inputIdCodeList"],
+                styleType=self.__documentStyle,
+                dataSelectors=["PUBLIC_RELEASE"],
+                failedFilePath=self.__failedFilePath,
+                saveInputFileListPath=None,
+                pruneDocumentSize=None,
+                logSize=False,
+                validationLevel=kwargs["validationLevel"],
+                mergeContentTypes=kwargs["mergeContentTypes"],
+                useNameFlag=False,
+                providerTypeExcludeL=self.__excludeTypeL,
+                restoreUseGit=True,
+                restoreUseStash=False,
+            )
+            self.assertTrue(ok)
+            ok = self.__loadStatus(mw.getLoadStatus())
+            self.assertTrue(ok)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+            self.fail()
+        return ok
+
+    def __loadStatus(self, statusList):
+        sectionName = "data_exchange_configuration"
+        dl = DocumentLoader(
+            self.__cfgOb,
+            self.__cachePath,
+            resourceName=self.__resourceName,
+            numProc=self.__numProc,
+            chunkSize=self.__chunkSize,
+            documentLimit=None,
+            verbose=self.__verbose,
+            readBackCheck=self.__readBackCheck,
+        )
+        #
+        databaseName = self.__cfgOb.get("DATABASE_NAME", sectionName=sectionName)
+        collectionName = self.__cfgOb.get("COLLECTION_UPDATE_STATUS", sectionName=sectionName)
+        ok = dl.load(databaseName, collectionName, loadType="append", documentList=statusList, indexAttributeList=["update_id", "database_name", "object_name"], keyNames=None)
+        return ok
+
+
+def mongoLoadPdbxSuite():
+    suiteSelect = unittest.TestSuite()
+    suiteSelect.addTest(PdbxLoaderFixture("testPdbxLoader"))
+    return suiteSelect
+
+
+if __name__ == "__main__":
+    mySuite = mongoLoadPdbxSuite()
+    unittest.TextTestRunner(verbosity=2).run(mySuite)