rcsb.exdb 1.31__py3-none-any.whl

rcsb/exdb/citation/CitationAdapter.py

@@ -0,0 +1,91 @@
+##
+# File: CitationAdapter.py
+# Date: 21-Nov-2019  jdw
+#
+# Selected utilities to update entry citations in the core_entry collection.
+#
+# Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import copy
+import logging
+from string import capwords
+
+from rcsb.exdb.utils.ObjectAdapterBase import ObjectAdapterBase
+
+logger = logging.getLogger(__name__)
+
+
+class CitationAdapter(ObjectAdapterBase):
+    """Selected utilities to update entry citations in the core_entry collection."""
+
+    def __init__(self, citationReferenceProvider, journalTitleAbbreviationProvider):
+        super(CitationAdapter, self).__init__()
+        #
+        self.__crP = citationReferenceProvider
+        self.__jtaP = journalTitleAbbreviationProvider
+
+    def filter(self, obj, **kwargs):
+        isTestMode = True
+        if isTestMode:
+            _, _ = self.__filter(copy.deepcopy(obj))
+            return True, obj
+        else:
+            return self.__filter(obj)
+
+    def __filter(self, obj):
+        ok = True
+        try:
+            rcsbId = obj["rcsb_id"]
+            if "citation" in obj:
+                for citObj in obj["citation"]:
+                    if citObj["id"].upper() != "PRIMARY":
+                        continue
+                    issn = citObj["journal_id_ISSN"] if "journal_id_ISSN" in citObj else None
+                    curAbbrev = citObj["journal_abbrev"] if "journal_abbrev" in citObj else None
+                    revAbbrev = self.__updateJournalAbbreviation(rcsbId, issn, curAbbrev)
+                    logger.debug("%s: revised: %r current: %r", rcsbId, revAbbrev, curAbbrev)
+
+        except Exception as e:
+            ok = False
+            logger.exception("Filter adapter failing with error with %s", str(e))
+        #
+        return ok, obj
+
+    def __updateJournalAbbreviation(self, rcsbId, issn, curAbbrev):
+        try:
+            revAbbrev = None
+            if issn:
+                medlineAbbrev = self.__crP.getMedlineJournalAbbreviation(issn)
+                # medlineIsoAbbrev = self.__crP.getMedlineJournalIsoAbbreviation(issn)
+                crIssn = issn.replace("-", "")
+                crTitle = self.__crP.getCrossRefJournalTitle(crIssn)
+                #
+                revAbbrev = medlineAbbrev
+                if not medlineAbbrev and not crTitle:
+                    logger.debug("%s: missing information for issn %r curAbbrev %r", rcsbId, issn, curAbbrev)
+                    revAbbrev = capwords(curAbbrev.replace(".", " "))
+                elif not medlineAbbrev:
+                    revAbbrev = self.__jtaP.getJournalAbbreviation(crTitle, usePunctuation=False)
+            else:
+                if curAbbrev.upper() in ["TO BE PUBLISHED", "IN PREPARATION"]:
+                    revAbbrev = "To be published"
+                elif curAbbrev.upper().startswith("THESIS"):
+                    revAbbrev = "Thesis"
+                else:
+                    revAbbrev = capwords(curAbbrev.replace(".", " "))
+                    logger.debug("%r: missing issn and non-standard abbrev for %r", rcsbId, curAbbrev)
+
+                if not curAbbrev:
+                    logger.info("%r: missing issn and journal abbrev", rcsbId)
+                #
+            logger.debug("%s: revised: %r current: %r", rcsbId, revAbbrev, curAbbrev)
+        except Exception as e:
+            logger.exception("Failing on %r %r %r with %r", rcsbId, issn, curAbbrev, str(e))
+
+        return revAbbrev

rcsb/exdb/citation/CitationExtractor.py

@@ -0,0 +1,190 @@
+##
+# File: CitationExtractor.py
+# Date: 19-Feb-2019  jdw
+#
+# Selected utilities to extract citation data from the core_entry exchange database schema.
+#
+# Updates:
+#
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+# import copy
+import logging
+import os
+
+from rcsb.exdb.utils.ObjectExtractor import ObjectExtractor
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+logger = logging.getLogger(__name__)
+
+
+class CitationExtractor(object):
+    """Utilities to extract citation related data from the core_entry collection."""
+
+    def __init__(self, cfgOb, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__resourceName = "MONGO_DB"
+        self.__databaseName = "pdbx_core"
+        self.__collectionName = "pdbx_core_entry"
+        #
+        self.__mU = MarshalUtil()
+        #
+        self.__entryD = self.__rebuildCache(**kwargs)
+        self.__idxD = self.__buildIndices(self.__entryD)
+        #
+
+    def __rebuildCache(self, **kwargs):
+        useCache = kwargs.get("useCache", True)
+        dirPath = kwargs.get("exdbDirPath", ".")
+        cacheKwargs = kwargs.get("cacheKwargs", {"fmt": "pickle"})
+        #
+        ext = "pic" if cacheKwargs["fmt"] == "pickle" else "json"
+        fn = "entry-citation-extracted-data-cache" + "." + ext
+        cacheFilePath = os.path.join(dirPath, fn)
+
+        cD = {"entryD": {}}
+        try:
+            if useCache and cacheFilePath and os.access(cacheFilePath, os.R_OK):
+                logger.info("Using cached entry citation file %s", cacheFilePath)
+                cD = self.__mU.doImport(cacheFilePath, **cacheKwargs)
+            else:
+                entryD = self.__extractCitations()
+                cD["entryD"] = entryD
+                if cacheFilePath:
+                    ok = self.__mU.mkdir(dirPath)
+                    ok = self.__mU.doExport(cacheFilePath, cD, **cacheKwargs)
+                    logger.info("Saved entry citation results (%d) status %r in %s", len(entryD), ok, cacheFilePath)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return cD["entryD"]
+
+    def __buildIndices(self, entryD):
+        """
+        Example:
+         "entryD": {
+                      "5KAL": {
+                         "citation": [
+                            {
+                               "country": "UK",
+                               "id": "primary",
+                               "journal_abbrev": "Nucleic Acids Res.",
+                               "journal_id_ASTM": "NARHAD",
+                               "journal_id_CSD": "0389",
+                               "journal_id_ISSN": "1362-4962",
+                               "journal_volume": "44",
+                               "page_first": "10862",
+                               "page_last": "10878",
+                               "title": "RNA Editing TUTase 1: structural foundation of substrate recognition, complex interactions and drug targeting.",
+                               "year": 2016,
+                               "pdbx_database_id_DOI": "10.1093/nar/gkw917",
+                               "pdbx_database_id_PubMed": 27744351,
+                               "rcsb_authors": [
+                                  "Rajappa-Titu, L.",
+                                  "Suematsu, T.",
+                                  "Munoz-Tello, P.",
+                                  "Long, M.",
+                                  "Demir, O.",
+                                  "Cheng, K.J.",
+                                  "Stagno, J.R.",
+                                  "Luecke, H.",
+                                  "Amaro, R.E.",
+                                  "Aphasizheva, I.",
+                                  "Aphasizhev, R.",
+                                  "Thore, S."
+                               ]
+                            }
+                         ],
+                         "_entry_id": "5KAL"
+                      },
+        """
+        indD = {}
+        missingCitationCount = 0
+        missingJournalName = 0
+        numPubMed = 0
+        numDOI = 0
+        numCitations = 0
+        mD = {}
+        issnD = {}
+        missingISSNCount = 0
+        missingPubMedCount = 0
+        try:
+            for entryId, eD in entryD.items():
+                cDL = eD["citation"] if "citation" in eD else None
+                if cDL:
+                    for cD in cDL[:1]:
+                        if cD and "journal_abbrev" in cD:
+                            indD[cD["journal_abbrev"]] = indD[cD["journal_abbrev"]] + 1 if cD["journal_abbrev"] in indD else 1
+                        else:
+                            logger.info("Missing journal name in entryId %s %r ", entryId, cD)
+                            missingJournalName += 1
+                        if cD and "pdbx_database_id_DOI" in cD:
+                            numDOI += 1
+
+                        if cD and "pdbx_database_id_PubMed" in cD:
+                            numPubMed += 1
+                        else:
+                            mD[cD["journal_abbrev"]] = mD[cD["journal_abbrev"]] + 1 if cD["journal_abbrev"] in mD else 1
+                            missingPubMedCount += 1
+
+                        if "journal_id_ISSN" in cD and len(cD["journal_id_ISSN"]) > 7:
+                            issnD[cD["journal_id_ISSN"]] = issnD[cD["journal_id_ISSN"]] + 1 if cD["journal_id_ISSN"] in issnD else 1
+                        else:
+                            missingISSNCount += 1
+
+                        if cD:
+                            numCitations += 1
+                else:
+                    missingCitationCount += 1
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        #
+        logger.info("Number of citatons %d", numCitations)
+        logger.info("Number of PubMed ids %d", numPubMed)
+        logger.info("Number of DOIs %d", numDOI)
+        logger.info("No citation category count %d missing journal name %d", missingCitationCount, missingJournalName)
+        #
+        logger.info("Journal index name length %d", len(indD))
+        # logger.info("Journal name length %r",indD.items())
+        #
+        logger.info("Missing pubmed index length %d", len(mD))
+        logger.info("Missing pubmed length %d", missingPubMedCount)
+        logger.info("Missing PubMed %r", mD.items())
+        #
+        logger.info("ISSN dictionary length %d", len(issnD))
+        logger.info("ISSN missing length %d", missingISSNCount)
+        #
+        return indD
+
+    def getEntryCount(self):
+        return len(self.__entryD)
+
+    def __extractCitations(self):
+        """Test case - extract unique entity source and host taxonomies"""
+        try:
+            obEx = ObjectExtractor(
+                self.__cfgOb,
+                databaseName=self.__databaseName,
+                collectionName=self.__collectionName,
+                cacheFilePath=None,
+                useCache=False,
+                keyAttribute="entry",
+                uniqueAttributes=["rcsb_id"],
+                cacheKwargs=None,
+                objectLimit=None,
+                selectionQuery={},
+                selectionList=["rcsb_id", "citation"],
+            )
+            eCount = obEx.getCount()
+            logger.info("Entry count is %d", eCount)
+            objD = obEx.getObjects()
+            # for ky, eD in objD.items():
+            #    logger.info("%s: %r", ky, eD)
+            return objD
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return {}

rcsb/exdb/citation/CitationUtils.py

@@ -0,0 +1,51 @@
+##
+# File: CitationExtractor.py
+# Date: 19-Feb-2019  jdw
+#
+# Selected utilities to process and normalize PDB citation data.
+#
+# Updates:
+#
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+
+from rcsb.exdb.citation.CitationExtractor import CitationExtractor
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+
+logger = logging.getLogger(__name__)
+
+
+class CitationUtils(object):
+    """Utilities to process and normalize PDB citation data."""
+
+    def __init__(self, cfgOb, **kwargs):
+        self.__cfgOb = cfgOb
+        self.__mU = MarshalUtil()
+        #
+        self.__ce = self.__getEntryCitations(**kwargs)
+
+    def getCitationEntryCount(self):
+        return self.__ce.getEntryCount()
+
+    def __getEntryCitations(self, **kwargs):
+        """Extract entry citations"""
+        ce = None
+        exdbDirPath = kwargs.get("exdbDirPath", None)
+        saveKwargs = kwargs.get("cacheKwargs", {"fmt": "pickle"})
+        useCache = kwargs.get("useCache", True)
+        entryLimit = kwargs.get("entryLimit", True)
+        try:
+            ce = CitationExtractor(self.__cfgOb, exdbDirPath=exdbDirPath, useCache=useCache, cacheKwargs=saveKwargs, entryLimit=entryLimit)
+            eCount = ce.getEntryCount()
+            logger.info("Using citation data for %d entries", eCount)
+            #
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ce

rcsb/exdb/entry/EntryInfoProvider.py

@@ -0,0 +1,148 @@
+##
+#  File:           EntryInfoProvider.py
+#  Date:           22-Sep-2021 jdw
+#
+#  Updated:
+#
+##
+"""
+Accessors for entry-level annotations.
+
+"""
+
+import logging
+import os.path
+import time
+
+from rcsb.exdb.utils.ObjectExtractor import ObjectExtractor
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+from rcsb.utils.io.StashableBase import StashableBase
+
+logger = logging.getLogger(__name__)
+
+
+class EntryInfoProvider(StashableBase):
+    """Accessors and generators for entry-level annotations."""
+
+    def __init__(self, **kwargs):
+        #
+        self.__version = "0.50"
+        cachePath = kwargs.get("cachePath", ".")
+        useCache = kwargs.get("useCache", True)
+        self.__dirName = "rcsb_entry_info"
+        self.__dirPath = os.path.join(cachePath, self.__dirName)
+        super(EntryInfoProvider, self).__init__(cachePath, [self.__dirName])
+        #
+        self.__mU = MarshalUtil(workPath=self.__dirPath)
+        self.__entryInfoD = self.__reload(fmt="json", useCache=useCache)
+        #
+
+    def testCache(self, minCount=1):
+        if minCount == 0:
+            return True
+        if self.__entryInfoD and minCount and "entryInfo" in self.__entryInfoD and len(self.__entryInfoD["entryInfo"]) > minCount:
+            logger.info("Entry annotations for (%d) entries", len(self.__entryInfoD["entryInfo"]))
+            return True
+        return False
+
+    def getEntryInfo(self, entryId):
+        """Return a dictionary of entry-level annotations.
+
+        Returns:
+            (dict): of entry-level annotations
+        """
+        try:
+            return self.__entryInfoD["entryInfo"][entryId.upper()] if entryId.upper() in self.__entryInfoD["entryInfo"] else {}
+        except Exception as e:
+            logger.error("Failing with %r", str(e))
+        return {}
+
+    def getEntriesByPolymerEntityCount(self, count):
+        oL = []
+        try:
+            for entryId, eD in self.__entryInfoD["entryInfo"].items():
+                if eD["polymer_entity_count"] == count:
+                    oL.append(entryId)
+        except Exception as e:
+            logger.error("Failing with %r", str(e))
+        return oL
+
+    def __getEntryInfoFilePath(self, fmt="json"):
+        baseFileName = "entry_info_details"
+        fExt = ".json" if fmt == "json" else ".pic"
+        fp = os.path.join(self.__dirPath, baseFileName + fExt)
+        return fp
+
+    def update(self, cfgOb, fmt="json", indent=3):
+        """Update branched entity glycan accession mapping cache.
+
+        Args:
+            cfgObj (object): ConfigInfo() object instance
+
+        Returns:
+            (bool): True for success for False otherwise
+        """
+        ok = False
+        try:
+            entryInfoD = self.__updateEntryInfo(cfgOb)
+
+            logger.info("Got entry_info for (%d)", len(entryInfoD))
+            #
+            tS = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
+            self.__entryInfoD = {"version": self.__version, "created": tS, "entryInfo": entryInfoD}
+            #
+            infoFilePath = self.__getEntryInfoFilePath(fmt=fmt)
+            kwargs = {"indent": indent} if fmt == "json" else {}
+            ok = self.__mU.doExport(infoFilePath, self.__entryInfoD, fmt=fmt, **kwargs)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok
+
+    def reload(self):
+        """Reload from the current cache file."""
+        ok = False
+        try:
+            self.__entryInfoD = self.__reload(fmt="json", useCache=True)
+            ok = self.__entryInfoD is not None
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok
+
+    def __reload(self, fmt="json", useCache=True):
+        entryInfoFilePath = self.__getEntryInfoFilePath(fmt=fmt)
+        tS = time.strftime("%Y %m %d %H:%M:%S", time.localtime())
+        pcD = {"version": self.__version, "created": tS, "identifiers": {}}
+
+        if useCache and self.__mU.exists(entryInfoFilePath):
+            logger.info("Reading entry-info cached path %r", entryInfoFilePath)
+            pcD = self.__mU.doImport(entryInfoFilePath, fmt=fmt)
+        return pcD
+
+    def __updateEntryInfo(self, cfgOb):
+        """Get entry_info data"""
+        rD = {}
+        try:
+            obEx = ObjectExtractor(
+                cfgOb,
+                databaseName="pdbx_core",
+                collectionName="pdbx_core_entry",
+                useCache=False,
+                keyAttribute="entry",
+                uniqueAttributes=["rcsb_id"],
+                selectionQuery={},
+                selectionList=["rcsb_id", "rcsb_entry_info.polymer_entity_count"],
+            )
+            #
+            eCount = obEx.getCount()
+            logger.info("Entry count is %d", eCount)
+
+            objD = obEx.getObjects()
+            for _, eD in objD.items():
+                rcsbId = eD["rcsb_id"]
+                try:
+                    rD[rcsbId] = eD["rcsb_entry_info"]
+                except Exception:
+                    pass
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return rD