rcsb.exdb 0.96__tar.gz → 0.98__tar.gz

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/HISTORY.txt

@@ -89,4 +89,7 @@
  9-Mar-2023 V0.94 Update ExDbWorkflow to make use of multiple processors for 'upd_ref_seq' operation;
                   Lower refChunkSize to 10 for requests to UniProt API
 13-Mar-2023 V0.95 Updates to PubChem workflow to use multiprocess count, disable git stash testing, remove obsolete entries from test data
-12-Apr-2023 V0.96 Add CARD ontology data to tree builder
+12-Apr-2023 V0.96 Add CARD ontology data to tree builder
+ 1-Jun-2023 V0.97 Don't back up resources to GitHub during cache update workflows
+ 8-Aug-2023 V0.98 Reduce memory and cpu footprint for Azure test cases;
+                  Load full (unfiltered) taxonomy tree node list, and stop loading GO tree

{rcsb.exdb-0.96/rcsb.exdb.egg-info → rcsb.exdb-0.98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.exdb
-Version: 0.96
+Version: 0.98
 Summary: RCSB Python ExDB data extraction and loading workflows
 Home-page: https://github.com/rcsb/py-rcsb_exdb
 Author: John Westbrook

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/cli/__init__.py

@@ -2,4 +2,4 @@ __docformat__ = "google en"
 __author__ = "John Westbrook"
 __email__ = "john.westbrook@rcsb.org"
 __license__ = "Apache 2.0"
-__version__ = "0.96"
+__version__ = "0.98"

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/tests/fixturePdbxLoader.py

@@ -64,8 +64,8 @@ class PdbxLoaderFixture(unittest.TestCase):
         self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt")
         self.__cachePath = os.path.join(TOPDIR, "CACHE")
         self.__readBackCheck = True
-        self.__numProc = 2
-        self.__chunkSize = 10
+        self.__numProc = 1
+        self.__chunkSize = 5
         self.__fileLimit = 38
         self.__documentStyle = "rowwise_by_name_with_cardinality"
         #
@@ -172,7 +172,7 @@ class PdbxLoaderFixture(unittest.TestCase):
             {
                 "databaseName": "pdbx_core",
                 "collectionNameList": None,
-                "loadType": "full",
+                "loadType": "replace",
                 "mergeContentTypes": ["vrpt"],
                 "validationLevel": "min",
                 "inputIdCodeList": self.__pdbIdList
@@ -228,7 +228,7 @@ class PdbxLoaderFixture(unittest.TestCase):
                 fileLimit=kwargs.get("fileLimit", self.__fileLimit),
                 verbose=self.__verbose,
                 readBackCheck=self.__readBackCheck,
-                maxStepLength=2000,
+                maxStepLength=1000,
                 useSchemaCache=True,
                 rebuildSchemaFlag=False,
             )

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/tests/testExDbWorkflow.py

@@ -59,7 +59,7 @@ class ExDbWorkflowTests(unittest.TestCase):
             "restoreUseGit": True,
             "restoreUseStash": False,
         }
-        self.__loadCommonD = {"readBackCheck": True, "numProc": 2, "chunkSize": 5, "refChunkSize": 5, "loadType": "full"}
+        self.__loadCommonD = {"readBackCheck": True, "numProc": 2, "chunkSize": 5, "refChunkSize": 5, "loadType": "full", "useFilteredLists": True}
         #
         # These are test source files for chemical component/BIRD indices
         ccUrlTarget = os.path.join(self.__dataPath, "components-abbrev.cif")

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/tests/testTreeNodeListWorker.py

@@ -63,6 +63,7 @@ class TreeNodeListWorkerTests(unittest.TestCase):
         self.__debugFlag = False
         self.__loadType = "full"
         self.__useCache = True
+        self.__useFilteredLists = True
         #
         self.__startTime = time.time()
         logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
@@ -88,6 +89,7 @@ class TreeNodeListWorkerTests(unittest.TestCase):
                 verbose=self.__debugFlag,
                 readBackCheck=self.__readBackCheck,
                 useCache=self.__useCache,
+                useFilteredLists=self.__useFilteredLists,
             )
             #
             ok = rhw.load(updateId, loadType=self.__loadType, doLoad=self.__doLoad)

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/tree/TreeNodeListWorker.py

@@ -7,6 +7,7 @@
 # Updates:
 #  9-Sep-2019 jdw add AtcProvider() and ChemrefExtractor() for ATC tree.
 # 12-Apr-2023 dwp add CARD ontology tree
+#  8-Aug-2023 dwp Load full (unfiltered) taxonomy tree node list, and stop loading GO tree (will be loaded in DW instead)
 #
 ##
 __docformat__ = "google en"
@@ -20,17 +21,17 @@ import os.path
 from rcsb.db.mongo.DocumentLoader import DocumentLoader
 from rcsb.db.processors.DataExchangeStatus import DataExchangeStatus
 from rcsb.exdb.chemref.ChemRefExtractor import ChemRefExtractor
-from rcsb.exdb.seq.AnnotationExtractor import AnnotationExtractor
-from rcsb.exdb.seq.TaxonomyExtractor import TaxonomyExtractor
 from rcsb.utils.chemref.AtcProvider import AtcProvider
 from rcsb.utils.ec.EnzymeDatabaseProvider import EnzymeDatabaseProvider
 from rcsb.utils.targets.CARDTargetOntologyProvider import CARDTargetOntologyProvider
-from rcsb.utils.go.GeneOntologyProvider import GeneOntologyProvider
 from rcsb.utils.struct.CathClassificationProvider import CathClassificationProvider
 from rcsb.utils.struct.EcodClassificationProvider import EcodClassificationProvider
 from rcsb.utils.struct.ScopClassificationProvider import ScopClassificationProvider
 from rcsb.utils.struct.Scop2ClassificationProvider import Scop2ClassificationProvider
 from rcsb.utils.taxonomy.TaxonomyProvider import TaxonomyProvider
+from rcsb.exdb.seq.TaxonomyExtractor import TaxonomyExtractor
+# from rcsb.utils.go.GeneOntologyProvider import GeneOntologyProvider
+# from rcsb.exdb.seq.AnnotationExtractor import AnnotationExtractor
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
 class TreeNodeListWorker(object):
     """Prepare and load repository holdings and repository update data."""
 
-    def __init__(self, cfgOb, cachePath, numProc=1, chunkSize=10, readBackCheck=False, documentLimit=None, verbose=False, useCache=False):
+    def __init__(self, cfgOb, cachePath, numProc=1, chunkSize=10, readBackCheck=False, documentLimit=None, verbose=False, useCache=False, useFilteredLists=False):
         self.__cfgOb = cfgOb
         self.__cachePath = os.path.abspath(cachePath)
         self.__readBackCheck = readBackCheck
@@ -50,6 +51,7 @@ class TreeNodeListWorker(object):
         self.__verbose = verbose
         self.__statusList = []
         self.__useCache = useCache
+        self.__useFilteredLists = useFilteredLists
 
     def __updateStatus(self, updateId, databaseName, collectionName, status, startTimestamp):
         try:
@@ -122,19 +124,21 @@ class TreeNodeListWorker(object):
             # collectionVersion = self.__cfgOb.get("COLLECTION_VERSION_STRING", sectionName=sectionName)
             # addValues = {"_schema_version": collectionVersion}
             addValues = None
-            # --- GO
-            goP = GeneOntologyProvider(goDirPath=os.path.join(self.__cachePath, "go"), useCache=useCache)
-            ok = goP.testCache()
-            anEx = AnnotationExtractor(self.__cfgOb)
-            goIdL = anEx.getUniqueIdentifiers("GO")
-            logger.info("Unique GO assignments %d", len(goIdL))
-            nL = goP.exportTreeNodeList(goIdL)
-            logger.info("GO tree node list length %d", len(nL))
-            if doLoad:
-                collectionName = "tree_go_node_list"
-                ok = dl.load(databaseName, collectionName, loadType=loadType, documentList=nL, indexAttributeList=["update_id"], keyNames=None, addValues=addValues, schemaLevel=None)
-                self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
-                # ---- CATH
+            #
+            # --- GO - TURNED OFF 08 Aug 2023 dwp (tree is now loaded in DW)
+            # goP = GeneOntologyProvider(goDirPath=os.path.join(self.__cachePath, "go"), useCache=useCache)
+            # ok = goP.testCache()
+            # anEx = AnnotationExtractor(self.__cfgOb)
+            # goIdL = anEx.getUniqueIdentifiers("GO")
+            # logger.info("Unique GO assignments %d", len(goIdL))
+            # nL = goP.exportTreeNodeList(goIdL)
+            # logger.info("GO tree node list length %d", len(nL))
+            # if doLoad:
+            #     collectionName = "tree_go_node_list"
+            #     ok = dl.load(databaseName, collectionName, loadType=loadType, documentList=nL, indexAttributeList=["update_id"], keyNames=None, addValues=addValues, schemaLevel=None)
+            #     self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
+            #
+            # ---- CATH
             ccu = CathClassificationProvider(cachePath=self.__cachePath, useCache=useCache)
             nL = ccu.getTreeNodeList()
             logger.info("Starting load SCOP node tree length %d", len(nL))
@@ -183,19 +187,23 @@ class TreeNodeListWorker(object):
                 ok = dl.load(databaseName, collectionName, loadType=loadType, documentList=nL, indexAttributeList=["update_id"], keyNames=None, addValues=addValues, schemaLevel=None)
                 self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
             # ---- Taxonomy
-            # Get the taxon coverage in the current data set -
-            epe = TaxonomyExtractor(self.__cfgOb)
-            tL = epe.getUniqueTaxons()
-            logger.info("Taxon coverage length %d", len(tL))
-            #
             tU = TaxonomyProvider(cachePath=self.__cachePath, useCache=useCache)
-            fD = {1}
-            for taxId in tL:
-                fD.update({k: True for k in tU.getLineage(taxId)})
-            logger.info("Taxon filter dictionary length %d", len(fD))
-            # logger.info("fD %r" % sorted(fD))
-            #
-            nL = tU.exportNodeList(filterD=fD)
+            if self.__useFilteredLists:
+                # Get the taxon coverage in the current data set -
+                epe = TaxonomyExtractor(self.__cfgOb)
+                tL = epe.getUniqueTaxons()
+                logger.info("Taxon coverage length %d", len(tL))
+                #
+                fD = {1}
+                for taxId in tL:
+                    fD.update({k: True for k in tU.getLineage(taxId)})
+                logger.info("Taxon filter dictionary length %d", len(fD))
+                logger.debug("fD %r", sorted(fD))
+                #
+                nL = tU.exportNodeList(filterD=fD)
+            else:
+                # Get the full taxon node list without filtering
+                nL = tU.exportNodeList()
             self.__checkTaxonNodeList(nL)
             logger.info("Starting load of taxonomy node tree length %d", len(nL))
             if doLoad:

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/wf/EntryInfoEtlWorkflow.py

@@ -5,6 +5,7 @@
 #  Workflow wrapper  --  Entry-level annotations extracted from ExDB
 #
 #  Updates:
+#   1-Jun-2023 aae Don't back up resources to GitHub during cache update workflows
 #
 ##
 __docformat__ = "google en"
@@ -59,7 +60,7 @@ class EntryInfoEtlWorkflow(object):
             eiP.update(self.__cfgOb, fmt="json", indent=3)
             #
             if backup:
-                ok = eiP.backup(self.__cfgOb, self.__configName, self.__stashRemotePrefix, useGit=True, useStash=True)
+                ok = eiP.backup(self.__cfgOb, self.__configName, self.__stashRemotePrefix, useGit=False, useStash=True)
                 logger.info("Backup entry-level annotations (%r)", ok)
             else:
                 ok = True

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/wf/ExDbWorkflow.py

@@ -91,6 +91,7 @@ class ExDbWorkflow(object):
             rebuildSequenceCache = kwargs.get("rebuildSequenceCache", False)
             useSequenceCache = not rebuildSequenceCache
             #
+            useFilteredLists = kwargs.get("useFilteredLists", False)
         except Exception as e:
             logger.exception("Argument or configuration processing failing with %s", str(e))
             return False
@@ -107,6 +108,7 @@ class ExDbWorkflow(object):
                     verbose=self.__debugFlag,
                     readBackCheck=readBackCheck,
                     useCache=self.__useCache,
+                    useFilteredLists=useFilteredLists,
                 )
                 ok = rhw.load(dataSetId, loadType=loadType)
                 okS = self.loadStatus(rhw.getLoadStatus(), readBackCheck=readBackCheck)

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/wf/GlycanEtlWorkflow.py

@@ -5,6 +5,7 @@
 #  Workflow wrapper  --  Glycan ETL utilities
 #
 #  Updates:
+#   1-Jun-2023 aae Don't back up resources to GitHub during cache update workflows
 #
 ##
 __docformat__ = "google en"
@@ -64,7 +65,7 @@ class GlycanEtlWorkflow(object):
             logger.info("Matched glycan identifiers (%d)", len(riD))
             #
             if backup:
-                ok2 = gP.backup(self.__cfgOb, self.__configName, self.__stashRemotePrefix, useGit=True, useStash=True)
+                ok2 = gP.backup(self.__cfgOb, self.__configName, self.__stashRemotePrefix, useGit=False, useStash=True)
                 logger.info("Backup matched glycan identifiers (%r)", ok2)
             else:
                 ok2 = True

{rcsb.exdb-0.96 → rcsb.exdb-0.98}/rcsb/exdb/wf/PubChemEtlWorkflow.py

@@ -6,6 +6,7 @@
 #
 #  Updates:
 #  13-Mar-2023 aae Updates to use multiprocess count, disable git stash testing
+#   1-Jun-2023 aae Don't back up resources to GitHub during cache update workflows
 ##
 __docformat__ = "google en"
 __author__ = "John Westbrook"
@@ -168,7 +169,7 @@ class PubChemEtlWorkflow(object):
             rebuildChemIndices = kwargs.get("rebuildChemIndices", True)
             exportPath = kwargs.get("exportPath", None)
             useStash = kwargs.get("useStash", True)
-            useGit = kwargs.get("useGit", True)
+            useGit = kwargs.get("useGit", False)
             #
             pcewP = PubChemEtlWrapper(self.__cfgOb, self.__cachePath, stashRemotePrefix=self.__stashRemotePrefix)
             ok1 = pcewP.updateIndex(
@@ -207,7 +208,7 @@ class PubChemEtlWorkflow(object):
             #  --
             numProc = kwargs.get("numProc", 2)
             useStash = kwargs.get("useStash", True)
-            useGit = kwargs.get("useGit", True)
+            useGit = kwargs.get("useGit", False)
             #
             pcewP = PubChemEtlWrapper(self.__cfgOb, self.__cachePath, stashRemotePrefix=self.__stashRemotePrefix)
             ok1 = pcewP.updateMatchedData(numProc=numProc)

{rcsb.exdb-0.96 → rcsb.exdb-0.98/rcsb.exdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.exdb
-Version: 0.96
+Version: 0.98
 Summary: RCSB Python ExDB data extraction and loading workflows
 Home-page: https://github.com/rcsb/py-rcsb_exdb
 Author: John Westbrook