rcsb.workflow 0.41__tar.gz → 0.43__tar.gz

{rcsb.workflow-0.41 → rcsb_workflow-0.43}/HISTORY.txt

@@ -31,3 +31,7 @@
 22-May-2023 - V0.39 Add retries to tox task for MMseqs2 download
  1-Jun-2023 - V0.40 Don't back up resources to GitHub during cache update workflows
 12-Jun-2023 - V0.41 Set useTaxonomy filter to False for CARD annotations
+19-Sep-2023 - V0.42 Don't overwrite Buildlocker files if there is no data;
+                    Reduce resource footprint for Azure tests;
+                    Make CARDTargetOntologyProvider stashable on Buildlocker
+21-May-2024 - V0.43 Fix pylinting

{rcsb.workflow-0.41/rcsb.workflow.egg-info → rcsb_workflow-0.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.workflow
-Version: 0.41
+Version: 0.43
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Home-page: https://github.com/rcsb/py-rcsb_workflow
 Author: John Westbrook
@@ -14,9 +14,18 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: OpenEye-toolkits>=2020.2.2
+Requires-Dist: rcsb.utils.chem>=0.75
+Requires-Dist: rcsb.utils.seq>=0.63
+Requires-Dist: rcsb.utils.targets>=0.57
+Requires-Dist: rcsb.utils.seqalign>=0.24
+Requires-Dist: rcsb.exdb>=0.87
+Requires-Dist: rcsb.utils.io>=1.20
 Provides-Extra: dev
+Requires-Dist: check-manifest; extra == "dev"
 Provides-Extra: test
-License-File: LICENSE
+Requires-Dist: coverage; extra == "test"
 
 # py-rcsb_workflow
 

{rcsb.workflow-0.41 → rcsb_workflow-0.43}/rcsb/workflow/chem/__init__.py

@@ -2,4 +2,4 @@ __docformat__ = "google en"
 __author__ = "John Westbrook"
 __email__ = "john.westbrook@rcsb.org"
 __license__ = "Apache 2.0"
-__version__ = "0.41"
+__version__ = "0.43"

{rcsb.workflow-0.41 → rcsb_workflow-0.43}/rcsb/workflow/targets/ProteinTargetSequenceExecutionWorkflow.py

@@ -227,20 +227,32 @@ def fullWorkflow():
     """Entry point for the full targets sequence and cofactor update workflow."""
     ptsWf = ProteinTargetSequenceExecutionWorkflow()
     ok = ptsWf.cacheTaxonomy()
+    logger.info("cacheTaxonomy status %r", ok)
     ok = ptsWf.updateUniProtTaxonomy() and ok
+    logger.info("updateUniProtTaxonomy status %r", ok)
     ok = ptsWf.fetchProteinEntityData() and ok
+    logger.info("fetchProteinEntityData status %r", ok)
     ok = ptsWf.fetchChemicalReferenceMappingData() and ok
+    logger.info("fetchChemicalReferenceMappingData status %r", ok)
     ok = ptsWf.fetchLigandNeighborMappingData() and ok
+    logger.info("fetchLigandNeighborMappingData status %r", ok)
     ok = ptsWf.exportFasta() and ok
+    logger.info("exportFasta status %r", ok)
     ok = ptsWf.createSearchDatabases() and ok
+    logger.info("createSearchDatabases status %r", ok)
     ok = ptsWf.searchDatabases() and ok
+    logger.info("searchDatabases status %r", ok)
     ok = ptsWf.buildFeatures() and ok
+    logger.info("buildFeatures status %r", ok)
+    # To rebuild ChEMBL-target-activity data from scratch (non-incremental), set skip=None in fetchTargetActivityDataMulti()
     ok = ptsWf.buildActivityData() and ok
+    logger.info("buildActivityData status %r", ok)
     ok = ptsWf.buildCofactorData() and ok
+    logger.info("buildCofactorData status %r", ok)
     ptsWf.resourceCheck()
     return ok
 
 
 if __name__ == "__main__":
     status = fullWorkflow()
-    print("Full workflow completion status (%r)", status)
+    logger.info("Full workflow completion status (%r)", status)

{rcsb.workflow-0.41 → rcsb_workflow-0.43}/rcsb/workflow/targets/ProteinTargetSequenceWorkflow.py

@@ -14,6 +14,7 @@
 #   5-May-2023 Restore from stash if fromDbPharos and reloadPharos parameters are False
 #   1-Jun-2023 aae Don't back up resources to GitHub during cache update workflows
 #  12-Jun-2023 dwp Set useTaxonomy filter to False for CARD annotations
+#   6-Jul-2023 aae Don't overwrite Buildlocker files if there is no data
 ##
 __docformat__ = "google en"
 __author__ = "John Westbrook"
@@ -33,6 +34,7 @@ from rcsb.utils.seqalign.MMseqsUtils import MMseqsUtils
 from rcsb.utils.seq.UniProtIdMappingProvider import UniProtIdMappingProvider
 from rcsb.utils.targets.CARDTargetProvider import CARDTargetProvider
 from rcsb.utils.targets.CARDTargetAnnotationProvider import CARDTargetAnnotationProvider
+from rcsb.utils.targets.CARDTargetOntologyProvider import CARDTargetOntologyProvider
 from rcsb.utils.targets.ChEMBLTargetProvider import ChEMBLTargetProvider
 from rcsb.utils.targets.ChEMBLTargetActivityProvider import ChEMBLTargetActivityProvider
 from rcsb.utils.targets.ChEMBLTargetCofactorProvider import ChEMBLTargetCofactorProvider
@@ -67,9 +69,12 @@ class ProteinTargetSequenceWorkflow(object):
         ok = False
         try:
             crmP = ChemRefMappingProvider(self.__cachePath, useCache=False)
-            okF = crmP.fetchChemRefMapping(self.__cfgOb)
-            okB = crmP.backup(self.__cfgOb, self.__configName, useStash=True, useGit=False)
-            ok = okF and okB
+            ok = crmP.fetchChemRefMapping(self.__cfgOb)
+            logger.info("Completed fetch ChemRefMappingProvider (%r)", ok)
+            crmP.reload()
+            if ok and crmP.testCache(minCount=1):
+                ok = crmP.backup(self.__cfgOb, self.__configName, useStash=True, useGit=False)
+                logger.info("Completed backup ChemRefMappingProvider (%r)", ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return ok
@@ -79,9 +84,12 @@ class ProteinTargetSequenceWorkflow(object):
         ok = False
         try:
             crmP = LigandNeighborMappingProvider(self.__cachePath, useCache=False)
-            okF = crmP.fetchLigandNeighborMapping(self.__cfgOb)
-            okB = crmP.backup(self.__cfgOb, self.__configName, useStash=True, useGit=False)
-            ok = okF and okB
+            ok = crmP.fetchLigandNeighborMapping(self.__cfgOb)
+            logger.info("Completed fetch LigandNeighborMappingProvider (%r)", ok)
+            crmP.reload()
+            if ok and crmP.testCache(minCount=1):
+                ok = crmP.backup(self.__cfgOb, self.__configName, useStash=True, useGit=False)
+                logger.info("Completed backup LigandNeighborMappingProvider (%r)", ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return ok
@@ -117,11 +125,12 @@ class ProteinTargetSequenceWorkflow(object):
         startTime = time.time()
         umP = UniProtIdMappingProvider(cachePath=self.__cachePath)
         umP.clearCache()
-        ok1 = umP.reload(useCache=True, useLegacy=False, fmt="tdd", mapNames=["NCBI-taxon"])
-        logger.info("Completed building UniProt Id mapping (%r) at %s (%.4f seconds)", ok1, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
-        if ok1:
-            ok2 = umP.backup(self.__cfgOb, self.__configName)
-        return ok1 & ok2
+        ok = umP.reload(useCache=True, useLegacy=False, fmt="tdd", mapNames=["NCBI-taxon"])
+        logger.info("Completed building UniProt Id mapping (%r) at %s (%.4f seconds)", ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
+        if ok and umP.testCache():
+            ok = umP.backup(self.__cfgOb, self.__configName)
+            logger.info("Completed backup UniProt Id mapping (%r)", ok)
+        return ok
 
     def exportTargetsFasta(self, resourceNameList=None, useCache=True, addTaxonomy=False, reloadPharos=False, fromDbPharos=False, backupPharos=False, remotePrefix=None):
         """Export the target FASTA files for the input data resources.
@@ -151,7 +160,7 @@ class ProteinTargetSequenceWorkflow(object):
                 backupPharos=backupPharos,
                 remotePrefix=remotePrefix
             )
-            logger.info("Completed loading %s targets (status %r)at %s (%.4f seconds)", resourceName, ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
+            logger.info("Completed loading %s targets (status %r) at %s (%.4f seconds)", resourceName, ok, time.strftime("%Y %m %d %H:%M:%S", time.localtime()), time.time() - startTime)
             retOk = retOk and ok
         return retOk
 
@@ -163,6 +172,7 @@ class ProteinTargetSequenceWorkflow(object):
             taxonPath = self.__getTaxonPath(resourceName)
             if resourceName == "card":
                 cardtP = CARDTargetProvider(cachePath=self.__cachePath, useCache=useCache)
+                cardtP.reload()
                 if cardtP.testCache():
                     ok = cardtP.exportCardFasta(fastaPath, taxonPath)
             elif resourceName == "drugbank":
@@ -173,6 +183,7 @@ class ProteinTargetSequenceWorkflow(object):
                     ok = dbtP.exportFasta(fastaPath, taxonPath, addTaxonomy=addTaxonomy)
             elif resourceName == "chembl":
                 chtP = ChEMBLTargetProvider(cachePath=self.__cachePath, useCache=useCache)
+                chtP.reload()
                 if chtP.testCache():
                     ok = chtP.exportFasta(fastaPath, taxonPath, addTaxonomy=addTaxonomy)
             elif resourceName == "pharos":
@@ -181,7 +192,7 @@ class ProteinTargetSequenceWorkflow(object):
                 ptP = PharosTargetProvider(cachePath=self.__cachePath, useCache=useCache, reloadDb=reloadPharos, fromDb=fromDbPharos, mysqlUser=user, mysqlPassword=pw)
                 if ptP.testCache():
                     ok = ptP.exportProteinFasta(fastaPath, taxonPath, addTaxonomy=addTaxonomy)
-                    if backupPharos:
+                    if ok and backupPharos and ptP.testCache():
                         okB = ptP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                         logger.info("%r targets backup status (%r)", resourceName, okB)
                 elif not (reloadPharos or fromDbPharos):
@@ -190,6 +201,7 @@ class ProteinTargetSequenceWorkflow(object):
                         ok = ptP.exportProteinFasta(fastaPath, taxonPath, addTaxonomy=addTaxonomy)
             elif resourceName == "sabdab":
                 stP = SAbDabTargetProvider(cachePath=self.__cachePath, useCache=False)
+                stP.reload()
                 if stP.testCache():
                     ok = stP.exportFasta(fastaPath)
             elif resourceName == "pdbprent":
@@ -341,22 +353,30 @@ class ProteinTargetSequenceWorkflow(object):
             if resourceName == "sabdab":
                 fP = SAbDabTargetFeatureProvider(cachePath=self.__cachePath, useCache=True)
                 ok = fP.buildFeatureList(resultPath)
-                if backup:
+                fP.reload()
+                if ok and backup and fP.testCache():
                     okB = fP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r features backup status (%r)", resourceName, okB)
             elif resourceName == "card":
                 fP = CARDTargetAnnotationProvider(cachePath=self.__cachePath, useCache=True)
                 ok = fP.buildAnnotationList(resultPath, useTaxonomy=useTaxonomy)
-                if backup:
+                fP.reload()
+                if ok and backup and fP.testCache():
                     okB = fP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r annotations backup status (%r)", resourceName, okB)
+                fP = CARDTargetOntologyProvider(cachePath=self.__cachePath, useCache=False)
+                ok = fP.reload()
+                if ok and backup and fP.testCache():
+                    okB = fP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
+                    logger.info("%r ontology backup status (%r)", resourceName, okB)
             elif resourceName == "imgt":
                 fP = IMGTTargetFeatureProvider(cachePath=self.__cachePath, useCache=True)
                 ok = fP.buildFeatureList(useCache=True)
-                if backup:
+                fP.reload()
+                if ok and backup and fP.testCache():
                     okB = fP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r features backup status (%r)", resourceName, okB)
-            return ok & okB
+            return ok and okB
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return False
@@ -394,7 +414,7 @@ class ProteinTargetSequenceWorkflow(object):
     def __buildActivityData(self, referenceResourceName, resourceName, backup=False, remotePrefix=None, maxTargets=None):
         """Build features inferred from sequence comparison results between the input resources."""
         try:
-            okB = okC = True
+            okB = okC = okD = True
             resultPath = self.__getFilteredSearchResultPath(resourceName, referenceResourceName)
             #
             if resourceName == "chembl":
@@ -406,26 +426,30 @@ class ProteinTargetSequenceWorkflow(object):
                     pass
                 targetIdList = aP.getTargetIdList(resultPath)
                 targetIdList = targetIdList[:maxTargets] if maxTargets else targetIdList
+                # To rebuild ChEMBL-target-activity data from scratch (non-incremental), change skip=None
                 ok = aP.fetchTargetActivityDataMulti(targetIdList, skip="tried", chunkSize=50, numProc=6)
                 #
-                if backup:
+                aP.reload()
+                if ok and backup and aP.testCache():
                     okB = aP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r activity backup status (%r)", resourceName, okB)
             elif resourceName == "pharos":
                 aP = PharosTargetActivityProvider(cachePath=self.__cachePath, useCache=True)
                 ok = aP.fetchTargetActivityData()
-                if backup:
+                aP.reload()
+                if ok and backup and aP.testCache(minCount=1):
                     okB = aP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r activity data backup status (%r)", resourceName, okB)
                 #
                 chemblIdList = aP.fetchCompoundIdentifiers()
                 phP = PharosProvider(cachePath=self.__cachePath, useCache=False)
-                phP.load(chemblIdList, "identifiers", fmt="json", indent=0)
-                if backup:
-                    okC = phP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
+                okC = phP.load(chemblIdList, "identifiers", fmt="json", indent=0)
+                phP.reload()
+                if okC and backup and phP.testCache(minCount=1):
+                    okD = phP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r identifier backup status (%r)", resourceName, okC)
 
-            return ok and okB and okC
+            return ok and okB and okC and okD
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return False
@@ -472,26 +496,34 @@ class ProteinTargetSequenceWorkflow(object):
             if resourceName == "chembl":
                 aP = ChEMBLTargetCofactorProvider(cachePath=self.__cachePath, useCache=True)
                 ok = aP.buildCofactorList(resultPath, crmpObj=crmpObj, lnmpObj=lnmpObj, maxActivity=maxActivity)
+                ok = aP.reload() and ok
+                logger.info("%r cofactor data build status (%r)", resourceName, ok)
                 #
-                if backup:
+                if ok and backup and aP.testCache():
                     okB = aP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
-                    logger.info("%r cofactor backup status (%r)", resourceName, okB)
+                    logger.info("%r cofactor data backup status (%r)", resourceName, okB)
 
             elif resourceName == "pharos":
                 aP = PharosTargetCofactorProvider(cachePath=self.__cachePath, useCache=True, useStash=True, useGit=True)
                 ok = aP.buildCofactorList(resultPath, crmpObj=crmpObj, lnmpObj=lnmpObj, maxActivity=maxActivity)
-                if backup:
+                ok = aP.reload() and ok
+                logger.info("%r cofactor data build status (%r)", resourceName, ok)
+                #
+                if ok and backup and aP.testCache():
                     okB = aP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r cofactor data backup status (%r)", resourceName, okB)
 
             elif resourceName == "drugbank":
                 aP = DrugBankTargetCofactorProvider(cachePath=self.__cachePath, useCache=True)
                 ok = aP.buildCofactorList(resultPath, crmpObj=crmpObj, lnmpObj=lnmpObj)
-                if backup:
+                ok = aP.reload() and ok
+                logger.info("%r cofactor data build status (%r)", resourceName, ok)
+                #
+                if ok and backup and aP.testCache():
                     okB = aP.backup(self.__cfgOb, self.__configName, remotePrefix=remotePrefix, useStash=True, useGit=False)
                     logger.info("%r cofactor data backup status (%r)", resourceName, okB)
-
-            return ok & okB
+            #
+            return ok and okB
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return False

{rcsb.workflow-0.41 → rcsb_workflow-0.43/rcsb.workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.workflow
-Version: 0.41
+Version: 0.43
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Home-page: https://github.com/rcsb/py-rcsb_workflow
 Author: John Westbrook
@@ -14,9 +14,18 @@ Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: OpenEye-toolkits>=2020.2.2
+Requires-Dist: rcsb.utils.chem>=0.75
+Requires-Dist: rcsb.utils.seq>=0.63
+Requires-Dist: rcsb.utils.targets>=0.57
+Requires-Dist: rcsb.utils.seqalign>=0.24
+Requires-Dist: rcsb.exdb>=0.87
+Requires-Dist: rcsb.utils.io>=1.20
 Provides-Extra: dev
+Requires-Dist: check-manifest; extra == "dev"
 Provides-Extra: test
-License-File: LICENSE
+Requires-Dist: coverage; extra == "test"
 
 # py-rcsb_workflow
 

{rcsb.workflow-0.41 → rcsb_workflow-0.43}/setup.cfg

@@ -2,7 +2,7 @@
 universal = 1
 
 [metadata]
-description-file = README.md
+description_file = README.md
 
 [egg_info]
 tag_build =