rcsb.workflow 0.64__tar.gz → 0.65__tar.gz

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/HISTORY.txt

@@ -58,3 +58,4 @@
 30-Sep-2025 - V0.62 Let jpg gen use model file over URL
 15-Oct-2025 - V0.63 Add ChemSearchIndex from weekly-update-workflow so it can be a cli call instead
  8-Dec-2025 - V0.64 Switch to 'pyproject.toml' file and update test pipelines to python 3.12
+ 6-Jan-2026 - V0.65 Reduce resource utilization for test suites

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rcsb.workflow
-Version: 0.64
+Version: 0.65
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Project-URL: Homepage, https://github.com/rcsb/py-rcsb_workflow
 Author-email: John Westbrook <john.westbrook@rcsb.org>
@@ -17,7 +17,7 @@ Requires-Python: >=3.9
 Requires-Dist: mmcif>=1.0.0
 Requires-Dist: openeye-toolkits>=2024.1.1
 Requires-Dist: rcsb-db>=1.812
-Requires-Dist: rcsb-exdb>=1.31
+Requires-Dist: rcsb-exdb>=1.33
 Requires-Dist: rcsb-utils-chem>=0.84
 Requires-Dist: rcsb-utils-io>=1.49
 Requires-Dist: rcsb-utils-seq>=0.84

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "rcsb.workflow"
 description = "RCSB Python data processing and ETL/ELT workflow entry points"
-version = "0.64"
+version = "0.65"
 readme = "README.md"
 authors = [
     { name="John Westbrook", email="john.westbrook@rcsb.org" }
@@ -38,7 +38,7 @@ dependencies = [
     "rcsb.utils.seq >= 0.84",
     "rcsb.utils.targets >= 0.83",
     "rcsb.utils.seqalign >= 0.32",
-    "rcsb.exdb >= 1.31",
+    "rcsb.exdb >= 1.33",
     "requests >= 2.32.3",
 ]
 

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/rcsb/workflow/tests/fixturePdbxLoader.py

@@ -24,6 +24,7 @@ import platform
 import resource
 import time
 import unittest
+import shutil
 
 from rcsb.db.mongo.DocumentLoader import DocumentLoader
 from rcsb.db.mongo.PdbxLoader import PdbxLoader
@@ -60,6 +61,9 @@ class PdbxLoaderFixture(unittest.TestCase):
         configName = "site_info_configuration"
         self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
         #
+        self._disk_before = shutil.disk_usage(HERE).used
+        logger.info("Filesystem disk usage start: %.2f MB", self._disk_before / (1024 ** 2))
+        #
         self.__resourceName = "MONGO_DB"
         self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt")
         self.__cachePath = os.path.join(TOPDIR, "CACHE")
@@ -200,6 +204,9 @@ class PdbxLoaderFixture(unittest.TestCase):
         rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
         logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
         endTime = time.time()
+        disk_after = shutil.disk_usage(HERE).used
+        disk_delta = disk_after - self._disk_before
+        logger.info("Filesystem disk usage delta: %.2f MB", disk_delta / (1024 ** 2))
         logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
 
     def testPdbxLoader(self):

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/rcsb/workflow/tests/testExDbWorkflow.py

@@ -24,6 +24,7 @@ import platform
 import resource
 import time
 import unittest
+import shutil
 
 from rcsb.workflow.wuw.ExDbWorkflow import ExDbWorkflow
 
@@ -48,6 +49,9 @@ class ExDbWorkflowTests(unittest.TestCase):
         cachePath = os.path.join(TOPDIR, "CACHE")
         # self.__dataPath = os.path.join(HERE, "test-data")
         #
+        self._disk_before = shutil.disk_usage(HERE).used
+        logger.info("Filesystem disk usage start: %.2f MB", self._disk_before / (1024 ** 2))
+        #
         self.__commonD = {
             "configPath": configPath,
             "mockTopPath": mockTopPath,
@@ -67,17 +71,24 @@ class ExDbWorkflowTests(unittest.TestCase):
         rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
         logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
         endTime = time.time()
+        disk_after = shutil.disk_usage(HERE).used
+        disk_delta = disk_after - self._disk_before
+        logger.info("Filesystem disk usage delta: %.2f MB", disk_delta / (1024 ** 2))
         logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
 
     def testExDbLoaderWorkflows(self):
         """Test run workflow steps ..."""
         try:
-            # opL = ["etl_chemref", "upd_ref_seq", "etl_tree_node_lists"]
-            opL = ["etl_chemref", "etl_tree_node_lists"]
             rlWf = ExDbWorkflow(**self.__commonD)
-            for op in opL:
-                ok = rlWf.load(op, **self.__loadCommonD)
-                self.assertTrue(ok)
+            ok = rlWf.load("etl_chemref", **self.__loadCommonD)
+            self.assertTrue(ok)
+            ok = rlWf.load("etl_tree_node_lists", treeCollectionList=["tree_ec"], **self.__loadCommonD)
+            self.assertTrue(ok)
+            try:
+                ok = rlWf.load("etl_tree_node_lists", treeCollectionList=["tree_scop"], **self.__loadCommonD)
+                self.assertFalse(ok)
+            except Exception:
+                logger.info("Expected failure if tree_scop node list is empty")
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/rcsb/workflow/tests/testProteinTargetSequenceWorkflow.py

@@ -6,6 +6,9 @@
 # Updates:
 #  13-Feb-2025 dwp Remove IMGT from feature building after service became unavailable February 2025
 #  19-Feb-2025 dwp Bring back IMGT
+#   6-Jan-2026 dwp Re-exclude IMGT from testing (consumes a lot of disk space by fetching 1.6 GB file,
+#                  and this is already tested by rcsb.utils.targets);
+#                  Exclude chembl and drugbank from testing to save on disk space
 #
 ##
 """
@@ -22,6 +25,7 @@ import platform
 import resource
 import time
 import unittest
+import shutil
 
 from rcsb.workflow.targets.ProteinTargetSequenceWorkflow import ProteinTargetSequenceWorkflow
 from rcsb.utils.config.ConfigUtil import ConfigUtil
@@ -46,6 +50,9 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
         self.__cachePath = os.path.join(HERE, "test-output", "CACHE")
         #
+        self._disk_before = shutil.disk_usage(HERE).used
+        logger.info("Filesystem disk usage start: %.2f MB", self._disk_before / (1024 ** 2))
+        #
         self.__workflowFixture()
         self.__startTime = time.time()
         logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
@@ -55,6 +62,9 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
         logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
         endTime = time.time()
+        disk_after = shutil.disk_usage(HERE).used
+        disk_delta = disk_after - self._disk_before
+        logger.info("Filesystem disk usage delta: %.2f MB", disk_delta / (1024 ** 2))
         logger.info("Completed %s at %s (%.4f seconds)\n", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
 
     def __workflowFixture(self):
@@ -137,7 +147,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         """Test case - export FASTA target files"""
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
-            ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=False, reloadPharos=False, fromDbPharos=False, resourceNameList=["sabdab", "card", "chembl", "pharos"])
+            # ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=False, reloadPharos=False, fromDbPharos=False, resourceNameList=["sabdab", "card", "chembl", "pharos"])
+            ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=False, reloadPharos=False, fromDbPharos=False, resourceNameList=["sabdab", "card", "pharos"])
             self.assertTrue(ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
@@ -148,7 +159,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         """Test case - export FASTA target files (and load Pharos from source)"""
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
-            ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=True, reloadPharos=True, fromDbPharos=True, resourceNameList=["sabdab", "card", "drugbank", "chembl", "pharos"])
+            # ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=True, reloadPharos=True, fromDbPharos=True, resourceNameList=["sabdab", "card", "drugbank", "chembl", "pharos"])
+            ok = ptsW.exportTargetsFasta(useCache=True, addTaxonomy=True, reloadPharos=True, fromDbPharos=True, resourceNameList=["sabdab", "card", "pharos"])
             self.assertTrue(ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
@@ -158,7 +170,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         """Test case - create search databases"""
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
-            ok = ptsW.createSearchDatabases(resourceNameList=["sabdab", "card", "chembl", "pharos", "pdbprent"], addTaxonomy=False, timeOutSeconds=3600, verbose=False)
+            # ok = ptsW.createSearchDatabases(resourceNameList=["sabdab", "card", "chembl", "pharos", "pdbprent"], addTaxonomy=False, timeOutSeconds=3600, verbose=False)
+            ok = ptsW.createSearchDatabases(resourceNameList=["sabdab", "pharos", "pdbprent"], addTaxonomy=False, timeOutSeconds=3600, verbose=False)
             self.assertTrue(ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
@@ -171,7 +184,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
             formatOutput = "query,target,pident,alnlen,mismatch,gapopen,qstart,qend,tstart,tend,evalue,raw,bits,qlen,tlen,qaln,taln,cigar"
             ok = ptsW.search(
                 referenceResourceName="pdbprent",
-                resourceNameList=["sabdab", "chembl", "pharos"],
+                # resourceNameList=["sabdab", "chembl", "pharos"],
+                resourceNameList=["sabdab", "pharos"],
                 identityCutoff=0.95,
                 sensitivity=4.5,
                 timeOutSeconds=1000,
@@ -179,7 +193,7 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
             )
             self.assertTrue(ok)
             ok = ptsW.search(
-                referenceResourceName="pdbprent", resourceNameList=["card"], identityCutoff=0.95, sensitivity=4.5, timeOutSeconds=1000, useBitScore=True, formatOutput=formatOutput
+                referenceResourceName="pdbprent", resourceNameList=["sabdab"], identityCutoff=0.95, sensitivity=4.5, timeOutSeconds=1000, useBitScore=True, formatOutput=formatOutput
             )
             self.assertTrue(ok)
         except Exception as e:
@@ -190,7 +204,7 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         """Test case - build features from search results"""
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
-            ok = ptsW.buildFeatureData(referenceResourceName="pdbprent", resourceNameList=["sabdab", "card", "imgt"], useTaxonomy=False, backup=False)
+            ok = ptsW.buildFeatureData(referenceResourceName="pdbprent", resourceNameList=["sabdab"], useTaxonomy=False, backup=False)
             self.assertTrue(ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
@@ -201,7 +215,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
             if self.__isMac:
-                ok = ptsW.buildActivityData(referenceResourceName="pdbprent", resourceNameList=["chembl", "pharos"], backup=False, maxTargets=50)
+                # ok = ptsW.buildActivityData(referenceResourceName="pdbprent", resourceNameList=["chembl", "pharos"], backup=False, maxTargets=50)
+                ok = ptsW.buildActivityData(referenceResourceName="pdbprent", resourceNameList=["pharos"], backup=False, maxTargets=50)
             else:
                 ok = ptsW.buildActivityData(referenceResourceName="pdbprent", resourceNameList=["pharos"], backup=False, maxTargets=50)
             self.assertTrue(ok)
@@ -214,7 +229,8 @@ class ProteinTargetSequenceWorkflowTests(unittest.TestCase):
         """Test case - build features from search results"""
         try:
             ptsW = ProteinTargetSequenceWorkflow(self.__cfgOb, self.__cachePath)
-            ok = ptsW.buildCofactorData(referenceResourceName="pdbprent", resourceNameList=["chembl", "pharos"], backup=False)
+            # ok = ptsW.buildCofactorData(referenceResourceName="pdbprent", resourceNameList=["chembl", "pharos"], backup=False)
+            ok = ptsW.buildCofactorData(referenceResourceName="pdbprent", resourceNameList=["pharos"], backup=False)
             self.assertTrue(ok)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
@@ -242,7 +258,7 @@ def abbrevSuite():
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testBBCreateSearchDatabases"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testCCSearchDatabases"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testDDBuildFeatures"))
-    suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testDDBuildActivityData"))
+    suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testEEBuildActivityData"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testFFBuildCofactorData"))
     return suiteSelect
 
@@ -257,7 +273,7 @@ def fullSuite():
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testBBCreateSearchDatabases"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testCCSearchDatabases"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testDDBuildFeatures"))
-    suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testDDBuildActivityData"))
+    suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testEEBuildActivityData"))
     suiteSelect.addTest(ProteinTargetSequenceWorkflowTests("testFFBuildCofactorData"))
     return suiteSelect
 

{rcsb_workflow-0.64 → rcsb_workflow-0.65}/rcsb/workflow/wuw/ExDbWorkflow.py

@@ -105,6 +105,7 @@ class ExDbWorkflow(object):
             useSequenceCache = not rebuildSequenceCache
             #
             useFilteredLists = kwargs.get("useFilteredLists", False)
+            treeCollectionList = kwargs.get("treeCollectionList", None)
 
         except Exception as e:
             logger.exception("Argument or configuration processing failing with %s", str(e))
@@ -124,6 +125,7 @@ class ExDbWorkflow(object):
                     readBackCheck=readBackCheck,
                     useCache=self.__useCache,
                     useFilteredLists=useFilteredLists,
+                    treeCollectionList=treeCollectionList,
                 )
                 ok = rhw.load(dataSetId, loadType=loadType)
                 okS = self.loadStatus(rhw.getLoadStatus(), readBackCheck=readBackCheck)