rcsb.workflow 0.71__tar.gz → 0.72__tar.gz

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/HISTORY.txt

@@ -66,3 +66,4 @@
 17-Jul-2026 - V0.69 Add shortlink support for bcif generation
 21-Jul-2026 - V0.70 Bcif hash support for extended ids
 19-Aug-2026 - V0.71 Support s3 stash and s3/minio for chem search
+22-Oct-2026 - V0.72 Remove unused reference sequence operations (upd_ref_seq, etl_uniprot_core) from ExDbWorkflow and exdb_wf_cli

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: rcsb.workflow
-Version: 0.71
+Version: 0.72
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Project-URL: Homepage, https://github.com/rcsb/py-rcsb_workflow
 Author-email: John Westbrook <john.westbrook@rcsb.org>

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "rcsb.workflow"
 description = "RCSB Python data processing and ETL/ELT workflow entry points"
-version = "0.71"
+version = "0.72"
 readme = "README.md"
 authors = [
     { name="John Westbrook", email="john.westbrook@rcsb.org" }

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/rcsb/workflow/cli/ExDbExec.py

@@ -15,6 +15,7 @@
 #                  (latter will only be used briefly, as will stop generating SDF and Mol2 files in Dec 2024)
 #  31-Jan-2025 mjt Moved this script from rcsb.exdb (to remove circluar dependencies)
 #  10-Jan-2025  cs Add ligand_quality_ref_gen and residue_rscc_ref_gen operations
+#  22-Oct-2026 dwp Remove unused reference sequence operations ('upd_ref_seq', 'etl_uniprot_core') and their arguments
 ##
 __docformat__ = "google en"
 __author__ = "John Westbrook"
@@ -46,9 +47,7 @@ def main():
         help="Loading operation to perform",
         choices=[
             "etl_chemref",  # ETL integrated chemical reference data
-            "etl_uniprot_core",  # ETL UniProt core reference data
             "etl_tree_node_lists",  # ETL tree node lists
-            "upd_ref_seq",  # Update reference sequence assignments
             "upd_neighbor_interactions",
             "upd_uniprot_taxonomy",
             "upd_targets",
@@ -80,7 +79,6 @@ def main():
     parser.add_argument("--document_limit", default=None, help="Load document limit for testing")
     #
     parser.add_argument("--rebuild_cache", default=False, action="store_true", help="Rebuild cached resource files")
-    parser.add_argument("--rebuild_sequence_cache", default=False, action="store_true", help="Rebuild cached resource files for reference sequence updates")
     parser.add_argument("--provider_types_exclude", default=None, help="Resource provider types to exclude")
     parser.add_argument("--use_filtered_tax_list", default=False, action="store_true", help="Use filtered list for taxonomy tree loading")
     parser.add_argument("--disable_read_back_check", default=False, action="store_true", help="Disable read back check on all documents")
@@ -88,12 +86,6 @@ def main():
     parser.add_argument("--mock", default=False, action="store_true", help="Use MOCK repository configuration for testing")
     parser.add_argument("--log_file_path", default=None, help="Path to runtime log file output.")
     #
-    # Arguments specific for op == 'upd_ref_seq'
-    parser.add_argument("--ref_chunk_size", default=10, help="Max chunk size for reference sequence updates (for op 'upd_ref_seq')")
-    parser.add_argument("--min_missing", default=0, help="Minimum number of allowed missing reference sequences (for op 'upd_ref_seq')")
-    parser.add_argument("--min_match_primary_percent", default=None, help="Minimum reference sequence match percentage (for op 'upd_ref_seq')")
-    parser.add_argument("--test_mode", default=False, action="store_true", help="Test mode for reference sequence updates (for op 'upd_ref_seq')")
-    #
     # Arguments specific for op == 'ccd_img_gen' or 'ccd_file_gen'
     parser.add_argument("--cc_output_path", default=None, help="The base local directory path where chemical component files (image, coordinates) are written (for op 'ccd_img_gen')")
     parser.add_argument("--cc_cache_path", default=None, help="The base local directory path where chemical component cache data are written (for op 'ccd_img_gen')")
@@ -121,7 +113,7 @@ def main():
     # Run the operation
     okR = False
     exWf = ExDbWorkflow(**commonD)
-    if op in ["etl_chemref", "etl_uniprot_core", "etl_tree_node_lists", "upd_ref_seq"]:
+    if op in ["etl_chemref", "etl_tree_node_lists"]:
         okR = exWf.load(op, **loadD)
     elif op in [
         "upd_neighbor_interactions",
@@ -219,12 +211,7 @@ def processArguments(args):
         "dbType": args.db_type,
         "documentLimit": int(args.document_limit) if args.document_limit else None,
         "readBackCheck": not args.disable_read_back_check,
-        "rebuildSequenceCache": args.rebuild_sequence_cache,
         "useFilteredLists": args.use_filtered_tax_list,
-        "refChunkSize": int(args.ref_chunk_size),
-        "minMissing": int(args.min_missing),
-        "minMatchPrimaryPercent": float(args.min_match_primary_percent) if args.min_match_primary_percent else None,
-        "testMode": args.test_mode,
         "rebuildAllNeighborInteractions": args.rebuild_all_neighbor_interactions,
         "ccFileNamePrefix": args.cc_file_prefix,
         "ccUrlTarget": args.cc_url_target,

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/rcsb/workflow/tests/testExDbWorkflow.py

@@ -6,6 +6,7 @@
 #
 # Updates:
 #  31-Jan-2025 mjt Moved this script from rcsb.exdb (to remove circluar dependencies)
+#  22-Oct-2026 dwp Remove skipped 'upd_ref_seq' test (operation removed)
 #
 ##
 """
@@ -62,7 +63,7 @@ class ExDbWorkflowTests(unittest.TestCase):
             "restoreUseGit": True,
             "restoreUseStash": False,
         }
-        self.__loadCommonD = {"readBackCheck": True, "numProc": 2, "chunkSize": 5, "refChunkSize": 5, "loadType": "full", "useFilteredLists": True}
+        self.__loadCommonD = {"readBackCheck": True, "numProc": 2, "chunkSize": 5, "loadType": "full", "useFilteredLists": True}
         self.__startTime = time.time()
         logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
 
@@ -93,33 +94,10 @@ class ExDbWorkflowTests(unittest.TestCase):
             logger.exception("Failing with %s", str(e))
             self.fail()
 
-    @unittest.skip("Troubleshooting test")
-    def testExDbLoaderWorkflowsWithCacheCheck(self):
-        """Test run sequence reference data update workflow step ..."""
-        #
-        try:
-            self.__commonD["rebuildCache"] = False
-            rlWf = ExDbWorkflow(**self.__commonD)
-            #
-            ok = rlWf.load("upd_ref_seq", minMatchPrimaryPercent=50.0, **self.__loadCommonD)
-            logger.info("Cache status is %r", ok)
-            self.assertTrue(ok)
-            if ok:
-                self.__commonD["rebuildCache"] = False
-                rlWf = ExDbWorkflow(**self.__commonD)
-                ok1 = rlWf.load("upd_ref_seq", **self.__loadCommonD)
-                self.assertTrue(ok1)
-                ok3 = rlWf.load("etl_tree_node_lists", **self.__loadCommonD)
-                self.assertTrue(ok3)
-        except Exception as e:
-            logger.exception("Failing with %s", str(e))
-            self.fail()
-
 
 def workflowLoadSuite():
     suiteSelect = unittest.TestSuite()
     suiteSelect.addTest(ExDbWorkflowTests("testExDbLoaderWorkflows"))
-    suiteSelect.addTest(ExDbWorkflowTests("testExDbLoaderWorkflowsWithCacheCheck"))
     return suiteSelect
 
 

{rcsb_workflow-0.71 → rcsb_workflow-0.72}/rcsb/workflow/wuw/ExDbWorkflow.py

@@ -11,6 +11,8 @@
 #  20-Aug-2024 dwp Add LoadTargetCofactors step; change name of UpdateTargetsCofactors step to UpdateTargetsData
 #  31-Jan-2025 mjt Moved this script from rcsb.exdb (to remove circluar dependencies)
 #  10-Jan-2026  cs Added ligand quality and residue RSCC reference data generation workflows
+#  22-Oct-2026 dwp Remove unused reference sequence operations ('upd_ref_seq', 'etl_uniprot_core') and
+#                  doReferenceSequenceUpdate(), whose rcsb.exdb modules are being removed
 ##
 __docformat__ = "google en"
 __author__ = "John Westbrook"
@@ -32,11 +34,7 @@ from rcsb.workflow.chem.ChemCompFileWorkflow import ChemCompFileWorkflow
 from rcsb.workflow.refstats.LigandQualityReferenceGenerator import LigandQualityReferenceGenerator
 from rcsb.workflow.refstats.ResidueRsccReferenceGenerator import ResidueRsccReferenceGenerator
 from rcsb.exdb.chemref.ChemRefEtlWorker import ChemRefEtlWorker
-from rcsb.exdb.seq.ReferenceSequenceAnnotationAdapter import ReferenceSequenceAnnotationAdapter
-from rcsb.exdb.seq.ReferenceSequenceAnnotationProvider import ReferenceSequenceAnnotationProvider
-from rcsb.exdb.seq.UniProtCoreEtlWorker import UniProtCoreEtlWorker
 from rcsb.exdb.tree.TreeNodeListWorker import TreeNodeListWorker
-from rcsb.exdb.utils.ObjectTransformer import ObjectTransformer
 from rcsb.exdb.wf.EntryInfoEtlWorkflow import EntryInfoEtlWorkflow
 from rcsb.exdb.wf.GlycanEtlWorkflow import GlycanEtlWorkflow
 from rcsb.exdb.wf.PubChemEtlWorkflow import PubChemEtlWorkflow
@@ -81,31 +79,22 @@ class ExDbWorkflow(object):
         logger.info("Starting operation %r\n", op)
         #
         # argument processing
-        if op not in ["etl_tree_node_lists", "etl_chemref", "etl_uniprot_core", "upd_ref_seq", "upd_ref_seq_comp_models", "refresh_pubchem"]:
+        if op not in ["etl_tree_node_lists", "etl_chemref", "refresh_pubchem"]:
             logger.error("Unsupported operation %r - exiting", op)
             return False
         if not self.__cacheStatus:
             raise ValueError("Failed to rebuild CACHE in ExDBWorkflow - exiting")
         try:
-            # test mode and UniProt accession primary match minimum count for doReferenceSequenceUpdate()
-            testMode = kwargs.get("testMode", False)
-            minMatchPrimaryPercent = kwargs.get("minMatchPrimaryPercent", None)
-            minMissing = kwargs.get("minMissing", 0)
-            #
             readBackCheck = kwargs.get("readBackCheck", True)
             numProc = int(kwargs.get("numProc", 1))
             chunkSize = int(kwargs.get("chunkSize", 10))
             maxStepLength = int(kwargs.get("maxStepLength", 500))
-            refChunkSize = int(kwargs.get("refChunkSize", 10))
             documentLimit = kwargs.get("documentLimit", None)
             documentLimit = int(documentLimit) if documentLimit else None
             loadType = kwargs.get("loadType", "full")  # or replace
             dbType = kwargs.get("dbType", "mongo")
             tU = TimeUtil()
             dataSetId = kwargs.get("dataSetId") if "dataSetId" in kwargs else tU.getCurrentWeekSignature()
-            #  Rebuild or reuse reference sequence cache
-            rebuildSequenceCache = kwargs.get("rebuildSequenceCache", False)
-            useSequenceCache = not rebuildSequenceCache
             #
             useFilteredLists = kwargs.get("useFilteredLists", False)
             treeCollectionList = kwargs.get("treeCollectionList", None)
@@ -147,39 +136,6 @@ class ExDbWorkflow(object):
                 )
                 ok = crw.load(dataSetId, extResource="DrugBank", loadType=loadType)
                 okS = self.loadStatus(crw.getLoadStatus(), readBackCheck=readBackCheck)
-
-            elif op == "etl_uniprot_core":
-                crw = UniProtCoreEtlWorker(
-                    self.__cfgOb,
-                    self.__cachePath,
-                    numProc=numProc,
-                    chunkSize=chunkSize,
-                    maxStepLength=maxStepLength,
-                    documentLimit=documentLimit,
-                    verbose=self.__debugFlag,
-                    readBackCheck=readBackCheck,
-                    useCache=self.__useCache,
-                )
-                ok = crw.load(dataSetId, extResource="UniProt", loadType=loadType)
-                okS = self.loadStatus(crw.getLoadStatus(), readBackCheck=readBackCheck)
-
-            elif op == "upd_ref_seq":
-                databaseName = "pdbx_core"
-                collectionName = "pdbx_core_polymer_entity"
-                polymerType = "Protein"
-                ok = self.doReferenceSequenceUpdate(
-                    databaseName,
-                    collectionName,
-                    polymerType,
-                    fetchLimit=documentLimit,
-                    useSequenceCache=useSequenceCache,
-                    testMode=testMode,
-                    minMatchPrimaryPercent=minMatchPrimaryPercent,
-                    minMissing=minMissing,
-                    refChunkSize=refChunkSize,
-                    numProc=numProc
-                )
-                okS = ok
         logger.info("Completed operation %r with status %r\n", op, ok and okS)
         return ok and okS
 
@@ -224,44 +180,6 @@ class ExDbWorkflow(object):
             logger.exception("Failing with %s", str(e))
         return ret
 
-    def doReferenceSequenceUpdate(
-        self,
-        databaseName,
-        collectionName,
-        polymerType,
-        fetchLimit=None,
-        useSequenceCache=False,
-        testMode=False,
-        minMatchPrimaryPercent=None,
-        minMissing=0,
-        refChunkSize=10,
-        numProc=2,
-        **kwargs
-    ):
-        try:
-            _ = kwargs
-            _ = testMode
-            ok = False
-            # -------
-            rsaP = ReferenceSequenceAnnotationProvider(
-                self.__cfgOb, databaseName, collectionName, polymerType, useCache=useSequenceCache, cachePath=self.__cachePath, maxChunkSize=refChunkSize, numProc=numProc
-            )
-            ok = rsaP.testCache(minMatchPrimaryPercent=minMatchPrimaryPercent, minMissing=minMissing)
-            if ok:
-                logger.info("Cached reference data count is %d", rsaP.getRefDataCount())
-                rsa = ReferenceSequenceAnnotationAdapter(rsaP)
-                obTr = ObjectTransformer(self.__cfgOb, objectAdapter=rsa)
-                ok = obTr.doTransform(
-                    databaseName=databaseName, collectionName=collectionName, fetchLimit=fetchLimit, selectionQuery={"entity_poly.rcsb_entity_polymer_type": polymerType}
-                )
-            else:
-                logger.error("Reference sequence data cache build failing")
-                return False
-            return ok
-        except Exception as e:
-            logger.exception("Failing with %s", str(e))
-        return False
-
     def generateCcdFiles(self, op, **kwargs):
         logger.info("Starting operation %r\n", op)
         #