rcsb.exdb 0.99__tar.gz → 1.1__tar.gz

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/HISTORY.txt

@@ -95,3 +95,7 @@
                   Load full (unfiltered) taxonomy tree node list, and stop loading GO tree
 19-Sep-2023 V0.99 Add reload method to ChemRefMappingProvider and LigandNeighborMappingProvider;
                   Add documentation to reference sequence providers
+ 9-Jan-2024 V1.00 Update PolymerEntityExtractor to turn off usage of uniprot_exdb as source data;
+                  This package update also coincides with the turning off of uniprot_exdb data loading during the weekly workflow
+ 6-May-2024 V1.1  Update ExDbExec CLI and ExDbWorkflow to support CLI usage from weekly-update workflow;
+                  Update unit tests and setuptools config

{rcsb.exdb-0.99/rcsb.exdb.egg-info → rcsb_exdb-1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.exdb
-Version: 0.99
+Version: 1.1
 Summary: RCSB Python ExDB data extraction and loading workflows
 Home-page: https://github.com/rcsb/py-rcsb_exdb
 Author: John Westbrook
@@ -30,6 +30,7 @@ Requires-Dist: rcsb.utils.seq>=0.63
 Requires-Dist: rcsb.utils.struct>=0.37
 Requires-Dist: rcsb.utils.taxonomy>=0.39
 Requires-Dist: rcsb.utils.dictionary>=0.71
+Requires-Dist: rcsb.workflow>=0.42
 Requires-Dist: statistics; python_version < "3.0"
 Provides-Extra: dev
 Requires-Dist: check-manifest; extra == "dev"

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/chemref/ChemRefEtlWorker.py

@@ -29,13 +29,14 @@ logger = logging.getLogger(__name__)
 class ChemRefEtlWorker(object):
     """Prepare and load chemical reference data collections."""
 
-    def __init__(self, cfgOb, cachePath, useCache=True, numProc=2, chunkSize=10, readBackCheck=False, documentLimit=None, verbose=False):
+    def __init__(self, cfgOb, cachePath, useCache=True, numProc=2, chunkSize=10, maxStepLength=2000, readBackCheck=False, documentLimit=None, verbose=False):
         self.__cfgOb = cfgOb
         self.__cachePath = cachePath
         self.__useCache = useCache
         self.__readBackCheck = readBackCheck
         self.__numProc = numProc
         self.__chunkSize = chunkSize
+        self.__maxStepLength = maxStepLength
         self.__documentLimit = documentLimit
         #
         self.__resourceName = "MONGO_DB"
@@ -95,6 +96,7 @@ class ChemRefEtlWorker(object):
                 self.__resourceName,
                 numProc=self.__numProc,
                 chunkSize=self.__chunkSize,
+                maxStepLength=self.__maxStepLength,
                 documentLimit=self.__documentLimit,
                 verbose=self.__verbose,
                 readBackCheck=self.__readBackCheck,

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/chemref/PubChemIndexCacheProvider.py

@@ -196,7 +196,7 @@ class PubChemUpdateWorker(object):
 
     def __chunker(self, iList, chunkSize):
         chunkSize = max(1, chunkSize)
-        return (iList[i : i + chunkSize] for i in range(0, len(iList), chunkSize))
+        return (iList[i: i + chunkSize] for i in range(0, len(iList), chunkSize))
 
 
 class PubChemIndexCacheProvider(StashableBase):
@@ -319,6 +319,7 @@ class PubChemIndexCacheProvider(StashableBase):
         #
         matchD = {}
         matchedIdList = []
+        ok = False
         try:
             # ---
             # Get current the indices of source chemical reference data -
@@ -346,7 +347,10 @@ class PubChemIndexCacheProvider(StashableBase):
             else:
                 logger.info("No reference data updates required")
             # --
-            return ok
+            if not ok:
+                logger.warning("updateMissing completed with status %r failures %r", ok, len(failList))
+            #
+            return True
         except Exception as e:
             logger.exception("Failing with %s", str(e))
         return ok
@@ -569,8 +573,10 @@ class PubChemIndexCacheProvider(StashableBase):
         """Rebuild source indices of chemical component definitions."""
         logger.info("Rebuilding chemical definition index.")
         ok1, ccidxP = self.__buildChemCompIndex(**kwargs)
+        logger.info("__buildChemCompIndex completed with status %r", ok1)
         logger.info("Rebuilding chemical search indices.")
         ok2, ccsidxP = self.__buildChemCompSearchIndex(numProc, **kwargs)
+        logger.info("__buildChemCompSearchIndex completed with status %r", ok2)
         return ok1 & ok2, ccidxP, ccsidxP
 
     def __buildChemCompIndex(self, **kwargs):

rcsb_exdb-1.1/rcsb/exdb/cli/ExDbExec.py

@@ -0,0 +1,211 @@
+##
+# File: ExDbExec.py
+# Date: 22-Apr-2019  jdw
+#
+#  Execution wrapper  --  for extract and load operations -
+#
+#  Updates:
+#   4-Sep-2019 jdw add Tree and Drugbank loaders
+#  14-Feb-2020 jdw change over to ReferenceSequenceAnnotationProvider/Adapter
+#   9-Mar-2023 dwp Lower refChunkSize to 10 (UniProt API having trouble streaming XML responses)
+#  25-Apr-2024 dwp Add arguments and logic to support CLI usage from weekly-update workflow;
+#                  Add support for logging output to a specific file
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import os
+import sys
+import argparse
+import logging
+
+from rcsb.utils.config.ConfigUtil import ConfigUtil
+from rcsb.exdb.wf.ExDbWorkflow import ExDbWorkflow
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+
+# logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s", stream=sys.stdout)
+logger = logging.getLogger()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    #
+    parser.add_argument(
+        "--op",
+        default=None,
+        required=True,
+        help="Loading operation to perform",
+        choices=[
+            "etl_chemref",  # ETL integrated chemical reference data
+            "etl_uniprot_core",  # ETL UniProt core reference data
+            "etl_tree_node_lists",  # ETL tree node lists
+            "upd_ref_seq",  # Update reference sequence assignments
+            "upd_neighbor_interactions",
+            "upd_uniprot_taxonomy",
+            "upd_targets_cofactors",
+            "upd_pubchem",
+            "upd_entry_info",
+            "upd_glycan_idx",
+            "upd_resource_stash",
+        ]
+    )
+    parser.add_argument(
+        "--load_type",
+        default="full",
+        help="Type of load ('full' for complete and fresh single-worker load, 'replace' for incremental and multi-worker load)",
+        choices=["full", "replace"],
+    )
+    #
+    parser.add_argument("--config_path", default=None, help="Path to configuration options file")
+    parser.add_argument("--config_name", default="site_info_remote_configuration", help="Configuration section name")
+    parser.add_argument("--cache_path", default=None, help="Cache path for resource files")
+    parser.add_argument("--num_proc", default=2, help="Number of processes to execute (default=2)")
+    parser.add_argument("--chunk_size", default=10, help="Number of files loaded per process")
+    parser.add_argument("--max_step_length", default=500, help="Maximum subList size (default=500)")
+    parser.add_argument("--db_type", default="mongo", help="Database server type (default=mongo)")
+    parser.add_argument("--document_limit", default=None, help="Load document limit for testing")
+    #
+    parser.add_argument("--rebuild_cache", default=False, action="store_true", help="Rebuild cached resource files")
+    parser.add_argument("--rebuild_sequence_cache", default=False, action="store_true", help="Rebuild cached resource files for reference sequence updates")
+    parser.add_argument("--provider_type_exclude", default=None, help="Resource provider types to exclude")
+    parser.add_argument("--use_filtered_tax_list", default=False, action="store_true", help="Use filtered list for taxonomy tree loading")
+    parser.add_argument("--disable_read_back_check", default=False, action="store_true", help="Disable read back check on all documents")
+    parser.add_argument("--debug", default=False, action="store_true", help="Turn on verbose logging")
+    parser.add_argument("--mock", default=False, action="store_true", help="Use MOCK repository configuration for testing")
+    parser.add_argument("--log_file_path", default=None, help="Path to runtime log file output.")
+    #
+    # Arguments specific for op == 'upd_ref_seq'
+    parser.add_argument("--ref_chunk_size", default=10, help="Max chunk size for reference sequence updates (for op 'upd_ref_seq')")
+    parser.add_argument("--min_missing", default=0, help="Minimum number of allowed missing reference sequences (for op 'upd_ref_seq')")
+    parser.add_argument("--min_match_primary_percent", default=None, help="Minimum reference sequence match percentage (for op 'upd_ref_seq')")
+    parser.add_argument("--test_mode", default=False, action="store_true", help="Test mode for reference sequence updates (for op 'upd_ref_seq')")
+    #
+    # Arguments buildExdbResources
+    parser.add_argument("--rebuild_all_neighbor_interactions", default=False, action="store_true", help="Rebuild all neighbor interactions from scratch (default is incrementally)")
+    parser.add_argument("--cc_file_prefix", default="cc-full", help="File name discriminator for index sets")
+    parser.add_argument("--cc_url_target", default=None, help="target url for chemical component dictionary resource file (default: None=all public)")
+    parser.add_argument("--bird_url_target", default=None, help="target url for bird dictionary resource file (cc format) (default: None=all public)")
+    #
+    args = parser.parse_args()
+    #
+    try:
+        op, commonD, loadD = processArguments(args)
+    except Exception as err:
+        logger.exception("Argument processing problem %s", str(err))
+        raise ValueError("Argument processing problem") from err
+    #
+    #
+    # Log input arguments
+    loadLogD = {k: v for d in [commonD, loadD] for k, v in d.items() if k != "inputIdCodeList"}
+    logger.info("running load op %r on loadLogD %r:", op, loadLogD)
+    #
+    # Run the operation
+    okR = False
+    exWf = ExDbWorkflow(**commonD)
+    if op in ["etl_chemref", "etl_uniprot_core", "etl_tree_node_lists", "upd_ref_seq"]:
+        okR = exWf.load(op, **loadD)
+    elif op in ["upd_neighbor_interactions", "upd_uniprot_taxonomy", "upd_targets_cofactors", "upd_pubchem", "upd_entry_info", "upd_glycan_idx", "upd_resource_stash"]:
+        okR = exWf.buildExdbResource(op, **loadD)
+    else:
+        logger.error("Unsupported op %r", op)
+    #
+    logger.info("Operation %r completed with status %r", op, okR)
+    #
+    if not okR:
+        logger.error("Operation %r failed with status %r", op, okR)
+        raise ValueError("Operation %r failed" % op)
+
+
+def processArguments(args):
+    # Logging details
+    logFilePath = args.log_file_path
+    debugFlag = args.debug
+    if debugFlag:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    if logFilePath:
+        logDir = os.path.dirname(logFilePath)
+        if not os.path.isdir(logDir):
+            os.makedirs(logDir)
+        handler = logging.FileHandler(logFilePath, mode="a")
+        if debugFlag:
+            handler.setLevel(logging.DEBUG)
+        else:
+            handler.setLevel(logging.INFO)
+        formatter = logging.Formatter("%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    #
+    # Configuration details
+    configPath = args.config_path
+    configName = args.config_name
+    if not (configPath and configName):
+        logger.error("Config path and/or name not provided: %r, %r", configPath, configName)
+        raise ValueError("Config path and/or name not provided: %r, %r" % (configPath, configName))
+    mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") if args.mock else None
+    logger.info("Using configuration file %r (section %r)", configPath, configName)
+    cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath)
+    cfgObTmp = cfgOb.exportConfig()
+    logger.info("Length of config object (%r)", len(cfgObTmp))
+    if len(cfgObTmp) == 0:
+        logger.error("Missing or access issue for config file %r", configPath)
+        raise ValueError("Missing or access issue for config file %r" % configPath)
+    else:
+        del cfgObTmp
+    #
+    # Do any additional argument checking
+    op = args.op
+    if not op:
+        raise ValueError("Must supply a value to '--op' argument")
+    #
+    cachePath = args.cache_path if args.cache_path else "."
+    cachePath = os.path.abspath(cachePath)
+
+    if args.db_type != "mongo":
+        logger.error("Unsupported database type %r (must be 'mongo')", args.db_type)
+        raise ValueError("Unsupported database type %r (must be 'mongo')" % args.db_type)
+
+    # Now collect arguments into dictionaries
+    commonD = {
+        "configPath": configPath,
+        "configName": configName,
+        "cachePath": cachePath,
+        "mockTopPath": mockTopPath,
+        "debugFlag": debugFlag,
+        "rebuildCache": args.rebuild_cache,
+        "providerTypeExclude": args.provider_type_exclude,
+    }
+    loadD = {
+        "loadType": args.load_type,
+        "numProc": int(args.num_proc),
+        "chunkSize": int(args.chunk_size),
+        "maxStepLength": int(args.max_step_length),
+        "dbType": args.db_type,
+        "documentLimit": int(args.document_limit) if args.document_limit else None,
+        "readBackCheck": not args.disable_read_back_check,
+        "rebuildSequenceCache": args.rebuild_sequence_cache,
+        "useFilteredLists": args.use_filtered_tax_list,
+        "refChunkSize": int(args.ref_chunk_size),
+        "minMissing": int(args.min_missing),
+        "minMatchPrimaryPercent": float(args.min_match_primary_percent) if args.min_match_primary_percent else None,
+        "testMode": args.test_mode,
+        "rebuildAllNeighborInteractions": args.rebuild_all_neighbor_interactions,
+        "ccFileNamePrefix": args.cc_file_prefix,
+        "ccUrlTarget": args.cc_url_target,
+        "birdUrlTarget": args.bird_url_target,
+    }
+
+    return op, commonD, loadD
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        logger.exception("Run failed %s", str(e))
+        sys.exit(1)

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/cli/__init__.py

@@ -2,4 +2,4 @@ __docformat__ = "google en"
 __author__ = "John Westbrook"
 __email__ = "john.westbrook@rcsb.org"
 __license__ = "Apache 2.0"
-__version__ = "0.99"
+__version__ = "1.1"

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/seq/PolymerEntityExtractor.py

@@ -6,6 +6,7 @@
 #
 #
 # Updates:
+#  9-Jan-2024 dwp Turn off use of uniprot_exdb DB for enriching protein entity details file (data not used)
 #
 ##
 __docformat__ = "google en"
@@ -16,7 +17,6 @@ __license__ = "Apache 2.0"
 import logging
 import os
 
-from rcsb.exdb.seq.UniProtExtractor import UniProtExtractor
 from rcsb.exdb.utils.ObjectExtractor import ObjectExtractor
 from rcsb.utils.io.MarshalUtil import MarshalUtil
 
@@ -56,9 +56,6 @@ class PolymerEntityExtractor(object):
         missingSrcD = {}
         rD = {}
         try:
-            unpEx = UniProtExtractor(self.__cfgOb)
-            unpD = unpEx.getReferenceSequenceDetails()
-            #
             obEx = ObjectExtractor(
                 self.__cfgOb,
                 databaseName="pdbx_core",
@@ -155,9 +152,13 @@ class PolymerEntityExtractor(object):
                             uD["refDbId"] = tD["reference_database_accession"]
                             uD["refDbName"] = tD["reference_database_name"]
                             uD["provSource"] = tD["provenance_source"]
-                            if tD["reference_database_accession"] in unpD:
-                                # This adds {"accession": rId, "taxId": taxId, "scientific_name": sn, "gene": gn, "name": pn, "sequence": sequence}
-                                uD.update(unpD[tD["reference_database_accession"]])
+                            #
+                            # Skip the below step now that uniprot_exdb DB is no longer being updated in weekly workflow.
+                            # The data added here isn't used by subsequent tasks. It simply provides
+                            # additional information in the pdbprent-details.json file (under "alignmentL")
+                            # if tD["reference_database_accession"] in unpD:
+                            #     # This adds {"accession": rId, "taxId": taxId, "scientific_name": sn, "gene": gn, "name": pn, "sequence": sequence}
+                            #     uD.update(unpD[tD["reference_database_accession"]])
                             aL = []
                             for qD in tD["aligned_regions"]:
                                 if qD["entity_beg_seq_id"] + qD["length"] - 1 > seqLen:
@@ -223,44 +224,44 @@ class PolymerEntityExtractor(object):
 
         Example:
             "5H7D_1": {
-                    "alignmentL": [
-                        {
-                            "refDbId": "P42588",
-                            "refDbName": "UniProt",
-                            "provSource": "PDB",
-                            "accession": "P42588",
-                            "taxId": 83333,
-                            "scientific_name": "Escherichia coli (strain K12)",
-                            "gene": "patA",
-                            "name": "PATase",
-                            "alignList": [
-                            {
-                                "srcId": "1",
-                                "entitySeqBeg": 5,
-                                "refSeqBeg": 7,
-                                "length": 447
-                            }
-                            ]
-                        },
-                        {
-                            "refDbId": "P38507",
-                            "refDbName": "UniProt",
-                            "provSource": "PDB",
-                            "accession": "P38507",
-                            "taxId": 1280,
-                            "scientific_name": "Staphylococcus aureus",
-                            "gene": "spa",
-                            "name": "IgG-binding protein A",
-                            "alignList": [
-                            {
-                                "srcId": "2",
-                                "entitySeqBeg": 452,
-                                entitySeqBeg"220,
-                                "length": 48
-                            }
-                            ]
-                        }
-                    ],
+                    # "alignmentL": [
+                    #     {
+                    #         "refDbId": "P42588",
+                    #         "refDbName": "UniProt",
+                    #         "provSource": "PDB",
+                    #         "accession": "P42588",
+                    #         "taxId": 83333,
+                    #         "scientific_name": "Escherichia coli (strain K12)",
+                    #         "gene": "patA",
+                    #         "name": "PATase",
+                    #         "alignList": [
+                    #         {
+                    #             "srcId": "1",
+                    #             "entitySeqBeg": 5,
+                    #             "refSeqBeg": 7,
+                    #             "length": 447
+                    #         }
+                    #         ]
+                    #     },
+                    #     {
+                    #         "refDbId": "P38507",
+                    #         "refDbName": "UniProt",
+                    #         "provSource": "PDB",
+                    #         "accession": "P38507",
+                    #         "taxId": 1280,
+                    #         "scientific_name": "Staphylococcus aureus",
+                    #         "gene": "spa",
+                    #         "name": "IgG-binding protein A",
+                    #         "alignList": [
+                    #         {
+                    #             "srcId": "2",
+                    #             "entitySeqBeg": 452,
+                    #             entitySeqBeg"220,
+                    #             "length": 48
+                    #         }
+                    #         ]
+                    #     }
+                    # ],
                     "sourceOrgL": [
                         {
                             "srcId": "1",
@@ -300,7 +301,7 @@ class PolymerEntityExtractor(object):
                     seqEnd = int(sD["entitySeqEnd"])
                     seqLen = 1 + (seqEnd - seqBeg)
                     # orgName = sD["orgName"]
-                    cD = {"sequence": seq[seqBeg - 1 : seqEnd], "entityId": eId, "srcId": srcId, "seqBeg": seqBeg, "seqEnd": seqEnd, "seqLen": seqLen, "taxId": taxId}
+                    cD = {"sequence": seq[seqBeg - 1: seqEnd], "entityId": eId, "srcId": srcId, "seqBeg": seqBeg, "seqEnd": seqEnd, "seqLen": seqLen, "taxId": taxId}
                     seqId = ""
                     cL = []
                     for k, v in cD.items():

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/seq/UniProtCoreEtlWorker.py

@@ -32,13 +32,14 @@ logger = logging.getLogger(__name__)
 class UniProtCoreEtlWorker(object):
     """Prepare and load UniProt 'core' sequence reference data collections."""
 
-    def __init__(self, cfgOb, cachePath, useCache=True, numProc=2, chunkSize=10, readBackCheck=False, documentLimit=None, doValidate=False, verbose=False):
+    def __init__(self, cfgOb, cachePath, useCache=True, numProc=2, chunkSize=10, maxStepLength=2000, readBackCheck=False, documentLimit=None, doValidate=False, verbose=False):
         self.__cfgOb = cfgOb
         self.__cachePath = cachePath
         self.__useCache = useCache
         self.__readBackCheck = readBackCheck
         self.__numProc = numProc
         self.__chunkSize = chunkSize
+        self.__maxStepLength = maxStepLength
         self.__documentLimit = documentLimit
         #
         self.__resourceName = "MONGO_DB"
@@ -128,6 +129,7 @@ class UniProtCoreEtlWorker(object):
                 self.__resourceName,
                 numProc=self.__numProc,
                 chunkSize=self.__chunkSize,
+                maxStepLength=self.__maxStepLength,
                 documentLimit=self.__documentLimit,
                 verbose=self.__verbose,
                 readBackCheck=self.__readBackCheck,

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/fixturePdbxLoader.py

@@ -65,7 +65,7 @@ class PdbxLoaderFixture(unittest.TestCase):
         self.__cachePath = os.path.join(TOPDIR, "CACHE")
         self.__readBackCheck = True
         self.__numProc = 1
-        self.__chunkSize = 5
+        self.__chunkSize = 2
         self.__fileLimit = 38
         self.__documentStyle = "rowwise_by_name_with_cardinality"
         #
@@ -121,44 +121,44 @@ class PdbxLoaderFixture(unittest.TestCase):
         ]
         #
         self.__pdbIdList = [
-            "1ah1",
-            "1b5f",
-            "1bmv",
-            "1c58",
-            "1dsr",
-            "1dul",
-            "1kqe",
-            "1o3q",
-            "1sfo",
-            "2hw3",
-            "2hyv",
-            "2osl",
-            "2voo",
-            "2wmg",
-            "3ad7",
-            "3hya",
-            "3iyd",
-            "3mbg",
-            "3rer",
-            "3vd8",
-            "3vfj",
-            "3x11",
-            "3ztj",
-            "4e2o",
-            "4en8",
-            "4mey",
-            "5eu8",
-            "5kds",
-            "5tm0",
-            "5vh4",
-            "5vp2",
-            "6fsz",
-            "6lu7",
-            "6nn7",
-            "6q20",
-            "6rfk",
-            "6rku",
-            "6yrq",
+            "1AH1",
+            "1B5F",
+            "1BMV",
+            "1C58",
+            "1DSR",
+            "1DUL",
+            "1KQE",
+            "1O3Q",
+            "1SFO",
+            "2HW3",
+            "2HYV",
+            "2OSL",
+            "2VOO",
+            "2WMG",
+            "3AD7",
+            "3HYA",
+            "3IYD",
+            "3MBG",
+            "3RER",
+            "3VD8",
+            "3VFJ",
+            "3X11",
+            "3ZTJ",
+            "4E2O",
+            "4EN8",
+            "4MEY",
+            "5EU8",
+            "5KDS",
+            # "5TM0",
+            "5VH4",
+            # "5VP2",
+            # "6FSZ",
+            "6LU7",
+            "6NN7",
+            # "6Q20",
+            "6RFK",
+            "6RKU",
+            "6YRQ",
         ]
         self.__ldList = [
             {
@@ -213,10 +213,12 @@ class PdbxLoaderFixture(unittest.TestCase):
     def testPdbxLoader(self):
         #
         for ld in self.__ldList:
-            self.__pdbxLoaderWrapper(**ld)
+            ok = self.__pdbxLoaderWrapper(**ld)
+            self.assertTrue(ok)
 
     def __pdbxLoaderWrapper(self, **kwargs):
         """Wrapper for the PDBx loader module"""
+        ok = False
         try:
             logger.info("Loading %s", kwargs["databaseName"])
             mw = PdbxLoader(
@@ -257,6 +259,7 @@ class PdbxLoaderFixture(unittest.TestCase):
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()
+        return ok
 
     def __loadStatus(self, statusList):
         sectionName = "data_exchange_configuration"

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/testExDbWorkflow.py

@@ -88,7 +88,8 @@ class ExDbWorkflowTests(unittest.TestCase):
     def testExDbLoaderWorkflows(self):
         """Test run workflow steps ..."""
         try:
-            opL = ["etl_chemref", "upd_ref_seq", "etl_tree_node_lists"]
+            # opL = ["etl_chemref", "upd_ref_seq", "etl_tree_node_lists"]
+            opL = ["etl_chemref", "etl_tree_node_lists"]
             rlWf = ExDbWorkflow(**self.__commonD)
             for op in opL:
                 ok = rlWf.load(op, **self.__loadCommonD)

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/testObjectExtractor.py

@@ -118,7 +118,7 @@ class ObjectExtractorTests(unittest.TestCase):
             eCount = obEx.getCount()
             logger.info("Entry count is %d", eCount)
             logger.info("Entries are %r", list(eD.keys()))
-            self.assertGreaterEqual(eCount, 6)
+            self.assertGreaterEqual(eCount, 5)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/testPolymerEntityExtractor.py

@@ -64,7 +64,7 @@ class PolymerEntityExtractorTests(unittest.TestCase):
             pEx = PolymerEntityExtractor(self.__cfgOb)
             pD, _ = pEx.getProteinSequenceDetails()
             #
-            self.assertGreaterEqual(len(pD), 100)
+            self.assertGreaterEqual(len(pD), 70)
             logger.info("Polymer entity count %d", len(pD))
         except Exception as e:
             logger.exception("Failing with %s", str(e))

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/testReferenceSequenceAssignmentProvider.py

@@ -91,7 +91,7 @@ class ReferenceSequenceAssignmentProviderTests(unittest.TestCase):
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 90)
+            self.assertGreaterEqual(numRef, 49)
             #
             # ---  Reload from cache ---
             rsaP = ReferenceSequenceAssignmentProvider(
@@ -100,7 +100,7 @@ class ReferenceSequenceAssignmentProviderTests(unittest.TestCase):
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 90)
+            self.assertGreaterEqual(numRef, 49)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()

{rcsb.exdb-0.99 → rcsb_exdb-1.1}/rcsb/exdb/tests/testReferenceSequenceCacheProvider.py

@@ -70,14 +70,14 @@ class ReferenceSequenceCacheProviderTests(unittest.TestCase):
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 90)
+            self.assertGreaterEqual(numRef, 49)
             #
             # ---  Reload from cache ---
             rsaP = ReferenceSequenceCacheProvider(self.__cfgOb, databaseName, collectionName, polymerType, maxChunkSize=50, numProc=2, expireDays=14)
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 90)
+            self.assertGreaterEqual(numRef, 49)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()