rcsb.exdb 1.1__tar.gz → 1.21__tar.gz

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/HISTORY.txt

@@ -99,3 +99,5 @@
                   This package update also coincides with the turning off of uniprot_exdb data loading during the weekly workflow
  6-May-2024 V1.1  Update ExDbExec CLI and ExDbWorkflow to support CLI usage from weekly-update workflow;
                   Update unit tests and setuptools config
+ 9-May-2024 V1.2  Adjust provider type exclusion input to accept a list of types
+13-May-2024 V1.21 Update requirements; fix linting

{rcsb_exdb-1.1/rcsb.exdb.egg-info → rcsb_exdb-1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.exdb
-Version: 1.1
+Version: 1.21
 Summary: RCSB Python ExDB data extraction and loading workflows
 Home-page: https://github.com/rcsb/py-rcsb_exdb
 Author: John Westbrook
@@ -18,18 +18,18 @@ License-File: LICENSE
 Requires-Dist: OpenEye-toolkits>=2020.2.2
 Requires-Dist: numpy
 Requires-Dist: jsonschema>=2.6.0
-Requires-Dist: rcsb.utils.io>=1.28
-Requires-Dist: rcsb.db>=1.691
-Requires-Dist: rcsb.utils.chemref>=0.79
-Requires-Dist: rcsb.utils.chem>=0.75
-Requires-Dist: rcsb.utils.citation>=0.16
-Requires-Dist: rcsb.utils.config>=0.35
-Requires-Dist: rcsb.utils.ec>=0.22
-Requires-Dist: rcsb.utils.go>=0.17
-Requires-Dist: rcsb.utils.seq>=0.63
-Requires-Dist: rcsb.utils.struct>=0.37
-Requires-Dist: rcsb.utils.taxonomy>=0.39
-Requires-Dist: rcsb.utils.dictionary>=0.71
+Requires-Dist: rcsb.utils.io>=1.46
+Requires-Dist: rcsb.db>=1.721
+Requires-Dist: rcsb.utils.chem>=0.79
+Requires-Dist: rcsb.utils.chemref>=0.91
+Requires-Dist: rcsb.utils.citation>=0.22
+Requires-Dist: rcsb.utils.config>=0.40
+Requires-Dist: rcsb.utils.ec>=0.25
+Requires-Dist: rcsb.utils.go>=0.18
+Requires-Dist: rcsb.utils.seq>=0.79
+Requires-Dist: rcsb.utils.struct>=0.47
+Requires-Dist: rcsb.utils.taxonomy>=0.43
+Requires-Dist: rcsb.utils.dictionary>=1.24
 Requires-Dist: rcsb.workflow>=0.42
 Requires-Dist: statistics; python_version < "3.0"
 Provides-Extra: dev

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/chemref/ChemRefEtlWorker.py

@@ -65,6 +65,7 @@ class ChemRefEtlWorker(object):
             self.__statusList = []
             desp = DataExchangeStatus()
             statusStartTimestamp = desp.setStartTime()
+            addValues = {}
             #
             if extResource == "DrugBank":
                 databaseName = "drugbank_core"
@@ -85,10 +86,10 @@ class ChemRefEtlWorker(object):
                 indexL = sD.getDocumentIndex(collectionName, "primary")
                 logger.info("Database %r collection %r index attributes %r", databaseName, collectionName, indexL)
                 #
-                collectionVersion = sD.getCollectionVersion(collectionName)
-                addValues = {"_schema_version": collectionVersion}
-                #
-                addValues = {}
+                # For some reason, 'addValues' was being overwritten with an empty dict (https://github.com/rcsb/py-rcsb_exdb/commit/26bd79e9a2fffc97c034b4116dece9248d1c1f39)
+                # Will need to review this -- do we want to add the schema version values or not? (Also, see similar logic in UniProtCoreEtlWorker.py)
+                # collectionVersion = sD.getCollectionVersion(collectionName)
+                # addValues = {"_schema_version": collectionVersion}
             #
             dl = DocumentLoader(
                 self.__cfgOb,

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/chemref/PubChemIndexCacheProvider.py

@@ -64,6 +64,8 @@ class PubChemUpdateWorker(object):
             (list): list of ChemicalIdentifier() objects corresponding to the input chemical component.
         """
         chemIdList = []
+        idType = None
+        descr = None
         if ccId in self.__lookupD:
             for sD in self.__lookupD[ccId]:
                 if "inchi-key" in sD:

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/cli/ExDbExec.py

@@ -71,7 +71,7 @@ def main():
     #
     parser.add_argument("--rebuild_cache", default=False, action="store_true", help="Rebuild cached resource files")
     parser.add_argument("--rebuild_sequence_cache", default=False, action="store_true", help="Rebuild cached resource files for reference sequence updates")
-    parser.add_argument("--provider_type_exclude", default=None, help="Resource provider types to exclude")
+    parser.add_argument("--provider_types_exclude", default=None, help="Resource provider types to exclude")
     parser.add_argument("--use_filtered_tax_list", default=False, action="store_true", help="Use filtered list for taxonomy tree loading")
     parser.add_argument("--disable_read_back_check", default=False, action="store_true", help="Disable read back check on all documents")
     parser.add_argument("--debug", default=False, action="store_true", help="Turn on verbose logging")
@@ -178,7 +178,7 @@ def processArguments(args):
         "mockTopPath": mockTopPath,
         "debugFlag": debugFlag,
         "rebuildCache": args.rebuild_cache,
-        "providerTypeExclude": args.provider_type_exclude,
+        "providerTypeExcludeL": args.provider_types_exclude,
     }
     loadD = {
         "loadType": args.load_type,

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/cli/__init__.py

@@ -2,4 +2,4 @@ __docformat__ = "google en"
 __author__ = "John Westbrook"
 __email__ = "john.westbrook@rcsb.org"
 __license__ = "Apache 2.0"
-__version__ = "1.1"
+__version__ = "1.21"

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/seq/UniProtCoreEtlWorker.py

@@ -95,6 +95,7 @@ class UniProtCoreEtlWorker(object):
             #
             dList = indexL = []
             databaseName = collectionName = collectionVersion = None
+            addValues = {}
             #
             if extResource == "UniProt":
                 databaseName = "uniprot_core"
@@ -114,7 +115,6 @@ class UniProtCoreEtlWorker(object):
                 collectionVersion = cDL[0]["VERSION"]
                 indexL = self.__docHelper.getDocumentIndexAttributes(collectionName, "primary")
                 logger.info("Database %r collection %r version %r index attributes %r", databaseName, collectionName, collectionVersion, indexL)
-                addValues = {}
             else:
                 logger.error("Unsupported external resource %r", extResource)
             #

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/tests/fixturePdbxLoader.py

@@ -48,7 +48,7 @@ class PdbxLoaderFixture(unittest.TestCase):
         #
         #
         self.__isMac = platform.system() == "Darwin"
-        self.__excludeType = None if self.__isMac else "optional"
+        self.__excludeTypeL = None if self.__isMac else ["optional"]
         self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
         configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
         # configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example-local.yml")
@@ -249,7 +249,7 @@ class PdbxLoaderFixture(unittest.TestCase):
                 validationLevel=kwargs["validationLevel"],
                 mergeContentTypes=kwargs["mergeContentTypes"],
                 useNameFlag=False,
-                providerTypeExclude=self.__excludeType,
+                providerTypeExcludeL=self.__excludeTypeL,
                 restoreUseGit=True,
                 restoreUseStash=False,
             )

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/tests/testExDbWorkflow.py

@@ -42,7 +42,7 @@ class ExDbWorkflowTests(unittest.TestCase):
 
     def setUp(self):
         self.__isMac = platform.system() == "Darwin"
-        self.__excludeType = None if self.__isMac else "optional"
+        self.__excludeTypeL = None if self.__isMac else ["optional"]
         mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
         configPath = os.path.join(mockTopPath, "config", "dbload-setup-example.yml")
         configName = "site_info_configuration"
@@ -55,7 +55,7 @@ class ExDbWorkflowTests(unittest.TestCase):
             "configName": configName,
             "cachePath": cachePath,
             "rebuildCache": False,
-            "providerTypeExclude": self.__excludeType,
+            "providerTypeExcludeL": self.__excludeTypeL,
             "restoreUseGit": True,
             "restoreUseStash": False,
         }

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/tests/testReferenceSequenceAnnotationAdapter.py

@@ -60,6 +60,7 @@ class ReferenceSequenceAnnotationAdapterTests(unittest.TestCase):
         endTime = time.time()
         logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
 
+    # @unittest.skip("Disable test - no longer using in production, and fails too frequently with 'Bad xml text' when fetching from UniProt")
     def testAnnotationAdapter(self):
         """Test case - create and read cache reference sequences assignments and related data."""
         try:
@@ -70,13 +71,13 @@ class ReferenceSequenceAnnotationAdapterTests(unittest.TestCase):
             rsaP = ReferenceSequenceAnnotationProvider(
                 self.__cfgOb, databaseName, collectionName, polymerType, fetchLimit=self.__fetchLimit, siftsAbbreviated="TEST", cachePath=self.__cachePath, useCache=True
             )
-            ok = rsaP.testCache()
+            ok = rsaP.testCache(minMissing=10)
             self.assertTrue(ok)
             numRef1 = rsaP.getRefDataCount()
             #
             # ---  Reload from cache ---
             rsaP = ReferenceSequenceAnnotationProvider(self.__cfgOb, databaseName, collectionName, polymerType, cachePath=self.__cachePath, useCache=True)
-            ok = rsaP.testCache()
+            ok = rsaP.testCache(minMissing=10)
             self.assertTrue(ok)
             numRef2 = rsaP.getRefDataCount()
             self.assertEqual(numRef1, numRef2)

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/tests/testReferenceSequenceCacheProvider.py

@@ -70,14 +70,14 @@ class ReferenceSequenceCacheProviderTests(unittest.TestCase):
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 49)
+            self.assertGreaterEqual(numRef, 44)
             #
             # ---  Reload from cache ---
             rsaP = ReferenceSequenceCacheProvider(self.__cfgOb, databaseName, collectionName, polymerType, maxChunkSize=50, numProc=2, expireDays=14)
             ok = rsaP.testCache()
             self.assertTrue(ok)
             numRef = rsaP.getRefDataCount()
-            self.assertGreaterEqual(numRef, 49)
+            self.assertGreaterEqual(numRef, 44)
         except Exception as e:
             logger.exception("Failing with %s", str(e))
             self.fail()

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/utils/ObjectTransformer.py

@@ -47,6 +47,7 @@ class ObjectTransformer(object):
         docSelectList = docSelectList[:fetchLimit] if fetchLimit else docSelectList
         ok = self.__transform(databaseName, collectionName, docSelectList)
         #
+        okS = True
         if updateId:
             okS = self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
         return ok and okS

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/utils/ObjectValidator.py

@@ -78,6 +78,7 @@ class ObjectValidator(object):
 
         ok = self.__transform(databaseName, collectionName, docSelectList)
         #
+        okS = True
         if updateId:
             okS = self.__updateStatus(updateId, databaseName, collectionName, ok, statusStartTimestamp)
         return ok and okS

{rcsb_exdb-1.1 → rcsb_exdb-1.21}/rcsb/exdb/wf/ExDbWorkflow.py

@@ -54,7 +54,7 @@ class ExDbWorkflow(object):
         # Rebuild cache (default False)
         rebuildCache = kwargs.get("rebuildCache", False)
         self.__useCache = not rebuildCache
-        providerTypeExclude = kwargs.get("providerTypeExclude", None)
+        providerTypeExcludeL = kwargs.get("providerTypeExcludeL", None)
         restoreUseGit = kwargs.get("restoreUseGit", True)
         restoreUseStash = kwargs.get("restoreUseStash", True)
         self.__cacheStatus = True
@@ -62,7 +62,7 @@ class ExDbWorkflow(object):
             logger.info("Rebuilding cache %r", rebuildCache)
             self.__cacheStatus = self.buildResourceCache(
                 rebuildCache=rebuildCache,
-                providerTypeExclude=providerTypeExclude,
+                providerTypeExcludeL=providerTypeExcludeL,
                 restoreUseStash=restoreUseStash,
                 restoreUseGit=restoreUseGit,
             )
@@ -204,7 +204,7 @@ class ExDbWorkflow(object):
             logger.exception("Failing with %s", str(e))
         return ret
 
-    def buildResourceCache(self, rebuildCache=False, providerTypeExclude=None, restoreUseStash=True, restoreUseGit=True):
+    def buildResourceCache(self, rebuildCache=False, providerTypeExcludeL=None, restoreUseStash=True, restoreUseGit=True):
         """Generate and cache resource dependencies."""
         ret = False
         try:
@@ -223,7 +223,7 @@ class ExDbWorkflow(object):
                 cachePath=self.__cachePath,
                 restoreUseStash=restoreUseStash,
                 restoreUseGit=restoreUseGit,
-                providerTypeExclude=providerTypeExclude,
+                providerTypeExcludeL=providerTypeExcludeL,
             )
             ret = rP.cacheResources(useCache=useCache, doBackup=False)
             logger.info("useCache %r cache reload status (%r)", useCache, ret)

{rcsb_exdb-1.1 → rcsb_exdb-1.21/rcsb.exdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rcsb.exdb
-Version: 1.1
+Version: 1.21
 Summary: RCSB Python ExDB data extraction and loading workflows
 Home-page: https://github.com/rcsb/py-rcsb_exdb
 Author: John Westbrook
@@ -18,18 +18,18 @@ License-File: LICENSE
 Requires-Dist: OpenEye-toolkits>=2020.2.2
 Requires-Dist: numpy
 Requires-Dist: jsonschema>=2.6.0
-Requires-Dist: rcsb.utils.io>=1.28
-Requires-Dist: rcsb.db>=1.691
-Requires-Dist: rcsb.utils.chemref>=0.79
-Requires-Dist: rcsb.utils.chem>=0.75
-Requires-Dist: rcsb.utils.citation>=0.16
-Requires-Dist: rcsb.utils.config>=0.35
-Requires-Dist: rcsb.utils.ec>=0.22
-Requires-Dist: rcsb.utils.go>=0.17
-Requires-Dist: rcsb.utils.seq>=0.63
-Requires-Dist: rcsb.utils.struct>=0.37
-Requires-Dist: rcsb.utils.taxonomy>=0.39
-Requires-Dist: rcsb.utils.dictionary>=0.71
+Requires-Dist: rcsb.utils.io>=1.46
+Requires-Dist: rcsb.db>=1.721
+Requires-Dist: rcsb.utils.chem>=0.79
+Requires-Dist: rcsb.utils.chemref>=0.91
+Requires-Dist: rcsb.utils.citation>=0.22
+Requires-Dist: rcsb.utils.config>=0.40
+Requires-Dist: rcsb.utils.ec>=0.25
+Requires-Dist: rcsb.utils.go>=0.18
+Requires-Dist: rcsb.utils.seq>=0.79
+Requires-Dist: rcsb.utils.struct>=0.47
+Requires-Dist: rcsb.utils.taxonomy>=0.43
+Requires-Dist: rcsb.utils.dictionary>=1.24
 Requires-Dist: rcsb.workflow>=0.42
 Requires-Dist: statistics; python_version < "3.0"
 Provides-Extra: dev

rcsb_exdb-1.21/rcsb.exdb.egg-info/requires.txt

@@ -0,0 +1,25 @@
+OpenEye-toolkits>=2020.2.2
+numpy
+jsonschema>=2.6.0
+rcsb.utils.io>=1.46
+rcsb.db>=1.721
+rcsb.utils.chem>=0.79
+rcsb.utils.chemref>=0.91
+rcsb.utils.citation>=0.22
+rcsb.utils.config>=0.40
+rcsb.utils.ec>=0.25
+rcsb.utils.go>=0.18
+rcsb.utils.seq>=0.79
+rcsb.utils.struct>=0.47
+rcsb.utils.taxonomy>=0.43
+rcsb.utils.dictionary>=1.24
+rcsb.workflow>=0.42
+
+[:python_version < "3.0"]
+statistics
+
+[dev]
+check-manifest
+
+[test]
+coverage

rcsb_exdb-1.21/requirements.txt

@@ -0,0 +1,18 @@
+--extra-index-url https://pypi.anaconda.org/OpenEye/simple
+OpenEye-toolkits >= 2020.2.2
+numpy
+jsonschema >= 2.6.0
+rcsb.utils.io >= 1.46
+rcsb.db >= 1.721
+rcsb.utils.chem >= 0.79
+rcsb.utils.chemref >= 0.91
+rcsb.utils.citation >= 0.22
+rcsb.utils.config >= 0.40
+rcsb.utils.ec >= 0.25
+rcsb.utils.go >= 0.18
+rcsb.utils.seq >= 0.79
+rcsb.utils.struct >= 0.47
+rcsb.utils.taxonomy >= 0.43
+rcsb.utils.dictionary >= 1.24
+rcsb.workflow >= 0.42
+statistics; python_version < "3.0"

rcsb_exdb-1.1/rcsb.exdb.egg-info/requires.txt

@@ -1,25 +0,0 @@
-OpenEye-toolkits>=2020.2.2
-numpy
-jsonschema>=2.6.0
-rcsb.utils.io>=1.28
-rcsb.db>=1.691
-rcsb.utils.chemref>=0.79
-rcsb.utils.chem>=0.75
-rcsb.utils.citation>=0.16
-rcsb.utils.config>=0.35
-rcsb.utils.ec>=0.22
-rcsb.utils.go>=0.17
-rcsb.utils.seq>=0.63
-rcsb.utils.struct>=0.37
-rcsb.utils.taxonomy>=0.39
-rcsb.utils.dictionary>=0.71
-rcsb.workflow>=0.42
-
-[:python_version < "3.0"]
-statistics
-
-[dev]
-check-manifest
-
-[test]
-coverage

rcsb_exdb-1.1/requirements.txt

@@ -1,18 +0,0 @@
---extra-index-url https://pypi.anaconda.org/OpenEye/simple
-OpenEye-toolkits >= 2020.2.2
-numpy
-jsonschema >= 2.6.0
-rcsb.utils.io >= 1.28
-rcsb.db >= 1.691
-rcsb.utils.chemref >= 0.79
-rcsb.utils.chem >= 0.75
-rcsb.utils.citation >= 0.16
-rcsb.utils.config >= 0.35
-rcsb.utils.ec >= 0.22
-rcsb.utils.go >= 0.17
-rcsb.utils.seq >= 0.63
-rcsb.utils.struct >= 0.37
-rcsb.utils.taxonomy >= 0.39
-rcsb.utils.dictionary >= 0.71
-rcsb.workflow >= 0.42
-statistics; python_version < "3.0"