rcsb.workflow 0.63__tar.gz → 0.65__tar.gz

rcsb_workflow-0.65/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+LOGTODAY
+ .idea/
+.vscode/
+.env/
+test-output/
+!test-output/.gitkeep
+CACHE/
+LOGTOX
+Archive/
+
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

{rcsb_workflow-0.63 → rcsb_workflow-0.65}/HISTORY.txt

@@ -56,5 +56,6 @@
  4-Sep-2025 - V0.60 Update requirements and docker file
 10-Sep-2025 - V0.61 Return without error in bcif task functions, update content prefixes
 30-Sep-2025 - V0.62 Let jpg gen use model file over URL
- 9-Sep-2025 - V0.63 Add ChemSearchIndex from weekly-update-workflow so it can be a cli call instead
- 
+15-Oct-2025 - V0.63 Add ChemSearchIndex from weekly-update-workflow so it can be a cli call instead
+ 8-Dec-2025 - V0.64 Switch to 'pyproject.toml' file and update test pipelines to python 3.12
+ 6-Jan-2026 - V0.65 Reduce resource utilization for test suites

{rcsb_workflow-0.63/rcsb.workflow.egg-info → rcsb_workflow-0.65}/PKG-INFO

@@ -1,45 +1,37 @@
 Metadata-Version: 2.4
 Name: rcsb.workflow
-Version: 0.63
+Version: 0.65
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
-Home-page: https://github.com/rcsb/py-rcsb_workflow
-Author: John Westbrook
-Author-email: john.westbrook@rcsb.org
-License: Apache 2.0
-Classifier: Development Status :: 3 - Alpha
+Project-URL: Homepage, https://github.com/rcsb/py-rcsb_workflow
+Author-email: John Westbrook <john.westbrook@rcsb.org>
+Maintainer-email: Dennis Piehl <dennis.piehl@rcsb.org>, Michael Trumbull <michael.trumbull@rcsb.org>, James Smith <james.smith@rcsb.org>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: rcsb.utils.chem>=0.82
-Requires-Dist: rcsb.utils.seq>=0.84
-Requires-Dist: rcsb.utils.targets>=0.83
-Requires-Dist: rcsb.utils.seqalign>=0.31
-Requires-Dist: rcsb.utils.io>=1.49
-Requires-Dist: rcsb.exdb>=1.30
-Requires-Dist: rcsb.db>=1.808
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: mmcif>=1.0.0
+Requires-Dist: openeye-toolkits>=2024.1.1
+Requires-Dist: rcsb-db>=1.812
+Requires-Dist: rcsb-exdb>=1.33
+Requires-Dist: rcsb-utils-chem>=0.84
+Requires-Dist: rcsb-utils-io>=1.49
+Requires-Dist: rcsb-utils-seq>=0.84
+Requires-Dist: rcsb-utils-seqalign>=0.32
+Requires-Dist: rcsb-utils-targets>=0.83
 Requires-Dist: requests>=2.32.3
-Requires-Dist: mmcif>=0.91.0
-Provides-Extra: dev
-Requires-Dist: check-manifest; extra == "dev"
-Provides-Extra: test
-Requires-Dist: coverage; extra == "test"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license
-Dynamic: license-file
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: summary
+Provides-Extra: tests
+Requires-Dist: black>=21.5b1; extra == 'tests'
+Requires-Dist: check-manifest; extra == 'tests'
+Requires-Dist: coverage; extra == 'tests'
+Requires-Dist: flake8; extra == 'tests'
+Requires-Dist: pylint; extra == 'tests'
+Requires-Dist: tox; extra == 'tests'
+Description-Content-Type: text/markdown
 
 # py-rcsb_workflow
 
@@ -63,18 +55,11 @@ git submodule update --recursive --remote
 
 ```
 
-Optionally, run test suite (Python versions 3.8) using
-[setuptools](https://setuptools.readthedocs.io/en/latest/) or
+Optionally, run test suite (Python versions >= 3.9) using
 [tox](http://tox.readthedocs.io/en/latest/example/platform.html):
 
 ```bash
-
-  pip install -r requirements.txt
-  python setup.py test
-
-or simply run:
-
-  tox
+tox
 ```
 
 Installation is via the program [pip](https://pypi.python.org/pypi/pip).
@@ -85,4 +70,8 @@ pip install rcsb.workflow --extra-index-url https://pypi.anaconda.org/OpenEye/si
 To run tests from the source tree, the package must be installed in editable mode (i.e. -e):
 ```bash
 pip install -e . --extra-index-url https://pypi.anaconda.org/OpenEye/simple
+
+# or using uv
+
+uv pip install -e .
 ```

{rcsb_workflow-0.63 → rcsb_workflow-0.65}/README.md

@@ -20,18 +20,11 @@ git submodule update --recursive --remote
 
 ```
 
-Optionally, run test suite (Python versions 3.8) using
-[setuptools](https://setuptools.readthedocs.io/en/latest/) or
+Optionally, run test suite (Python versions >= 3.9) using
 [tox](http://tox.readthedocs.io/en/latest/example/platform.html):
 
 ```bash
-
-  pip install -r requirements.txt
-  python setup.py test
-
-or simply run:
-
-  tox
+tox
 ```
 
 Installation is via the program [pip](https://pypi.python.org/pypi/pip).
@@ -42,4 +35,8 @@ pip install rcsb.workflow --extra-index-url https://pypi.anaconda.org/OpenEye/si
 To run tests from the source tree, the package must be installed in editable mode (i.e. -e):
 ```bash
 pip install -e . --extra-index-url https://pypi.anaconda.org/OpenEye/simple
+
+# or using uv
+
+uv pip install -e .
 ```

rcsb_workflow-0.65/pyproject.toml

@@ -0,0 +1,103 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "rcsb.workflow"
+description = "RCSB Python data processing and ETL/ELT workflow entry points"
+version = "0.65"
+readme = "README.md"
+authors = [
+    { name="John Westbrook", email="john.westbrook@rcsb.org" }
+]
+maintainers = [
+    { name="Dennis Piehl", email="dennis.piehl@rcsb.org" },
+    { name="Michael Trumbull", email="michael.trumbull@rcsb.org" },
+    { name="James Smith", email="james.smith@rcsb.org" }
+]
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+requires-python = ">=3.9"  # Latest OpenEye (2025.1.1) only supports up to 3.12 (as of Nov. 2025)
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",  # Latest OpenEye (2025.1.1) only supports up to 3.12 (as of Nov. 2025)
+]
+
+dependencies = [
+    # OpenEye is needed here despite it not being a direct requirement of this package (it's used by rcsb.utils.chem)
+    "OpenEye-toolkits >= 2024.1.1",
+    #
+    "mmcif >= 1.0.0",
+    "rcsb.utils.io >= 1.49",
+    "rcsb.db >= 1.812",
+    "rcsb.utils.chem >= 0.84",
+    "rcsb.utils.seq >= 0.84",
+    "rcsb.utils.targets >= 0.83",
+    "rcsb.utils.seqalign >= 0.32",
+    "rcsb.exdb >= 1.33",
+    "requests >= 2.32.3",
+]
+
+[project.optional-dependencies]
+tests = ["tox", "pylint", "black>=21.5b1", "flake8", "coverage", "check-manifest"]
+
+[project.urls]
+Homepage = "https://github.com/rcsb/py-rcsb_workflow"
+
+[project.scripts]
+exdb_wf_cli = "rcsb.workflow.cli.ExDbExec:main"
+imgs_exec_cli = "rcsb.workflow.cli.ImgExec:main"
+bcif_exec_cli = "rcsb.workflow.cli.BcifExec:main"
+chem_search_index_cli = "rcsb.workflow.cli.ChemSearchIndexExec:main"
+
+
+# ----------------  hatch configuration ----------------
+[tool.hatch.build.targets.sdist]
+include = [
+    "/rcsb/**",
+    "/README.md",
+    "/LICENSE",
+    "/HISTORY.txt",
+    "/pyproject.toml",
+]
+exclude = [
+    "/rcsb/mock-*",
+    "/rcsb/workflow/test*/test*-*",
+    "/rcsb/workflow/test*/**.tdd",
+    "/rcsb/workflow/test*/**.txt",
+    "/rcsb/workflow/test*/**.sh",
+    "/rcsb/workflow/test*/**.gz",
+    "/rcsb/workflow/test*/**.pic",
+    "/rcsb/workflow/test*/**.cif",
+    "/rcsb/workflow/test*/**.json",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["rcsb"]
+exclude = [
+    "/rcsb/workflow/test*",
+    "/rcsb/mock-*",
+]
+
+[tool.hatch.envs.default]
+skip-install = true
+dependencies = ["OpenEye-toolkits>=2024.1.1"]
+
+[tool.hatch.envs.default.env-vars]
+PIP_EXTRA_INDEX_URL = "https://pypi.anaconda.org/OpenEye/simple"
+
+# ----------------  test configuration ----------------
+[tool.hatch.envs.hatch-test]
+dependencies = ["tox"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = ["tox"]
+
+# ----------------  UV configuration ----------------
+[tool.uv]
+extra-index-url = ["https://pypi.anaconda.org/OpenEye/simple"]
+managed = false  # This prevents uv from automatically creating venvs within individual packages when running 'uv run ...' from within the package directory

rcsb_workflow-0.65/rcsb/workflow/tests/fixturePdbxLoader.py

@@ -0,0 +1,291 @@
+##
+# File:    PdbxLoaderFixture.py
+# Author:  J. Westbrook
+# Date:    4-Sep-2019
+# Version: 0.001
+#
+# Updates:
+#  04-Feb-2025 mjt Copied this file over from rcsb.exdb
+##
+"""
+Fixture for loading the chemical reference and pdbx_core collections in a loca mongo instance.
+
+"""
+
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+# import glob
+import logging
+import os
+import platform
+import resource
+import time
+import unittest
+import shutil
+
+from rcsb.db.mongo.DocumentLoader import DocumentLoader
+from rcsb.db.mongo.PdbxLoader import PdbxLoader
+from rcsb.utils.config.ConfigUtil import ConfigUtil
+# from rcsb.utils.io.FileUtil import FileUtil
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+
+
+class PdbxLoaderFixture(unittest.TestCase):
+
+    def __init__(self, methodName="runTest"):
+        super(PdbxLoaderFixture, self).__init__(methodName)
+        self.__verbose = True
+
+    def setUp(self):
+        #
+        #
+        self.__isMac = platform.system() == "Darwin"
+        self.__excludeTypeL = None if self.__isMac else ["optional"]
+        self.__mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data")
+        configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example.yml")
+        # configPath = os.path.join(TOPDIR, "rcsb", "mock-data", "config", "dbload-setup-example-local.yml")
+        # To Do: Investigate why GitUtil sometimes gives divergence error when using 'DISCOVERY_MODE: remote', but not with 'local':
+        #            stderr: 'fatal: Need to specify how to reconcile divergent branches.'
+        #        Behavior isn't entirely predictable, since it happens sometimes but not all the time.
+        #        To fully debug, will need to add more logging statements to GitUtil, StashableBase, & StashUtil (in rcsb.utils.io)
+        #        Or, can try to resolve error directly by specifying how to reconcile diverent branches in git.Repo class.
+        configName = "site_info_configuration"
+        self.__cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=self.__mockTopPath)
+        #
+        self._disk_before = shutil.disk_usage(HERE).used
+        logger.info("Filesystem disk usage start: %.2f MB", self._disk_before / (1024 ** 2))
+        #
+        self.__resourceName = "MONGO_DB"
+        self.__failedFilePath = os.path.join(HERE, "test-output", "failed-list.txt")
+        self.__cachePath = os.path.join(TOPDIR, "CACHE")
+        self.__readBackCheck = True
+        self.__numProc = 1
+        self.__chunkSize = 2
+        self.__fileLimit = 38
+        self.__documentStyle = "rowwise_by_name_with_cardinality"
+        #
+        self.__birdChemCompCoreIdList = [
+            "PRD_000010",
+            "PRD_000060",
+            "PRD_000220",
+            "PRD_000882",
+            "PRD_000154",
+            "PRD_000877",
+            "PRD_000198",
+            "PRD_000009",
+            "PRD_000979",
+            "PRDCC_000010",
+            "PRDCC_000220",
+            "PRDCC_000882",
+            "PRDCC_000154",
+            "PRDCC_000198",
+            "PRDCC_000009",
+            "FAM_000010",
+            "FAM_000210",
+            "FAM_000220",
+            "FAM_000001",
+            "FAM_000391",
+            "FAM_000093",
+            "FAM_000084",
+            "FAM_000016",
+            "FAM_000336",
+            "1G1",
+            "2RT",
+            "2XL",
+            "2XN",
+            "ATP",
+            "BJA",
+            "BM3",
+            "CNC",
+            "DAL",
+            "DDZ",
+            "DHA",
+            "DSN",
+            "GTP",
+            "HKL",
+            "NAC",
+            "NAG",
+            "NND",
+            "PTR",
+            "SEP",
+            "SMJ",
+            "STL",
+            "UNK",
+            "UNX",
+            "UVL",
+        ]
+        #
+        self.__pdbIdList = [
+            "1AH1",
+            "1B5F",
+            "1BMV",
+            "1C58",
+            "1DSR",
+            "1DUL",
+            "1KQE",
+            "1O3Q",
+            "1SFO",
+            "2HW3",
+            "2HYV",
+            "2OSL",
+            "2VOO",
+            "2WMG",
+            "3AD7",
+            "3HYA",
+            "3IYD",
+            "3MBG",
+            "3RER",
+            "3VD8",
+            "3VFJ",
+            "3X11",
+            "3ZTJ",
+            "4E2O",
+            "4EN8",
+            "4MEY",
+            "5EU8",
+            "5KDS",
+            # "5TM0",
+            "5VH4",
+            # "5VP2",
+            # "6FSZ",
+            "6LU7",
+            "6NN7",
+            # "6Q20",
+            "6RFK",
+            "6RKU",
+            "6YRQ",
+        ]
+        self.__ldList = [
+            {
+                # "databaseName": "dw",
+                "collectionGroupName": "core_chem_comp",
+                "contentType": "bird_chem_comp_core",
+                "collectionNameList": None,
+                "loadType": "full",
+                "mergeContentTypes": None,
+                "validationLevel": "min",
+                "inputIdCodeList": self.__birdChemCompCoreIdList
+            },
+            {
+                # "databaseName": "pdbx_core",
+                "collectionGroupName": "pdbx_core",
+                "contentType": "pdbx_core",
+                "collectionNameList": None,
+                "loadType": "replace",
+                "mergeContentTypes": ["vrpt"],
+                "validationLevel": "min",
+                "inputIdCodeList": self.__pdbIdList
+            },
+            # {
+            #     "databaseName": "pdbx_comp_model_core",
+            #     "collectionNameList": None,
+            #     "loadType": "full",
+            #     "mergeContentTypes": None,
+            #     "validationLevel": "min",
+            #     "inputIdCodeList": None
+            # },
+        ]
+        #
+        # self.__modelFixture()
+        self.__startTime = time.time()
+        logger.debug("Starting %s at %s", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()))
+
+    def tearDown(self):
+        unitS = "MB" if platform.system() == "Darwin" else "GB"
+        rusageMax = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        logger.info("Maximum resident memory size %.4f %s", rusageMax / 10 ** 6, unitS)
+        endTime = time.time()
+        disk_after = shutil.disk_usage(HERE).used
+        disk_delta = disk_after - self._disk_before
+        logger.info("Filesystem disk usage delta: %.2f MB", disk_delta / (1024 ** 2))
+        logger.info("Completed %s at %s (%.4f seconds)", self.id(), time.strftime("%Y %m %d %H:%M:%S", time.localtime()), endTime - self.__startTime)
+
+    def testPdbxLoader(self):
+        #
+        for ld in self.__ldList:
+            ok = self.__pdbxLoaderWrapper(**ld)
+            self.assertTrue(ok)
+
+    def __pdbxLoaderWrapper(self, **kwargs):
+        """Wrapper for the PDBx loader module"""
+        ok = False
+        try:
+            logger.info("Loading %s", kwargs["collectionGroupName"])
+            mw = PdbxLoader(
+                self.__cfgOb,
+                cachePath=self.__cachePath,
+                resourceName=self.__resourceName,
+                numProc=self.__numProc,
+                chunkSize=self.__chunkSize,
+                fileLimit=kwargs.get("fileLimit", self.__fileLimit),
+                verbose=self.__verbose,
+                readBackCheck=self.__readBackCheck,
+                maxStepLength=1000,
+                useSchemaCache=True,
+                rebuildSchemaFlag=False,
+            )
+            ok = mw.load(
+                collectionGroupName=kwargs["collectionGroupName"],
+                collectionLoadList=kwargs["collectionNameList"],
+                contentType=kwargs["contentType"],
+                loadType=kwargs["loadType"],
+                inputPathList=None,
+                inputIdCodeList=kwargs["inputIdCodeList"],
+                styleType=self.__documentStyle,
+                dataSelectors=["PUBLIC_RELEASE"],
+                failedFilePath=self.__failedFilePath,
+                saveInputFileListPath=None,
+                pruneDocumentSize=None,
+                logSize=False,
+                validationLevel=kwargs["validationLevel"],
+                mergeContentTypes=kwargs["mergeContentTypes"],
+                useNameFlag=False,
+                providerTypeExcludeL=self.__excludeTypeL,
+                restoreUseGit=True,
+                restoreUseStash=False,
+            )
+            self.assertTrue(ok)
+            ok = self.__loadStatus(mw.getLoadStatus())
+            self.assertTrue(ok)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+            self.fail()
+        return ok
+
+    def __loadStatus(self, statusList):
+        sectionName = "data_exchange_configuration"
+        dl = DocumentLoader(
+            self.__cfgOb,
+            self.__cachePath,
+            resourceName=self.__resourceName,
+            numProc=self.__numProc,
+            chunkSize=self.__chunkSize,
+            documentLimit=None,
+            verbose=self.__verbose,
+            readBackCheck=self.__readBackCheck,
+        )
+        #
+        databaseName = self.__cfgOb.get("DATABASE_NAME", sectionName=sectionName)
+        collectionName = self.__cfgOb.get("COLLECTION_UPDATE_STATUS", sectionName=sectionName)
+        ok = dl.load(databaseName, collectionName, loadType="append", documentList=statusList, indexAttributeList=["update_id", "database_name", "object_name"], keyNames=None)
+        return ok
+
+
+def mongoLoadPdbxSuite():
+    suiteSelect = unittest.TestSuite()
+    suiteSelect.addTest(PdbxLoaderFixture("testPdbxLoader"))
+    return suiteSelect
+
+
+if __name__ == "__main__":
+    mySuite = mongoLoadPdbxSuite()
+    unittest.TextTestRunner(verbosity=2).run(mySuite)