rcsb.workflow 0.65__tar.gz → 0.67__tar.gz

{rcsb_workflow-0.65 → rcsb_workflow-0.67}/HISTORY.txt

@@ -59,3 +59,6 @@
 15-Oct-2025 - V0.63 Add ChemSearchIndex from weekly-update-workflow so it can be a cli call instead
  8-Dec-2025 - V0.64 Switch to 'pyproject.toml' file and update test pipelines to python 3.12
  6-Jan-2026 - V0.65 Reduce resource utilization for test suites
+ 2-Mar-2026 - V0.66 Add refstats subpackage for ligand and RSCC reference generation;
+                    Update test pipelines to python 3.13
+ 7-May-2026 - V0.67 Configure logging at start of CLI executable scripts

{rcsb_workflow-0.65 → rcsb_workflow-0.67}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rcsb.workflow
-Version: 0.65
+Version: 0.67
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Project-URL: Homepage, https://github.com/rcsb/py-rcsb_workflow
 Author-email: John Westbrook <john.westbrook@rcsb.org>
@@ -12,18 +12,22 @@ Classifier: Intended Audience :: Developers
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9
 Requires-Dist: mmcif>=1.0.0
+Requires-Dist: numpy>=2.1.3
 Requires-Dist: openeye-toolkits>=2024.1.1
-Requires-Dist: rcsb-db>=1.812
-Requires-Dist: rcsb-exdb>=1.33
+Requires-Dist: pymongo>=4.8.0
+Requires-Dist: rcsb-db>=1.814
+Requires-Dist: rcsb-exdb>=1.35
 Requires-Dist: rcsb-utils-chem>=0.84
+Requires-Dist: rcsb-utils-chemref>=0.98
 Requires-Dist: rcsb-utils-io>=1.49
 Requires-Dist: rcsb-utils-seq>=0.84
 Requires-Dist: rcsb-utils-seqalign>=0.32
 Requires-Dist: rcsb-utils-targets>=0.83
 Requires-Dist: requests>=2.32.3
+Requires-Dist: scikit-learn>=1.8.0
 Provides-Extra: tests
 Requires-Dist: black>=21.5b1; extra == 'tests'
 Requires-Dist: check-manifest; extra == 'tests'

{rcsb_workflow-0.65 → rcsb_workflow-0.67}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "rcsb.workflow"
 description = "RCSB Python data processing and ETL/ELT workflow entry points"
-version = "0.65"
+version = "0.67"
 readme = "README.md"
 authors = [
     { name="John Westbrook", email="john.westbrook@rcsb.org" }
@@ -17,29 +17,34 @@ maintainers = [
 ]
 license = "Apache-2.0"
 license-files = ["LICENSE"]
-requires-python = ">=3.9"  # Latest OpenEye (2025.1.1) only supports up to 3.12 (as of Nov. 2025)
+requires-python = ">=3.9"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Natural Language :: English",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",  # Latest OpenEye (2025.1.1) only supports up to 3.12 (as of Nov. 2025)
+    "Programming Language :: Python :: 3.13",  # Latest OpenEye (2025.2) supports up to 3.13
 ]
 
 dependencies = [
     # OpenEye is needed here despite it not being a direct requirement of this package (it's used by rcsb.utils.chem)
     "OpenEye-toolkits >= 2024.1.1",
     #
+    "requests >= 2.32.3",
+    "scikit-learn >= 1.8.0",
+    "numpy >= 2.1.3",
+    "pymongo >= 4.8.0",
+    #
     "mmcif >= 1.0.0",
     "rcsb.utils.io >= 1.49",
-    "rcsb.db >= 1.812",
+    "rcsb.db >= 1.814",
     "rcsb.utils.chem >= 0.84",
     "rcsb.utils.seq >= 0.84",
     "rcsb.utils.targets >= 0.83",
+    "rcsb.utils.chemref >= 0.98",
     "rcsb.utils.seqalign >= 0.32",
-    "rcsb.exdb >= 1.33",
-    "requests >= 2.32.3",
+    "rcsb.exdb >= 1.35",
 ]
 
 [project.optional-dependencies]

{rcsb_workflow-0.65 → rcsb_workflow-0.67}/rcsb/workflow/cli/ExDbExec.py

@@ -14,6 +14,7 @@
 #  22-Oct-2024 dwp Add ccd_img_gen and ccd_file_gen operations
 #                  (latter will only be used briefly, as will stop generating SDF and Mol2 files in Dec 2024)
 #  31-Jan-2025 mjt Moved this script from rcsb.exdb (to remove circluar dependencies)
+#  10-Jan-2025  cs Add ligand_quality_ref_gen and residue_rscc_ref_gen operations
 ##
 __docformat__ = "google en"
 __author__ = "John Westbrook"
@@ -31,7 +32,7 @@ from rcsb.workflow.wuw.ExDbWorkflow import ExDbWorkflow
 HERE = os.path.abspath(os.path.dirname(__file__))
 TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
 
-# logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s", stream=sys.stdout)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
 logger = logging.getLogger()
 
 
@@ -58,6 +59,8 @@ def main():
             "upd_resource_stash",
             "ccd_img_gen",
             "ccd_file_gen",
+            "ligand_quality_ref_gen",
+            "residue_rscc_ref_gen",
         ]
     )
     parser.add_argument(
@@ -129,6 +132,8 @@ def main():
         "upd_entry_info",
         "upd_glycan_idx",
         "upd_resource_stash",
+        "ligand_quality_ref_gen",
+        "residue_rscc_ref_gen",
     ]:
         okR = exWf.buildExdbResource(op, **loadD)
     elif op in [

rcsb_workflow-0.67/rcsb/workflow/refstats/LigandQualityReferenceGenerator.py

@@ -0,0 +1,342 @@
+##
+#  File:           LigandQualityReferenceGenerator.py
+#  Date:           2025-12-25 Chenghua Shao
+#
+#  Update:
+##
+"""
+Generate ligand quality reference data that is used for ligand quality score computation.
+
+"""
+import os
+import csv
+import logging
+import time
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import StandardScaler
+
+from rcsb.utils.io.FileUtil import FileUtil
+from rcsb.db.mongo.Connection import Connection
+from rcsb.utils.chemref.RcsbLigandScoreProvider import RcsbLigandScoreProvider
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseError(Exception):
+    """Custom exception class for errors in MongoDB connection and query execution."""
+
+
+class AnalysisError(Exception):
+    """Custom exception class for errors during data analysis."""
+
+
+class OutputError(Exception):
+    """Custom exception class for errors during output file writing."""
+
+
+class LigandQualityReferenceGenerator:
+    """
+    This class generates ligand quality reference data by performing the following steps:
+    1. Query ligand quality metrics from RCSB MongoDB resource;
+    2. Process the ligand quality data by filtering, aggregating, and formatting;
+    3. Generate ligand reference data through PCA;
+        Attributes:
+            data: final output of each pdb_ligand combination and their quality scores
+            qdata: direct data from MongoDB query
+        Methods:
+            fetchLigand(pdb_id): Fetch, filter, reduce, and reformat ligand quality metrics
+            analyze(): Run PCA on the filtered and reduced ligand quality scores to generate reference data.
+            generate(pdb_ids): Full pipeline to generate ligand quality reference data by running the steps of
+                fetch -> analyze.
+            writeReference(output_file): Write the generated ligand quality reference data to a csv file.
+    """
+    def __init__(self, cfgOb, cachePath, **kwargs):
+        """Initiate the class and the MongoDB connection"""
+        self.__cfgOb = cfgOb
+        self.__configName = kwargs.get("configName", "site_info_configuration")
+        self.__cachePath = cachePath
+        self.__dirName = "rcsb-ligand-score"
+        self.__dirPath = os.path.join(self.__cachePath, self.__dirName)
+        #
+        self.__resourceName = "MONGO_DB"
+        self.__databaseName = kwargs.get("databaseName", "pdbx_core")
+        self.__collectionName = kwargs.get("collectionName", "pdbx_core_nonpolymer_entity_instance")
+        #
+        # initilize output data of each pdb_ligand combination and their quality scores, which is a list of
+        # dictionaries with keys of pdb_ligand, mogul_bonds_RMSZ, mogul_angles_RMSZ, RSR, RSCC, fit_pc1, geo_pc1
+        self.refDataL = []
+        #
+        # initialize query data record from MongoDB, which is a list of dictionaries with keys of pdb_ligand, mogul_bonds_RMSZ,
+        # mogul_angles_RMSZ, RSR, RSCC, and count (number of instances for the same ligand in the same PDB entry)
+        self.qDataL = None
+
+    def generate(self, pdb_ids: list[str] = None, backup: bool = False) -> bool:
+        """
+        Full pipeline to generate ligand quality reference data by running the steps of
+        query -> filter -> reduce -> analyze.
+
+        Args:
+            pdb_ids (list): List of specified PDB IDs, default to [] which leads to all PDB structures being queried.
+            backup (bool): Whether to backup generated file to stash (default False).
+
+        Returns:
+            (bool): True for success or False otherwise
+            Also updates/modifies the self.qDataL and self.refDataL attributes.
+        """
+        #
+        # fetch ligand quality data from MongoDB, and process the data
+        try:
+            self.qDataL = self.fetchLigand(pdb_ids=pdb_ids)
+        except DatabaseError as e:
+            logger.error("Failed to fetch ligand quality data, STOP. ERROR: %s", e)
+            return False
+        # verify data is not empty
+        if not self.qDataL:
+            logger.error("no data fetched from MongoDB, cannot analyze, STOP")
+            return False
+        # analyze the data to generate reference scores
+        try:
+            self.refDataL = self.analyzeLigand(self.qDataL)
+        except AnalysisError as e:
+            logger.error("Failed to analyze ligand quality data, STOP. ERROR: %s", e)
+            return False
+        # output the reference data to file
+        try:
+            rlsP = RcsbLigandScoreProvider(cachePath=self.__cachePath, useCache=True)
+            output_file = rlsP.getLigandScoreDataPath()
+            self.writeReference(output_file=output_file)
+        except OutputError as e:
+            logger.error("Failed to write ligand quality reference data to file, STOP. ERROR: %s", e)
+            return False
+        #
+        logger.info("Finished generation workflow for ligand quality reference data.")
+        ok = True
+        #
+        logger.info("Refreshing RcsbLigandScoreProvider instance...")
+        rlsP.reload(useCache=True)
+        if backup and rlsP.testCache():
+            logger.info("Backing up data to stash...")
+            okB = rlsP.backup(self.__cfgOb, self.__configName, useStash=True, useGit=False)
+            logger.info("%r ligand score backup status (%r)", self.__dirName, okB)
+            ok = ok and okB
+        #
+        return ok
+
+    def fetchLigand(self, pdb_ids: list[str] = None):
+        """
+        Fetch, filter, reduce, and reformat ligand quality metrics for given PDB IDs, defaulting to all structures.
+        Fetch ligand quality metrics from MongoDB;
+        Filter the queried ligand quality scores based on defined criteria;
+        Reduce the data by combining multiple instances of the same ligand in the same PDB entry;
+        Reformat through projection.
+
+        :param pdb_ids: List of specified PDB IDs, default to [] which leads to all PDB structures being queried.
+        """
+        qdata = None
+
+        if pdb_ids:
+            # Match the PDB entries by provided IDs
+            pipeline = [{
+                "$match": {
+                    "rcsb_nonpolymer_entity_instance_container_identifiers.entry_id": {
+                        "$in": pdb_ids
+                    }
+                }
+            }]
+        else:
+            # With no matching PDB IDs provided, All PDB entries are chosen
+            pipeline = []
+
+        pipeline.extend([
+            # Project combined ID and filter scores
+            {
+                "$project": {
+                    "_id": 0,
+                    "id": {
+                        "$concat": [
+                            "$rcsb_nonpolymer_entity_instance_container_identifiers.entry_id",
+                            "-",
+                            "$rcsb_nonpolymer_entity_instance_container_identifiers.comp_id"
+                        ]
+                    },
+                    "scores": {
+                        "$filter": {
+                            "input": "$rcsb_nonpolymer_instance_validation_score",
+                            "as": "s",
+                            "cond": {
+                                "$and": [
+                                    {"$gte": ["$$s.average_occupancy", 0.9]},
+                                    {"$lte": ["$$s.average_occupancy", 1]},
+                                    {"$gte": ["$$s.completeness", 0.9]},
+                                    {"$lte": ["$$s.completeness", 1]},
+                                    {"$gt": ["$$s.mogul_bonds_RMSZ", 0]},
+                                    {"$lte": ["$$s.mogul_bonds_RMSZ", 10]},
+                                    {"$gt": ["$$s.mogul_angles_RMSZ", 0]},
+                                    {"$lte": ["$$s.mogul_angles_RMSZ", 10]},
+                                    {"$gt": ["$$s.RSR", 0]},
+                                    {"$lte": ["$$s.RSR", 1]},
+                                    {"$gt": ["$$s.RSCC", 0]},
+                                    {"$lte": ["$$s.RSCC", 1]}
+                                ]
+                            }
+                        }
+                    }
+                }
+            },
+
+            # Remove documents with no valid scores
+            {"$match": {"scores.0": {"$exists": True}}},
+
+            # Unwind and group by ID to calculate averages
+            {"$unwind": "$scores"},
+            {
+                "$group": {
+                    "_id": "$id",
+                    "mogul_bonds_RMSZ": {"$avg": "$scores.mogul_bonds_RMSZ"},
+                    "mogul_angles_RMSZ": {"$avg": "$scores.mogul_angles_RMSZ"},
+                    "RSR": {"$avg": "$scores.RSR"},
+                    "RSCC": {"$avg": "$scores.RSCC"},
+                    "count": {"$sum": 1}
+                }
+            },
+
+            # Final projection
+            {
+                "$project": {
+                    "_id": 0,
+                    "pdb_ligand": "$_id",
+                    "mogul_bonds_RMSZ": 1,
+                    "mogul_angles_RMSZ": 1,
+                    "RSR": 1,
+                    "RSCC": 1,
+                    "count": 1
+                }
+            },
+
+            # Optional limit
+            # {"$limit": 10}
+        ])
+
+        # Run aggregation
+        with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
+            try:
+                db = client[self.__databaseName]
+                collection = db[self.__collectionName]
+                startTime = time.time()
+                qdata = list(collection.aggregate(pipeline))
+                logger.info(
+                    "Fetched %s unique pdb_ligand combination records from MongoDB: %s, collectionName: %s at %s (%.4f seconds)",
+                    len(qdata),
+                    self.__databaseName,
+                    self.__collectionName,
+                    time.strftime("%Y %m %d %H:%M:%S", time.localtime()),
+                    time.time() - startTime,
+                )
+            except Exception as e:
+                qdata = None
+                raise DatabaseError(f"Failed to run aggregation to fetch MongoDB ligand quality data. ERROR {e}") from e
+
+        return qdata
+
+    def analyzeLigand(self, qDataL: list):
+        """
+        Run PCA on the filtered and reduced ligand quality scores to generate reference data.
+        The first principal component (PC1) is used as the ligand quality reference score.
+
+        """
+        refDataL = []
+        # Prepare data for PCA, convert each score type into a separate list for matrix construction
+        l_pdb_ligand = []
+        l_mogul_bonds_RMSZ = []
+        l_mogul_angles_RMSZ = []
+        l_RSR = []
+        l_RSCC = []
+        for d_lig in qDataL:
+            l_pdb_ligand.append(d_lig["pdb_ligand"])
+            l_mogul_bonds_RMSZ.append(d_lig["mogul_bonds_RMSZ"])
+            l_mogul_angles_RMSZ.append(d_lig["mogul_angles_RMSZ"])
+            l_RSR.append(d_lig["RSR"])
+            l_RSCC.append(d_lig["RSCC"])
+        # Construct data matrices for geometry and fit scores
+        matrix_geo = np.column_stack([l_mogul_bonds_RMSZ, l_mogul_angles_RMSZ])
+        matrix_fit = np.column_stack([l_RSR, l_RSCC])
+        # Run PCA separately on geometry and fit scores
+        try:
+            logging.info("To Run PCA on geometry scores")
+            l_geo_pc1 = self.runPca(matrix_geo)
+            logging.info("To Run PCA on fit scores")
+            l_fit_pc1 = self.runPca(matrix_fit)
+        except Exception as e:
+            raise AnalysisError(f"Failed to run PCA on ligand quality metrics. ERROR {e}") from e
+        # Combine the two PC1 scores into a final reference score
+        for i, pdb_ligand in enumerate(l_pdb_ligand):
+            ref_score = {
+                "pdb_ligand": pdb_ligand,
+                "rsr": round(l_RSR[i], 4),
+                "rscc": round(l_RSCC[i], 4),
+                "mogul_bonds_rmsz": round(l_mogul_bonds_RMSZ[i], 4),
+                "mogul_angles_rmsz": round(l_mogul_angles_RMSZ[i], 4),
+                "fit_pc1": round(l_fit_pc1[i], 5),
+                "geo_pc1": round(l_geo_pc1[i], 5)
+            }
+            refDataL.append(ref_score)
+        logger.info("%s unique pdb-ligand pairs analzed by PCA", len(refDataL))
+
+        return refDataL
+
+    def runPca(self, xarr: np.ndarray) -> list:
+        """
+        Run PCA on numpy matrix and output the first principal components.
+        Loadings and explained variance can be examined in log but not in outcome.
+        Ensure lower scores correspond to better quality by adjusting the sign of PC1 if needed.
+
+        :param xarr: numpy matrix of any dimension
+        :return: first principal components as a list
+        """
+        # Scale the data
+        scaler = StandardScaler()
+        xstd = scaler.fit_transform(xarr)
+        # Perform PCA
+        pca = PCA(n_components=1)
+        principal_components = pca.fit_transform(xstd)
+        logger.info("Explained variance ratios: %s", pca.explained_variance_ratio_)  # should be > 0.7(70%) for ligand geo and fit PC1
+        loadings = pca.components_[0]
+        logger.info("Resulting PCA loadings: %s", loadings)  # should be [sqrt(0.5), sqrt(0.5)] for scaled PCA or with minus sign on one component
+        # Code below ensures loading on the first variable (RSR for fitting, and mogul_bonds_RMSZ for geometry) to be positive,
+        # This is because PCA direction is arbitrary, i.e. 0.707,-0.707 and -0.707,0.707 are equivalent, but we want to ensure
+        # lower scores correspond to better quality, which gives a correct direction for subsequent pecentile calculation.
+        if loadings[0] < 0:
+            loadings = -loadings
+            logger.info("Adjusted PCA loadings: %s", loadings)
+            principal_components = -principal_components
+            logger.info("Change sign of the first principal components to ensure correct directionality.")
+        return principal_components[:, 0].tolist()
+
+    def writeReference(self, output_file: str):
+        """Write the generated ligand quality reference data to a csv file.
+
+        Args:
+            output_file (str): Path to the output csv file.
+        """
+        fU = FileUtil()
+        fU.mkdir(self.__dirPath)
+        #
+        if not self.refDataL:
+            raise OutputError("No data to write. Please run generate() first.")
+        if type(self.refDataL) is not list:
+            raise OutputError("Data format incorrect. Expected a list of dictionaries after generate().")
+        if type(self.refDataL[0]) is not dict:
+            raise OutputError("Data format incorrect. Expected a list of dictionaries after generate().")
+        #
+        fieldnames = self.refDataL[0].keys()
+        #
+        try:
+            with open(output_file, mode="w", newline="", encoding="utf-8") as file:
+                writer = csv.DictWriter(file, fieldnames=fieldnames)
+                writer.writeheader()  # write column headers
+                writer.writerows(self.refDataL)  # write all rows
+        except Exception as e:
+            raise OutputError(f"Failed to write ligand quality reference data to file: {e}") from e
+        #
+        logger.info("Finished generating ligand quality reference data file at %s", output_file)