rcsb.workflow 0.64__py3-none-any.whl

rcsb/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

rcsb/workflow/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

rcsb/workflow/bcif/task_functions.py

@@ -0,0 +1,333 @@
+##
+# File:    task_functions.py
+# Author:  James Smith
+# Date:    21-Feb-2025
+##
+
+"""
+Workflow task descriptors.
+"""
+
+__docformat__ = "google en"
+__author__ = "James Smith"
+__email__ = "james.smith@rcsb.org"
+__license__ = "Apache 2.0"
+
+import multiprocessing
+import os
+import shutil
+import tempfile
+import logging
+from enum import Enum
+from typing import Optional
+from mmcif.api.DictionaryApi import DictionaryApi
+from mmcif.io.IoAdapterPy import IoAdapterPy as IoAdapter
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ContentTypeEnum = Enum(
+    "ContentTypeEnum",
+    [("EXPERIMENTAL", "pdb"), ("COMPUTATIONAL", "csm"), ("INTEGRATIVE", "ihm")],
+)
+
+
+def convertCifFilesToBcif(
+    listFileName: str,
+    listFileBase: str,
+    remotePath: str,
+    updateBase: str,
+    outfileSuffix: str,
+    contentType: str,
+    outputContentType: bool,
+    outputHash: bool,
+    inputHash: bool,
+    batchSize: int,
+    maxFiles: int,
+    pdbxDict: str,
+    maDict: str,
+    rcsbDict: str,
+    ihmDict: str,
+    flrDict: str,
+):
+    """
+    Converts CIF files to BCIF format based on a given list file.
+
+    Raises:
+        FileNotFoundError: If the specified list file does not exist.
+        ValueError: If the list file is empty.
+        RuntimeError: If processing fails for some files.
+    """
+
+    # read sublist
+    listfilepath = os.path.join(listFileBase, listFileName)
+    logger.info(
+        "reading list file %s and remote path %s",
+        listfilepath,
+        remotePath,
+    )
+    files = []
+    if not os.path.exists(listfilepath):
+        logger.warning("No list file found at %s", listfilepath)
+        return
+    f = open(listfilepath, "r", encoding="utf-8")
+    for line in f:
+        files.append(line.strip())
+        if 0 < maxFiles <= len(files):
+            break
+    f.close()
+    if len(files) < 1:
+        raise ValueError("No files found in %s" % listfilepath)
+
+    # determine batch size
+    if (batchSize is None) or not str(batchSize).isdigit():
+        batchSize = 1
+    batchSize = int(batchSize)
+    if batchSize <= 0:
+        batchSize = 1
+    logger.info("distributing %d files across %d sublists", len(files), batchSize)
+
+    # form dictionary object
+    dictionaryApi = getDictionaryApi(pdbxDict, maDict, rcsbDict, ihmDict, flrDict)
+
+    # traverse sublist and send each input file to converter
+    temppath = tempfile.mkdtemp()
+    if batchSize == 1:
+        # process one file at a time
+        for line in files:
+            args = (
+                line,
+                remotePath,
+                updateBase,
+                outfileSuffix,
+                outputContentType,
+                outputHash,
+                inputHash,
+                contentType,
+                dictionaryApi,
+                temppath,
+            )
+            singleTask(*args)
+    elif batchSize > 1:
+        pool = multiprocessing.Pool(processes=batchSize)
+        results = []
+        for line in files:
+            entry = line.strip()
+            args = (
+                entry,
+                remotePath,
+                updateBase,
+                outfileSuffix,
+                outputContentType,
+                outputHash,
+                inputHash,
+                contentType,
+                dictionaryApi,
+                temppath,
+            )
+            results.append(pool.apply_async(singleTask, args))
+        for r in results:
+            r.get(timeout=60 * 5)
+        pool.close()
+        results.clear()
+
+    try:
+        if os.path.exists(temppath):
+            shutil.rmtree(temppath)
+    except Exception as e:
+        logger.error(str(e))
+
+    logger.info("Done with CIF to BCIF conversion")
+
+
+def getDictionaryApi(
+    pdbxDict: str, maDict: str, rcsbDict: str, ihmDict: str, flrDict: str
+) -> DictionaryApi:
+    paths = [pdbxDict, maDict, rcsbDict, ihmDict, flrDict]
+    try:
+        adapter = IoAdapter(raiseExceptions=True)
+        containers = []
+        for path in paths:
+            containers += adapter.readFile(inputFilePath=path)
+        dictionaryApi = DictionaryApi(containerList=containers, consolidate=True)
+    except Exception as e:
+        raise FileNotFoundError("failed to create dictionary api: %s" % str(e)) from e
+    return dictionaryApi
+
+
+def singleTask(
+    pdbId: str,
+    remotePath: str,
+    updateBase: str,
+    outfileSuffix: str,
+    outputContentType: bool,
+    outputHash: bool,
+    inputHash: bool,
+    contentType: str,
+    dictionaryApi: DictionaryApi,
+    temppath: str,
+) -> None:
+    remoteFileName = "%s%s" % (
+        getInputPdbId(pdbId, contentType),
+        outfileSuffix.replace(".bcif.gz", ".cif.gz").replace(".bcif", ".cif"),
+    )
+
+    # form input cifFilePath
+    if not remotePath.startswith("http"):
+        # local file
+        cifFilePath = os.path.join(remotePath, remoteFileName)
+        if inputHash:
+            cifFilePath = os.path.join(
+                remotePath,
+                getHash(getInputPdbId(pdbId, contentType), contentType),
+                remoteFileName,
+            )
+        if not os.path.exists(cifFilePath):
+            logger.warning("%s not found", cifFilePath)
+            return
+    else:
+        cifFilePath = getRemoteFilePath(
+            getInputPdbId(pdbId, contentType), contentType, remotePath, remoteFileName
+        )
+
+    # form output bcifFilePath
+    bcifFilePath = getBcifFilePath(
+        pdbId, outfileSuffix, updateBase, contentType, outputContentType, outputHash
+    )
+    if not bcifFilePath:
+        raise ValueError("could not form bcif file path for %s" % pdbId)
+
+    if os.path.exists(bcifFilePath):
+        # earlier timestamp ... overwrite
+        os.unlink(bcifFilePath)
+        if os.path.exists(bcifFilePath):
+            raise PermissionError("file %s not removed" % bcifFilePath)
+
+    # make nested directories
+    dirs = os.path.dirname(bcifFilePath)
+    if not os.path.exists(dirs):
+        os.makedirs(dirs)
+
+    # convert to bcif
+    result = convert(cifFilePath, bcifFilePath, temppath, dictionaryApi)
+    if not result:
+        raise Exception("failed to convert %s" % cifFilePath)
+
+
+def getInputPdbId(pdbId: str, contentType: str) -> str:
+    """determine upper or lowercase id names for remote and local .cif input files"""
+    if contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        return pdbId.upper()
+    return pdbId.lower()
+
+
+def getOutputPdbId(pdbId: str) -> str:
+    """determine upper or lowercase id names for local .bcif output files"""
+    return pdbId.lower()
+
+
+def getOutputContentType(contentType: str) -> str:
+    """prepended output folder name for local .bcif output files"""
+    if contentType in [ContentTypeEnum.EXPERIMENTAL.value, ContentTypeEnum.INTEGRATIVE.value]:
+        return "pdb"
+    return "csm"
+
+
+def getHash(pdbId: str, contentType: str) -> str:
+    """get upper/lowercase pdb id prior to getting hash"""
+    result = pdbId[1:3]
+    if contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        result = os.path.join(pdbId[0:2], pdbId[-6:-4], pdbId[-4:-2])
+    return result
+
+
+def getRemoteFilePath(
+    pdbId: str, contentType: str, remotePath: str, remoteFileName: str
+) -> str:
+    """form hashed pash for remote cif file, only for input files"""
+    pdbId = getInputPdbId(pdbId, contentType)
+    result = os.path.join(remotePath, getHash(pdbId, contentType), remoteFileName)
+    if contentType == ContentTypeEnum.INTEGRATIVE.value:
+        result = os.path.join(
+            remotePath, getHash(pdbId, contentType), pdbId, "structures", remoteFileName
+        )
+    return result
+
+
+def getBcifFilePath(
+    pdbId: str,
+    outfileSuffix: str,
+    updateBase: str,
+    contentType: str,
+    outputContentType: bool,
+    outputHash: bool,
+) -> Optional[str]:
+    """form optionally hashed path for local bcif file, only for output files"""
+    pdbId = getOutputPdbId(pdbId)
+    bcifFileName = "%s%s" % (pdbId, outfileSuffix)
+    bcifFilePath = None
+    if contentType in [
+        ContentTypeEnum.EXPERIMENTAL.value,
+        ContentTypeEnum.INTEGRATIVE.value,
+    ]:
+        if outputContentType and outputHash:
+            bcifFilePath = os.path.join(
+                updateBase, getOutputContentType(contentType), getHash(pdbId, contentType), bcifFileName
+            )
+        elif outputContentType:
+            bcifFilePath = os.path.join(updateBase, getOutputContentType(contentType), bcifFileName)
+        elif outputHash:
+            bcifFilePath = os.path.join(
+                updateBase, getHash(pdbId, contentType), bcifFileName
+            )
+        else:
+            bcifFilePath = os.path.join(updateBase, bcifFileName)
+    elif contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        if outputContentType and outputHash:
+            bcifFilePath = os.path.join(
+                updateBase,
+                getOutputContentType(contentType),
+                getHash(pdbId, contentType),
+                bcifFileName,
+            )
+        elif outputContentType:
+            bcifFilePath = os.path.join(
+                updateBase,
+                getOutputContentType(contentType),
+                bcifFileName,
+            )
+        elif outputHash:
+            bcifFilePath = os.path.join(
+                updateBase,
+                getHash(pdbId, contentType),
+                bcifFileName,
+            )
+        else:
+            bcifFilePath = os.path.join(
+                updateBase,
+                bcifFileName,
+            )
+    return bcifFilePath
+
+
+def convert(
+    infile: str, outfile: str, workpath: str, dictionaryApi: DictionaryApi
+) -> bool:
+    mu = MarshalUtil(workPath=workpath)
+    data = mu.doImport(infile, fmt="mmcif")
+    result = mu.doExport(outfile, data, fmt="bcif", dictionaryApi=dictionaryApi)
+    if not result:
+        return False
+    return True
+
+
+def deconvert(
+    infile: str, outfile: str, workpath: str, dictionaryApi: DictionaryApi
+) -> bool:
+    mu = MarshalUtil(workPath=workpath)
+    data = mu.doImport(infile, fmt="bcif")
+    result = mu.doExport(outfile, data, fmt="mmcif", dictionaryApi=dictionaryApi)
+    if not result:
+        return False
+    return True

rcsb/workflow/chem/ChemCompFileWorkflow.py

@@ -0,0 +1,96 @@
+##
+# File: ChemCompFileWorkflow.py
+# Date: 10-Mar-2020  jdw
+#
+#  Workflow wrapper  --  chemical component file conversion generator --
+#
+#  Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+import os
+
+from rcsb.utils.chem.OeIoUtils import OeIoUtils
+from rcsb.utils.chem.OeMoleculeProvider import OeMoleculeProvider
+
+logger = logging.getLogger(__name__)
+
+
+class ChemCompFileWorkflow(object):
+    def __init__(self, **kwargs):
+        """Module entry point for chemical component definition 2D image generation
+
+        Args:
+            ccUrlTarget (str): URL or path for concatenated chemical component dictionary (default: public wwPDB ftp)
+            birdUrlTarget (str): URL or path for concatenated BIRD dictionary (default: public wwPDB ftp)
+            licenseFilePath (str) = path to OpenEye license text file
+            fileDirPath(str): directory containing generated image tree
+            cachePath(str): cache directory for temporary files
+            molBuildType(str): build type for constructing OE moleclues ('ideal-xyz', 'model-xyz' default: 'ideal-xyz')
+        """
+        #
+        ccUrlTarget = kwargs.get("ccUrlTarget", None)
+        birdUrlTarget = kwargs.get("birdUrlTarget", None)
+        self.__licensePath = kwargs.get("licenseFilePath", "oe_license.txt")
+        self.__fileDirPath = kwargs.get("fileDirPath", ".")
+        self.__molBuildType = kwargs.get("molBuildType", "ideal-xyz")
+        cachePath = kwargs.get("cachePath", ".")
+        cachePath = os.path.abspath(cachePath)
+
+        #
+        self.__oemp = OeMoleculeProvider(
+            ccUrlTarget=ccUrlTarget,
+            birdUrlTarget=birdUrlTarget,
+            ccFileNamePrefix="cc-full",
+            cachePath=cachePath,
+            molBuildType=self.__molBuildType,
+            useCache=True,
+            oeFileNamePrefix="oe-full",
+        )
+        self.__oeMolD = self.__oemp.getOeMolD()
+
+    def __setLicense(self, licensePath):
+        ok = False
+        try:
+            if os.environ.get("OE_LICENSE") and os.access(os.environ["OE_LICENSE"], os.R_OK):
+                logger.info("Using license from environment %r", os.environ["OE_LICENSE"])
+                ok = True
+            elif os.access(licensePath, os.R_OK):
+                os.environ["OE_LICENSE"] = licensePath
+                logger.info("Setting environmenal variable OE_LICENSE to %r", os.environ["OE_LICENSE"])
+                ok = True
+        except Exception as e:
+            logger.error("Setting license file %r failing %s", licensePath, str(e))
+        return ok
+
+    def testCache(self):
+        return self.__oemp.testCache() if self.__oemp else False
+
+    def makeFiles(self, fmt="sdf"):
+        """Create files (mol, mol2) for all public chemical components."""
+        try:
+
+            if fmt not in ["mol", "mol2", "mol2h", "sdf"]:
+                return False
+            if not self.__setLicense(self.__licensePath):
+                logger.error("Invalid license details - exiting")
+                return False
+            for ccId, oeMol in self.__oeMolD.items():
+                if self.__molBuildType == "ideal-xyz":
+                    filePath = os.path.join(self.__fileDirPath, fmt, ccId[0], ccId + "_ideal." + fmt)
+                    oeioU = OeIoUtils()
+                    oeioU.write(filePath, oeMol, constantMol=True)
+                else:
+                    filePath = os.path.join(self.__fileDirPath, fmt, ccId[0], ccId + "_model." + fmt)
+                    oeioU = OeIoUtils()
+                    oeioU.write(filePath, oeMol, constantMol=True)
+
+            return True
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return False

rcsb/workflow/chem/ChemCompImageWorkflow.py

@@ -0,0 +1,133 @@
+##
+# File: ChemCompImageWorkflow.py
+# Date: 10-Mar-2020  jdw
+#
+#  Workflow wrapper  --  chemical component image generation --
+#
+#  Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+import os
+
+from rcsb.utils.chem.OeDepict import OeDepict
+from rcsb.utils.chem.OeMoleculeProvider import OeMoleculeProvider
+
+logger = logging.getLogger(__name__)
+
+
+class ChemCompImageWorkflow(object):
+    def __init__(self, **kwargs):
+        """Module entry point for chemical component definition 2D image generation
+
+        Args:
+            ccUrlTarget (str): URL or path for concatenated chemical component dictionary (default: public wwPDB ftp)
+            birdUrlTarget (str): URL or path for concatenated BIRD dictionary (default: public wwPDB ftp)
+            licenseFilePath (str) = path to OpenEye license text file
+            imagePath(str): directory containing generated image tree
+            cachePath(str): cache directory for temporary files
+
+        """
+        #
+        ccUrlTarget = kwargs.get("ccUrlTarget", None)
+        birdUrlTarget = kwargs.get("birdUrlTarget", None)
+        self.__licensePath = kwargs.get("licenseFilePath", "oe_license.txt")
+        self.__imagePath = kwargs.get("imagePath", ".")
+        cachePath = kwargs.get("cachePath", ".")
+        cachePath = os.path.abspath(cachePath)
+
+        #
+        # changed molBuildType from "model-xyz" to "connection-table" per ticket RO-3041
+        self.__oemp = OeMoleculeProvider(
+            ccUrlTarget=ccUrlTarget,
+            birdUrlTarget=birdUrlTarget,
+            ccFileNamePrefix="cc-full",
+            cachePath=cachePath,
+            molBuildType="connection-table",
+            useCache=True,
+            oeFileNamePrefix="oe-full",
+        )
+        self.__oeMolD = self.__oemp.getOeMolD()
+
+    def __setLicense(self, licensePath):
+        ok = False
+        try:
+            if os.environ.get("OE_LICENSE") and os.access(os.environ["OE_LICENSE"], os.R_OK):
+                logger.info("Using license from environment %r", os.environ["OE_LICENSE"])
+                ok = True
+            elif os.access(licensePath, os.R_OK):
+                os.environ["OE_LICENSE"] = licensePath
+                logger.info("Setting environmenal variable OE_LICENSE to %r", os.environ["OE_LICENSE"])
+                ok = True
+        except Exception as e:
+            logger.error("Setting license file %r failing %s", licensePath, str(e))
+        return ok
+
+    def testCache(self):
+        return self.__oemp.testCache() if self.__oemp else False
+
+    def makeImages(self):
+        """Create images for all public chemical components with and without atom labels."""
+        try:
+            if not self.__setLicense(self.__licensePath):
+                logger.error("Invalid license details - exiting")
+                return False
+            for ccId, oeMol in self.__oeMolD.items():
+                imagePath = os.path.join(self.__imagePath, "image", ccId[0], ccId + ".svg")
+                oed = OeDepict()
+                title = ""
+                oed.setMolTitleList([(ccId, oeMol, title)])
+                # ---
+                bondDisplayWidth = 10.0
+                numAtoms = oeMol.NumAtoms()
+                if numAtoms > 100 and numAtoms <= 200:
+                    bondDisplayWidth = 6.0
+                elif numAtoms > 200:
+                    bondDisplayWidth = 4.0
+                # ---
+                oed.setDisplayOptions(
+                    labelAtomName=False,
+                    labelAtomCIPStereo=True,
+                    labelAtomIndex=False,
+                    labelBondIndex=False,
+                    labelBondCIPStereo=True,
+                    cellBorders=False,
+                    bondDisplayWidth=bondDisplayWidth,
+                )
+                oed.setGridOptions(rows=1, cols=1, cellBorders=False)
+                oed.prepare()
+                oed.write(imagePath)
+            for ccId, oeMol in self.__oeMolD.items():
+                imagePath = os.path.join(self.__imagePath, "image_labeled", ccId[0], ccId + ".svg")
+                oed = OeDepict()
+                title = ""
+                oed.setMolTitleList([(ccId, oeMol, title)])
+                # ---
+                bondDisplayWidth = 10.0
+                numAtoms = oeMol.NumAtoms()
+                if numAtoms > 100 and numAtoms <= 200:
+                    bondDisplayWidth = 6.0
+                elif numAtoms > 200:
+                    bondDisplayWidth = 4.0
+                # ---
+                oed.setDisplayOptions(
+                    labelAtomName=True,
+                    labelAtomCIPStereo=True,
+                    labelAtomIndex=False,
+                    labelBondIndex=False,
+                    labelBondCIPStereo=True,
+                    cellBorders=False,
+                    bondDisplayWidth=bondDisplayWidth,
+                )
+                oed.setGridOptions(rows=1, cols=1, cellBorders=False)
+                oed.prepare()
+                oed.write(imagePath)
+            return True
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return False

rcsb/workflow/chem/ChemCompSearchIndexWorkflow.py

@@ -0,0 +1,99 @@
+##
+# File: ChemCompIndexWorkflow.py
+# Date: 2-Jun-2020  jdw
+#
+#  Workflow wrapper  --  generate chemical component and BIRD search indices  --
+#
+#  Updates:
+#  10-Jun-2020 jdw Hookup to ChemCompSearchWrapper()
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import logging
+import os
+
+from rcsb.utils.chem.ChemCompSearchWrapper import ChemCompSearchWrapper
+
+logger = logging.getLogger(__name__)
+
+
+class ChemCompSearchIndexWorkflow(object):
+    def __init__(self, **kwargs):
+        """Module entry point for chemical component and BIRD search index generation workflow.
+
+        Args:
+            cachePath(str, optional): cache directory to store indices and temporary files (default: '.')
+            licenseFilePath (str, optional) = path to OpenEye license text file (default: use enviroment OE_LICENSE setting)
+            ccFilNamePrefix (str, optional) =  index prefix (default: "cc-full")
+        """
+        self.__licensePath = kwargs.get("licenseFilePath", "oe_license.txt")
+        cachePath = kwargs.get("cachePath", ".")
+        self.__cachePath = os.path.abspath(cachePath)
+        self.__ccFileNamePrefix = kwargs.get("ccFilNamePrefix", "cc-full")
+        self.__ccsw = ChemCompSearchWrapper(cachePath=self.__cachePath, ccFileNamePrefix=self.__ccFileNamePrefix)
+
+    def __setLicense(self, licensePath):
+        ok = False
+        try:
+            if os.environ.get("OE_LICENSE") and os.access(os.environ["OE_LICENSE"], os.R_OK):
+                logger.info("Using license from environment %r", os.environ["OE_LICENSE"])
+                ok = True
+            elif os.access(licensePath, os.R_OK):
+                os.environ["OE_LICENSE"] = licensePath
+                logger.info("Setting environmenal variable OE_LICENSE to %r", os.environ["OE_LICENSE"])
+                ok = True
+        except Exception as e:
+            logger.error("Setting Openeye license file %r failing %s", licensePath, str(e))
+        return ok
+
+    def testCache(self):
+        return True
+
+    def makeIndices(self, ccUrlTarget, birdUrlTarget, **kwargs):
+        """Build chemical component and BIRD search indices.
+
+        Args:
+            ccUrlTarget (str): URL or path for concatenated chemical component dictionary
+            birdUrlTarget (str): URL or path for concatenated BIRD dictionary
+
+            Other arguments may be supplied to change defaults for index generators (testing/troubleshooting)
+
+        Returns:
+            bool:  True for success or False otherwise
+
+        """
+        ok = False
+        if not self.__setLicense(self.__licensePath):
+            logger.error("Invalid OpenEye license details - exiting")
+            return ok
+        #
+        try:
+            ok = self.__ccsw.buildDependenices(ccUrlTarget, birdUrlTarget, **kwargs)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok
+
+    def stashIndices(self, url, dirPath, bundleLabel="A", userName=None, pw=None):
+        """Store a copy of the bundled search dependencies remotely -
+
+        Args:
+            url (str): URL string for the destination host (e.g. sftp://myserver.net or None for a local file)
+            dirPath (str): directory path on the remote resource
+            bundleLabel (str, optional): optional label preppended to the stashed dependency bundle artifact (default='A')
+            userName (str, optional): optional access information. Defaults to None.
+            password (str, optional): optional access information. Defaults to None.
+
+        Returns:
+          bool:  True for success or False otherwise
+
+        """
+        #
+        ok = False
+        try:
+            ok = self.__ccsw.stashDependencies(url, dirPath, bundleLabel=bundleLabel, userName=userName, pw=pw)
+        except Exception as e:
+            logger.exception("Failing with %s", str(e))
+        return ok