rcsb.workflow 0.52__tar.gz → 0.53__tar.gz

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/HISTORY.txt

@@ -46,3 +46,4 @@
                     Remove IMGT from feature building after service became unavailable February 2025
 19-Feb-2025 - V0.51 Bring back IMGT
  4-Apr-2025 - V0.52 Add multi cpu execution to jpg generation
+10-Apr-2025 - V0.53 Add BCIF conversion CLI

{rcsb_workflow-0.52/rcsb.workflow.egg-info → rcsb_workflow-0.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rcsb.workflow
-Version: 0.52
+Version: 0.53
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Home-page: https://github.com/rcsb/py-rcsb_workflow
 Author: John Westbrook
@@ -22,7 +22,9 @@ Requires-Dist: rcsb.utils.targets>=0.82
 Requires-Dist: rcsb.utils.seqalign>=0.31
 Requires-Dist: rcsb.utils.io>=1.48
 Requires-Dist: rcsb.exdb>=1.28
-Requires-Dist: rcsb.db>=1.800
+Requires-Dist: rcsb.db>=1.801
+Requires-Dist: requests>=2.32.3
+Requires-Dist: mmcif>=0.91.0
 Provides-Extra: dev
 Requires-Dist: check-manifest; extra == "dev"
 Provides-Extra: test

rcsb_workflow-0.53/rcsb/workflow/bcif/task_functions.py

@@ -0,0 +1,315 @@
+##
+# File:    task_functions.py
+# Author:  James Smith
+# Date:    21-Feb-2025
+##
+
+"""
+Workflow task descriptors.
+"""
+
+__docformat__ = "google en"
+__author__ = "James Smith"
+__email__ = "james.smith@rcsb.org"
+__license__ = "Apache 2.0"
+
+import multiprocessing
+import os
+import shutil
+import tempfile
+import logging
+from enum import Enum
+from typing import Optional
+from mmcif.api.DictionaryApi import DictionaryApi
+from mmcif.io.IoAdapterPy import IoAdapterPy as IoAdapter
+from rcsb.utils.io.MarshalUtil import MarshalUtil
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+ContentTypeEnum = Enum(
+    "ContentTypeEnum",
+    [("EXPERIMENTAL", "pdb"), ("COMPUTATIONAL", "csm"), ("INTEGRATIVE", "ihm")],
+)
+
+
+def convertCifFilesToBcif(
+    listFileName: str,
+    listFileBase: str,
+    remotePath: str,
+    updateBase: str,
+    outfileSuffix: str,
+    contentType: str,
+    outputContentType: bool,
+    outputHash: bool,
+    inputHash: bool,
+    batchSize: int,
+    maxFiles: int,
+    pdbxDict: str,
+    maDict: str,
+    rcsbDict: str,
+    ihmDict: str,
+):
+    """
+    Converts CIF files to BCIF format based on a given list file.
+
+    Raises:
+        FileNotFoundError: If the specified list file does not exist.
+        ValueError: If the list file is empty.
+        RuntimeError: If processing fails for some files.
+    """
+
+    # read sublist
+    listfilepath = os.path.join(listFileBase, listFileName)
+    logger.info(
+        "reading list file %s and remote path %s",
+        listfilepath,
+        remotePath,
+    )
+    files = []
+    if not os.path.exists(listfilepath):
+        raise FileNotFoundError("No list file found at %s" % listfilepath)
+    f = open(listfilepath, "r", encoding="utf-8")
+    for line in f:
+        files.append(line.strip())
+        if 0 < maxFiles <= len(files):
+            break
+    f.close()
+    if len(files) < 1:
+        raise ValueError("No files found in %s" % listfilepath)
+
+    # determine batch size
+    if (batchSize is None) or not str(batchSize).isdigit():
+        batchSize = 1
+    batchSize = int(batchSize)
+    if batchSize <= 0:
+        batchSize = 1
+    logger.info("distributing %d files across %d sublists", len(files), batchSize)
+
+    # form dictionary object
+    dictionaryApi = getDictionaryApi(pdbxDict, maDict, rcsbDict, ihmDict)
+
+    # traverse sublist and send each input file to converter
+    temppath = tempfile.mkdtemp()
+    if batchSize == 1:
+        # process one file at a time
+        for line in files:
+            args = (
+                line,
+                remotePath,
+                updateBase,
+                outfileSuffix,
+                outputContentType,
+                outputHash,
+                inputHash,
+                contentType,
+                dictionaryApi,
+                temppath,
+            )
+            singleTask(*args)
+    elif batchSize > 1:
+        pool = multiprocessing.Pool(processes=batchSize)
+        results = []
+        for line in files:
+            entry = line.strip()
+            args = (
+                entry,
+                remotePath,
+                updateBase,
+                outfileSuffix,
+                outputContentType,
+                outputHash,
+                inputHash,
+                contentType,
+                dictionaryApi,
+                temppath,
+            )
+            results.append(pool.apply_async(singleTask, args))
+        for r in results:
+            r.get(timeout=60 * 5)
+        pool.close()
+        results.clear()
+
+    try:
+        if os.path.exists(temppath):
+            shutil.rmtree(temppath)
+    except Exception as e:
+        logger.error(str(e))
+
+    logger.info("Done with CIF to BCIF conversion")
+
+
+def getDictionaryApi(
+    pdbxDict: str, maDict: str, rcsbDict: str, ihmDict: str
+) -> DictionaryApi:
+    paths = [pdbxDict, maDict, rcsbDict, ihmDict]
+    try:
+        adapter = IoAdapter(raiseExceptions=True)
+        containers = []
+        for path in paths:
+            containers += adapter.readFile(inputFilePath=path)
+        dictionaryApi = DictionaryApi(containerList=containers, consolidate=True)
+    except Exception as e:
+        raise FileNotFoundError("failed to create dictionary api: %s" % str(e)) from e
+    return dictionaryApi
+
+
+def singleTask(
+    pdbId: str,
+    remotePath: str,
+    updateBase: str,
+    outfileSuffix: str,
+    outputContentType: bool,
+    outputHash: bool,
+    inputHash: bool,
+    contentType: str,
+    dictionaryApi: DictionaryApi,
+    temppath: str,
+) -> None:
+    if contentType in [
+        ContentTypeEnum.EXPERIMENTAL.value,
+        ContentTypeEnum.INTEGRATIVE.value,
+    ]:
+        pdbId = pdbId.lower()
+    elif contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        pdbId = pdbId.upper()
+    remoteFileName = "%s%s" % (
+        pdbId,
+        outfileSuffix.replace(".bcif.gz", ".cif.gz").replace(".bcif", ".cif"),
+    )
+
+    # form input cifFilePath
+    if not remotePath.startswith("http"):
+        # local file
+        cifFilePath = os.path.join(remotePath, remoteFileName)
+        if inputHash:
+            cifFilePath = os.path.join(
+                remotePath, getHash(pdbId, contentType), remoteFileName
+            )
+        if not os.path.exists(cifFilePath):
+            logger.warning("%s not found", cifFilePath)
+            return
+    else:
+        cifFilePath = getRemoteFilePath(pdbId, contentType, remotePath, remoteFileName)
+
+    # form output bcifFilePath
+    bcifFilePath = getBcifFilePath(
+        pdbId, outfileSuffix, updateBase, contentType, outputContentType, outputHash
+    )
+    if not bcifFilePath:
+        raise ValueError("could not form bcif file path for %s" % pdbId)
+
+    if os.path.exists(bcifFilePath):
+        # earlier timestamp ... overwrite
+        os.unlink(bcifFilePath)
+        if os.path.exists(bcifFilePath):
+            raise PermissionError("file %s not removed" % bcifFilePath)
+
+    # make nested directories
+    dirs = os.path.dirname(bcifFilePath)
+    if not os.path.exists(dirs):
+        os.makedirs(dirs)
+
+    # convert to bcif
+    result = convert(cifFilePath, bcifFilePath, temppath, dictionaryApi)
+    if not result:
+        raise Exception("failed to convert %s" % cifFilePath)
+
+
+def getHash(pdbId: str, contentType: str) -> str:
+    if contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        pdbId = pdbId.upper()
+    else:
+        pdbId = pdbId.lower()
+    result = pdbId[1:3]
+    if contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        result = os.path.join(pdbId[0:2], pdbId[-6:-4], pdbId[-4:-2])
+    return result
+
+
+def getRemoteFilePath(
+    pdbId: str, contentType: str, remotePath: str, remoteFileName: str
+) -> str:
+    result = os.path.join(remotePath, getHash(pdbId, contentType), remoteFileName)
+    if contentType == ContentTypeEnum.INTEGRATIVE.value:
+        result = os.path.join(
+            remotePath, getHash(pdbId, contentType), pdbId, "structures", remoteFileName
+        )
+    return result
+
+
+def getBcifFilePath(
+    pdbId: str,
+    outfileSuffix: str,
+    updateBase: str,
+    contentType: str,
+    outputContentType: bool,
+    outputHash: bool,
+) -> Optional[str]:
+    bcifFileName = "%s%s" % (pdbId, outfileSuffix)
+    bcifFilePath = None
+    if contentType in [
+        ContentTypeEnum.EXPERIMENTAL.value,
+        ContentTypeEnum.INTEGRATIVE.value,
+    ]:
+        if outputContentType and outputHash:
+            bcifFilePath = os.path.join(
+                updateBase, contentType, getHash(pdbId, contentType), bcifFileName
+            )
+        elif outputContentType:
+            bcifFilePath = os.path.join(updateBase, contentType, bcifFileName)
+        elif outputHash:
+            bcifFilePath = os.path.join(
+                updateBase, getHash(pdbId, contentType), bcifFileName
+            )
+        else:
+            bcifFilePath = os.path.join(updateBase, bcifFileName)
+    elif contentType == ContentTypeEnum.COMPUTATIONAL.value:
+        if outputContentType and outputHash:
+            bcifFilePath = os.path.join(
+                updateBase,
+                contentType,
+                getHash(pdbId, contentType),
+                bcifFileName,
+            )
+        elif outputContentType:
+            bcifFilePath = os.path.join(
+                updateBase,
+                contentType,
+                bcifFileName,
+            )
+        elif outputHash:
+            bcifFilePath = os.path.join(
+                updateBase,
+                getHash(pdbId, contentType),
+                bcifFileName,
+            )
+        else:
+            bcifFilePath = os.path.join(
+                updateBase,
+                bcifFileName,
+            )
+    return bcifFilePath
+
+
+def convert(
+    infile: str, outfile: str, workpath: str, dictionaryApi: DictionaryApi
+) -> bool:
+    mu = MarshalUtil(workPath=workpath)
+    data = mu.doImport(infile, fmt="mmcif")
+    result = mu.doExport(outfile, data, fmt="bcif", dictionaryApi=dictionaryApi)
+    if not result:
+        return False
+    return True
+
+
+def deconvert(
+    infile: str, outfile: str, workpath: str, dictionaryApi: DictionaryApi
+) -> bool:
+    mu = MarshalUtil(workPath=workpath)
+    data = mu.doImport(infile, fmt="bcif")
+    result = mu.doExport(outfile, data, fmt="mmcif", dictionaryApi=dictionaryApi)
+    if not result:
+        return False
+    return True

rcsb_workflow-0.53/rcsb/workflow/cli/BcifExec.py

@@ -0,0 +1,174 @@
+##
+# File:    BcifExec.py
+# Author:  James Smith
+# Date:    21-Feb-2025
+##
+
+"""
+Entry point for bcif command line interface.
+"""
+
+__docformat__ = "google en"
+__author__ = "James Smith"
+__email__ = "james.smith@rcsb.org"
+__license__ = "Apache 2.0"
+
+import argparse
+import os
+import logging
+import time
+from rcsb.workflow.wuw.BcifWorkflow import BcifWorkflow
+
+logger = logging.getLogger()
+
+
+def main():
+
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    # paths
+    parser.add_argument(
+        "--listFileBase",
+        default="/tmp",
+        required=True,
+        help="path for input lists",
+    )
+    parser.add_argument(
+        "--listFileName",
+        default="pdbx_core_ids-1.txt",
+        required=True,
+        help="name of list file to read",
+    )
+    parser.add_argument(
+        "--remotePath",
+        default="http://prereleaseftp-external-east.rcsb.org/pdb/data/structures/divided/mmCIF/",
+        required=True,
+        help="url or directory for cif files",
+    )
+    parser.add_argument(
+        "--outputPath",
+        default="/mnt/vdb1/out",
+        required=True,
+        help="output directory for bcif files",
+    )
+    # settings
+    parser.add_argument(
+        "--contentType",
+        default="pdb",
+        required=True,
+        choices=["pdb", "csm", "ihm"],
+        help="which type of experiment was performed",
+    )
+    parser.add_argument(
+        "--nfiles",
+        default=0,
+        required=False,
+        help="set 0 for all files, set less than N for a test run, will not produce exactly n files",
+    )
+    parser.add_argument(
+        "--outfileSuffix",
+        default=".bcif.gz",
+        required=False,
+        choices=[".bcif", ".bcif.gz"],
+        help="whether to use additional gzip compression",
+    )
+    parser.add_argument(
+        "--batchSize",
+        default=1,
+        required=False,
+        help="optional subdivisions of sublists for pdb list and csm list",
+    )
+    # output folder structure, default none (save all output files in one folder)
+    parser.add_argument(
+        "--outputContentType",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether output paths should include a directory for the content type (pdb, csm)",
+    )
+    parser.add_argument(
+        "--outputHash",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether output paths should include the hash for the entry",
+    )
+    # input folder structure, default none (read all input files from one folder)
+    parser.add_argument(
+        "--inputHash",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether local input paths should include the hash for the entry",
+    )
+    # config
+    parser.add_argument(
+        "--pdbxDict",
+        default="https://raw.githubusercontent.com/wwpdb-dictionaries/mmcif_pdbx/master/dist/mmcif_pdbx_v5_next.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--maDict",
+        default="https://raw.githubusercontent.com/ihmwg/ModelCIF/master/dist/mmcif_ma_ext.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--rcsbDict",
+        default="https://raw.githubusercontent.com/rcsb/py-rcsb_exdb_assets/master/dictionary_files/dist/rcsb_mmcif_ext.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--ihmDict",
+        default="https://raw.githubusercontent.com/rcsb/py-rcsb_exdb_assets/pdb-ihm-2/dictionary_files/reference/mmcif_ihm_ext.dic",
+        required=False,
+    )
+    # logging
+    parser.add_argument("--log_file_path", required=False)
+    parser.add_argument("--debug", action="store_true", default=False, required=False)
+
+    args = parser.parse_args()
+
+    logFilePath = args.log_file_path
+    debugFlag = args.debug
+    if debugFlag:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    if logFilePath:
+        logDir = os.path.dirname(logFilePath)
+        if not os.path.isdir(logDir):
+            os.makedirs(logDir)
+        handler = logging.FileHandler(logFilePath, mode="a")
+        if debugFlag:
+            handler.setLevel(logging.DEBUG)
+        else:
+            handler.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            fmt="%(asctime)s @%(process)s [%(levelname)s]-%(module)s: %(message)s"
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    else:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            fmt="%(asctime)s @%(process)s [%(levelname)s]-%(module)s: %(message)s"
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+    logger.info("bcif workflow initialized")
+
+    try:
+        (BcifWorkflow(args))()
+    except RuntimeError as e:
+        raise Exception(str(e)) from e
+
+
+if __name__ == "__main__":
+    t = time.time()
+    try:
+        main()
+    except Exception as e:
+        logger.exception(str(e))
+        raise e
+    t2 = time.time() - t
+    logger.info("completed in %.2f s", t2)

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/rcsb/workflow/cli/__init__.py

@@ -2,4 +2,4 @@ __docformat__ = "google en"
 __author__ = "John Westbrook"
 __email__ = "john.westbrook@rcsb.org"
 __license__ = "Apache 2.0"
-__version__ = "0.52"
+__version__ = "0.53"

rcsb_workflow-0.53/rcsb/workflow/wuw/BcifWorkflow.py

@@ -0,0 +1,85 @@
+##
+# File:    BcifWorkflow.py
+# Author:  James Smith
+# Date:    21-Feb-2025
+##
+
+"""
+Run workflow for pdb entries specified in input list.
+"""
+
+__docformat__ = "google en"
+__author__ = "James Smith"
+__email__ = "james.smith@rcsb.org"
+__license__ = "Apache 2.0"
+
+import os
+import logging
+from rcsb.workflow.bcif.task_functions import convertCifFilesToBcif
+
+logger = logging.getLogger(__name__)
+
+
+class BcifWorkflow:
+
+    def __init__(self, args):
+
+        # paths and files
+        self.listFileBase = args.listFileBase
+        self.listFileName = args.listFileName
+        self.remotePath = args.remotePath
+        self.outputPath = args.outputPath
+        # settings
+        self.contentType = args.contentType
+        self.nfiles = int(args.nfiles)
+        self.outfileSuffix = args.outfileSuffix
+        self.outputContentType = bool(args.outputContentType)
+        self.outputHash = bool(args.outputHash)
+        self.inputHash = bool(args.inputHash)
+        self.batchSize = int(args.batchSize)
+        # config
+        self.pdbxDict = args.pdbxDict
+        self.maDict = args.maDict
+        self.rcsbDict = args.rcsbDict
+        self.ihmDict = args.ihmDict
+        #
+        self.validate()
+
+    # public method required by pylint
+    def validate(self):
+
+        assert (
+            self.outfileSuffix == ".bcif" or self.outfileSuffix == ".bcif.gz"
+        ), "error - require either .bcif or .bcif.gz output file"
+
+        assert self.contentType in [
+            "pdb",
+            "csm",
+            "ihm",
+        ], "error - content type must be pdb, csm, or ihm"
+
+    def __call__(self):
+
+        logger.info(
+            "running bcif workflow on %s and %s",
+            os.path.join(self.listFileBase, self.listFileName),
+            self.remotePath,
+        )
+
+        convertCifFilesToBcif(
+            self.listFileName,
+            self.listFileBase,
+            self.remotePath,
+            self.outputPath,
+            self.outfileSuffix,
+            self.contentType,
+            self.outputContentType,
+            self.outputHash,
+            self.inputHash,
+            self.batchSize,
+            self.nfiles,
+            self.pdbxDict,
+            self.maDict,
+            self.rcsbDict,
+            self.ihmDict,
+        )

{rcsb_workflow-0.52 → rcsb_workflow-0.53/rcsb.workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rcsb.workflow
-Version: 0.52
+Version: 0.53
 Summary: RCSB Python data processing and ETL/ELT workflow entry points
 Home-page: https://github.com/rcsb/py-rcsb_workflow
 Author: John Westbrook
@@ -22,7 +22,9 @@ Requires-Dist: rcsb.utils.targets>=0.82
 Requires-Dist: rcsb.utils.seqalign>=0.31
 Requires-Dist: rcsb.utils.io>=1.48
 Requires-Dist: rcsb.exdb>=1.28
-Requires-Dist: rcsb.db>=1.800
+Requires-Dist: rcsb.db>=1.801
+Requires-Dist: requests>=2.32.3
+Requires-Dist: mmcif>=0.91.0
 Provides-Extra: dev
 Requires-Dist: check-manifest; extra == "dev"
 Provides-Extra: test

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/rcsb.workflow.egg-info/SOURCES.txt

@@ -14,16 +14,20 @@ rcsb.workflow.egg-info/not-zip-safe
 rcsb.workflow.egg-info/requires.txt
 rcsb.workflow.egg-info/top_level.txt
 rcsb/workflow/__init__.py
+rcsb/workflow/bcif/__init__.py
+rcsb/workflow/bcif/task_functions.py
 rcsb/workflow/chem/ChemCompFileWorkflow.py
 rcsb/workflow/chem/ChemCompImageWorkflow.py
 rcsb/workflow/chem/ChemCompSearchIndexWorkflow.py
 rcsb/workflow/chem/__init__.py
+rcsb/workflow/cli/BcifExec.py
 rcsb/workflow/cli/ExDbExec.py
 rcsb/workflow/cli/ImgExec.py
 rcsb/workflow/cli/__init__.py
 rcsb/workflow/targets/ProteinTargetSequenceExecutionWorkflow.py
 rcsb/workflow/targets/ProteinTargetSequenceWorkflow.py
 rcsb/workflow/targets/__init__.py
+rcsb/workflow/wuw/BcifWorkflow.py
 rcsb/workflow/wuw/ExDbWorkflow.py
 rcsb/workflow/wuw/PdbCsmImageWorkflow.py
 rcsb/workflow/wuw/__init__.py

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/rcsb.workflow.egg-info/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
+bcif_exec_cli = rcsb.workflow.cli.BcifExec:main
 exdb_wf_cli = rcsb.workflow.cli.ExDbExec:main
 imgs_exec_cli = rcsb.workflow.cli.ImgExec:main

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/rcsb.workflow.egg-info/requires.txt

@@ -4,7 +4,9 @@ rcsb.utils.targets>=0.82
 rcsb.utils.seqalign>=0.31
 rcsb.utils.io>=1.48
 rcsb.exdb>=1.28
-rcsb.db>=1.800
+rcsb.db>=1.801
+requests>=2.32.3
+mmcif>=0.91.0
 
 [dev]
 check-manifest

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/requirements.txt

@@ -6,4 +6,6 @@ rcsb.utils.targets >= 0.82
 rcsb.utils.seqalign >= 0.31
 rcsb.utils.io >= 1.48
 rcsb.exdb >= 1.28  # v1.28 is when circular dependency was removed
-rcsb.db >= 1.800
+rcsb.db >= 1.801
+requests >= 2.32.3
+mmcif >= 0.91.0

{rcsb_workflow-0.52 → rcsb_workflow-0.53}/setup.py

@@ -49,7 +49,7 @@ setup(
         "Programming Language :: Python :: 3.10",
     ],
     # entry_points={"console_scripts": ["cactvs_annotate_mol=rcsb.workflow.cactvsAnnotateMol:main"]},
-    entry_points={"console_scripts": ["exdb_wf_cli=rcsb.workflow.cli.ExDbExec:main", "imgs_exec_cli=rcsb.workflow.cli.ImgExec:main"]},
+    entry_points={"console_scripts": ["exdb_wf_cli=rcsb.workflow.cli.ExDbExec:main", "imgs_exec_cli=rcsb.workflow.cli.ImgExec:main", "bcif_exec_cli=rcsb.workflow.cli.BcifExec:main"]},
     #  The following is somewhat flakey --
     # dependency_links=[],
     install_requires=packagesRequired[1:],