rcsb.workflow 0.64__py3-none-any.whl

rcsb/workflow/cli/BcifExec.py

@@ -0,0 +1,179 @@
+##
+# File:    BcifExec.py
+# Author:  James Smith
+# Date:    21-Feb-2025
+##
+
+"""
+Entry point for bcif command line interface.
+"""
+
+__docformat__ = "google en"
+__author__ = "James Smith"
+__email__ = "james.smith@rcsb.org"
+__license__ = "Apache 2.0"
+
+import argparse
+import os
+import logging
+import time
+from rcsb.workflow.wuw.BcifWorkflow import BcifWorkflow
+
+logger = logging.getLogger()
+
+
+def main():
+
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    # paths
+    parser.add_argument(
+        "--listFileBase",
+        default="/tmp",
+        required=True,
+        help="path for input lists",
+    )
+    parser.add_argument(
+        "--listFileName",
+        default="pdbx_core_ids-1.txt",
+        required=True,
+        help="name of list file to read",
+    )
+    parser.add_argument(
+        "--remotePath",
+        default="http://prereleaseftp-external-east.rcsb.org/pdb/data/structures/divided/mmCIF/",
+        required=True,
+        help="url or directory for cif files",
+    )
+    parser.add_argument(
+        "--outputPath",
+        default="/mnt/vdb1/out",
+        required=True,
+        help="output directory for bcif files",
+    )
+    # settings
+    parser.add_argument(
+        "--contentType",
+        default="pdb",
+        required=True,
+        choices=["pdb", "csm", "ihm"],
+        help="which type of experiment was performed",
+    )
+    parser.add_argument(
+        "--nfiles",
+        default=0,
+        required=False,
+        help="set 0 for all files, set less than N for a test run, will not produce exactly n files",
+    )
+    parser.add_argument(
+        "--outfileSuffix",
+        default=".bcif.gz",
+        required=False,
+        choices=[".bcif", ".bcif.gz"],
+        help="whether to use additional gzip compression",
+    )
+    parser.add_argument(
+        "--batchSize",
+        default=1,
+        required=False,
+        help="optional subdivisions of sublists for pdb list and csm list",
+    )
+    # output folder structure, default none (save all output files in one folder)
+    parser.add_argument(
+        "--outputContentType",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether output paths should include a directory for the content type (pdb, csm)",
+    )
+    parser.add_argument(
+        "--outputHash",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether output paths should include the hash for the entry",
+    )
+    # input folder structure, default none (read all input files from one folder)
+    parser.add_argument(
+        "--inputHash",
+        action="store_true",
+        default=False,
+        required=False,
+        help="whether local input paths should include the hash for the entry",
+    )
+    # config
+    parser.add_argument(
+        "--pdbxDict",
+        default="https://raw.githubusercontent.com/wwpdb-dictionaries/mmcif_pdbx/master/dist/mmcif_pdbx_v5_next.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--maDict",
+        default="https://raw.githubusercontent.com/ihmwg/ModelCIF/master/dist/mmcif_ma_ext.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--rcsbDict",
+        default="https://raw.githubusercontent.com/rcsb/py-rcsb_exdb_assets/master/dictionary_files/dist/rcsb_mmcif_ext.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--ihmDict",
+        default="https://raw.githubusercontent.com/rcsb/py-rcsb_exdb_assets/master/dictionary_files/reference/mmcif_ihm_ext.dic",
+        required=False,
+    )
+    parser.add_argument(
+        "--flrDict",
+        default="https://raw.githubusercontent.com/ihmwg/flrCIF/refs/heads/master/dist/mmcif_ihm_flr_ext.dic",
+        required=False,
+    )
+    # logging
+    parser.add_argument("--log_file_path", required=False)
+    parser.add_argument("--debug", action="store_true", default=False, required=False)
+
+    args = parser.parse_args()
+
+    logFilePath = args.log_file_path
+    debugFlag = args.debug
+    if debugFlag:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    if logFilePath:
+        logDir = os.path.dirname(logFilePath)
+        if not os.path.isdir(logDir):
+            os.makedirs(logDir)
+        handler = logging.FileHandler(logFilePath, mode="a")
+        if debugFlag:
+            handler.setLevel(logging.DEBUG)
+        else:
+            handler.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            fmt="%(asctime)s @%(process)s [%(levelname)s]-%(module)s: %(message)s"
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    else:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            fmt="%(asctime)s @%(process)s [%(levelname)s]-%(module)s: %(message)s"
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+
+    logger.info("bcif workflow initialized")
+
+    try:
+        (BcifWorkflow(args))()
+    except RuntimeError as e:
+        raise Exception(str(e)) from e
+
+
+if __name__ == "__main__":
+    t = time.time()
+    try:
+        main()
+    except Exception as e:
+        logger.exception(str(e))
+        raise e
+    t2 = time.time() - t
+    logger.info("completed in %.2f s", t2)

rcsb/workflow/cli/ChemSearchIndexExec.py

@@ -0,0 +1,65 @@
+
+from rcsb.workflow.chem.ChemCompSearchIndexWorkflow import ChemCompSearchIndexWorkflow
+import argparse
+import logging
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+logger = logging.getLogger()
+
+
+def main():
+    """Build indices and configuration files required to support the chemical search service."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cachePath", required=True, help="Cache path.")
+    parser.add_argument("--ccUrlTarget", required=True, help="Chem comp url target.")
+    parser.add_argument("--birdUrlTarget", required=True, help="Bird url target.")
+    parser.add_argument("--channel", required=True, help="Channel.")
+    parser.add_argument("--useCache", action="store_true", default=False, required=True, help="Should it use the cache.")
+    parser.add_argument("--ccFileNamePrefix", required=True, help="Chem comp file name prefix.")
+    parser.add_argument("--numProc", required=True, help="Number of procs for makeIndices.")
+    parser.add_argument("--blHostName", required=True, help="Build locker host name.")
+    parser.add_argument("--blUploadPathUnsynced", required=True, help="Build locker upload path unsynced.")
+    parser.add_argument("--sftpUserName", required=True, help="SFTP username.")
+    parser.add_argument("--sftpUserPassword", required=True, help="SFTP user password.")
+    args = parser.parse_args()
+
+    # Build chemical indices -
+    logger.info("Using cache path %r", args.cachePath)
+    logger.info("Using ccFileNamePrefix %s", args.ccFileNamePrefix)
+    ccidxWf = ChemCompSearchIndexWorkflow(cachePath=args.cachePath, ccFileNamePrefix=args.ccFileNamePrefix)
+    okI = ccidxWf.makeIndices(
+        args.ccUrlTarget, args.birdUrlTarget, numProc=int(args.numProc), useCache=args.useCache
+    )
+    logger.info("Chemical search index build completed with status %r", okI)
+    #
+    okS = False
+    if okI:
+        # Store chemical indices for future use -
+        sftpHost = args.blHostName
+        okS = ccidxWf.stashIndices(
+            "sftp://" + sftpHost,
+            args.blUploadPathUnsynced,
+            bundleLabel=args.channel.upper(),
+            userName=args.sftpUserName,
+            pw=args.sftpUserPassword,
+        )
+        logger.info("Chemical search index stashed with status %r", okS)
+    else:
+        msg = "Chemical search index completed with status %r" % okI
+        raise ValueError(msg)
+    #
+
+    if not okS:
+        error_msg = "Failed to stash chemical search index data"
+        raise ValueError(error_msg)
+
+    message = "Chemical Search Index completed with status %r stash status %r" % (okI, okS)
+    logger.info("Completion message is %s", message)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as err:
+        fail_msg = f"Run failed {err}"
+        raise RuntimeError(fail_msg) from err

rcsb/workflow/cli/ExDbExec.py

@@ -0,0 +1,240 @@
+##
+# File: ExDbExec.py
+# Date: 22-Apr-2019  jdw
+#
+#  Execution wrapper  --  for extract and load operations -
+#
+#  Updates:
+#   4-Sep-2019 jdw add Tree and Drugbank loaders
+#  14-Feb-2020 jdw change over to ReferenceSequenceAnnotationProvider/Adapter
+#   9-Mar-2023 dwp Lower refChunkSize to 10 (UniProt API having trouble streaming XML responses)
+#  25-Apr-2024 dwp Add arguments and logic to support CLI usage from weekly-update workflow;
+#                  Add support for logging output to a specific file
+#  20-Aug-2024 dwp Add load_target_cofactors operation; change name of upd_targets_cofactors to upd_targets
+#  22-Oct-2024 dwp Add ccd_img_gen and ccd_file_gen operations
+#                  (latter will only be used briefly, as will stop generating SDF and Mol2 files in Dec 2024)
+#  31-Jan-2025 mjt Moved this script from rcsb.exdb (to remove circluar dependencies)
+##
+__docformat__ = "google en"
+__author__ = "John Westbrook"
+__email__ = "jwest@rcsb.rutgers.edu"
+__license__ = "Apache 2.0"
+
+import os
+import sys
+import argparse
+import logging
+
+from rcsb.utils.config.ConfigUtil import ConfigUtil
+from rcsb.workflow.wuw.ExDbWorkflow import ExDbWorkflow
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+TOPDIR = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+
+# logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s", stream=sys.stdout)
+logger = logging.getLogger()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    #
+    parser.add_argument(
+        "--op",
+        default=None,
+        required=True,
+        help="Loading operation to perform",
+        choices=[
+            "etl_chemref",  # ETL integrated chemical reference data
+            "etl_uniprot_core",  # ETL UniProt core reference data
+            "etl_tree_node_lists",  # ETL tree node lists
+            "upd_ref_seq",  # Update reference sequence assignments
+            "upd_neighbor_interactions",
+            "upd_uniprot_taxonomy",
+            "upd_targets",
+            "load_target_cofactors",
+            "upd_pubchem",
+            "upd_entry_info",
+            "upd_glycan_idx",
+            "upd_resource_stash",
+            "ccd_img_gen",
+            "ccd_file_gen",
+        ]
+    )
+    parser.add_argument(
+        "--load_type",
+        default="full",
+        help="Type of load ('full' for complete and fresh single-worker load, 'replace' for incremental and multi-worker load)",
+        choices=["full", "replace"],
+    )
+    #
+    parser.add_argument("--config_path", default=None, help="Path to configuration options file")
+    parser.add_argument("--config_name", default="site_info_remote_configuration", help="Configuration section name")
+    parser.add_argument("--cache_path", default=None, help="Cache path for resource files")
+    parser.add_argument("--num_proc", default=2, help="Number of processes to execute (default=2)")
+    parser.add_argument("--chunk_size", default=10, help="Number of files loaded per process")
+    parser.add_argument("--max_step_length", default=500, help="Maximum subList size (default=500)")
+    parser.add_argument("--db_type", default="mongo", help="Database server type (default=mongo)")
+    parser.add_argument("--document_limit", default=None, help="Load document limit for testing")
+    #
+    parser.add_argument("--rebuild_cache", default=False, action="store_true", help="Rebuild cached resource files")
+    parser.add_argument("--rebuild_sequence_cache", default=False, action="store_true", help="Rebuild cached resource files for reference sequence updates")
+    parser.add_argument("--provider_types_exclude", default=None, help="Resource provider types to exclude")
+    parser.add_argument("--use_filtered_tax_list", default=False, action="store_true", help="Use filtered list for taxonomy tree loading")
+    parser.add_argument("--disable_read_back_check", default=False, action="store_true", help="Disable read back check on all documents")
+    parser.add_argument("--debug", default=False, action="store_true", help="Turn on verbose logging")
+    parser.add_argument("--mock", default=False, action="store_true", help="Use MOCK repository configuration for testing")
+    parser.add_argument("--log_file_path", default=None, help="Path to runtime log file output.")
+    #
+    # Arguments specific for op == 'upd_ref_seq'
+    parser.add_argument("--ref_chunk_size", default=10, help="Max chunk size for reference sequence updates (for op 'upd_ref_seq')")
+    parser.add_argument("--min_missing", default=0, help="Minimum number of allowed missing reference sequences (for op 'upd_ref_seq')")
+    parser.add_argument("--min_match_primary_percent", default=None, help="Minimum reference sequence match percentage (for op 'upd_ref_seq')")
+    parser.add_argument("--test_mode", default=False, action="store_true", help="Test mode for reference sequence updates (for op 'upd_ref_seq')")
+    #
+    # Arguments specific for op == 'ccd_img_gen' or 'ccd_file_gen'
+    parser.add_argument("--cc_output_path", default=None, help="The base local directory path where chemical component files (image, coordinates) are written (for op 'ccd_img_gen')")
+    parser.add_argument("--cc_cache_path", default=None, help="The base local directory path where chemical component cache data are written (for op 'ccd_img_gen')")
+    parser.add_argument("--oe_license_path", default=None, help="Path to OpenEye license file")
+    #
+    # Arguments buildExdbResources
+    parser.add_argument("--rebuild_all_neighbor_interactions", default=False, action="store_true", help="Rebuild all neighbor interactions from scratch (default is incrementally)")
+    parser.add_argument("--cc_file_prefix", default="cc-full", help="File name discriminator for index sets")
+    parser.add_argument("--cc_url_target", default=None, help="target url for chemical component dictionary resource file (default: None=all public)")
+    parser.add_argument("--bird_url_target", default=None, help="target url for bird dictionary resource file (cc format) (default: None=all public)")
+    #
+    args = parser.parse_args()
+    #
+    try:
+        op, commonD, loadD = processArguments(args)
+    except Exception as err:
+        logger.exception("Argument processing problem %s", str(err))
+        raise ValueError("Argument processing problem") from err
+    #
+    #
+    # Log input arguments
+    loadLogD = {k: v for d in [commonD, loadD] for k, v in d.items() if k != "inputIdCodeList"}
+    logger.info("running load op %r on loadLogD %r:", op, loadLogD)
+    #
+    # Run the operation
+    okR = False
+    exWf = ExDbWorkflow(**commonD)
+    if op in ["etl_chemref", "etl_uniprot_core", "etl_tree_node_lists", "upd_ref_seq"]:
+        okR = exWf.load(op, **loadD)
+    elif op in [
+        "upd_neighbor_interactions",
+        "upd_uniprot_taxonomy",
+        "upd_targets",
+        "load_target_cofactors",
+        "upd_pubchem",
+        "upd_entry_info",
+        "upd_glycan_idx",
+        "upd_resource_stash",
+    ]:
+        okR = exWf.buildExdbResource(op, **loadD)
+    elif op in [
+        "ccd_img_gen",
+        "ccd_file_gen",
+    ]:
+        okR = exWf.generateCcdFiles(op, **loadD)
+    else:
+        logger.error("Unsupported op %r", op)
+    #
+    logger.info("Operation %r completed with status %r", op, okR)
+    #
+    if not okR:
+        logger.error("Operation %r failed with status %r", op, okR)
+        raise ValueError("Operation %r failed" % op)
+
+
+def processArguments(args):
+    # Logging details
+    logFilePath = args.log_file_path
+    debugFlag = args.debug
+    if debugFlag:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    if logFilePath:
+        logDir = os.path.dirname(logFilePath)
+        if not os.path.isdir(logDir):
+            os.makedirs(logDir)
+        handler = logging.FileHandler(logFilePath, mode="a")
+        if debugFlag:
+            handler.setLevel(logging.DEBUG)
+        else:
+            handler.setLevel(logging.INFO)
+        formatter = logging.Formatter("%(asctime)s [%(levelname)s]-%(module)s.%(funcName)s: %(message)s")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    #
+    # Configuration details
+    configPath = args.config_path
+    configName = args.config_name
+    if not (configPath and configName):
+        logger.error("Config path and/or name not provided: %r, %r", configPath, configName)
+        raise ValueError("Config path and/or name not provided: %r, %r" % (configPath, configName))
+    mockTopPath = os.path.join(TOPDIR, "rcsb", "mock-data") if args.mock else None
+    logger.info("Using configuration file %r (section %r)", configPath, configName)
+    cfgOb = ConfigUtil(configPath=configPath, defaultSectionName=configName, mockTopPath=mockTopPath)
+    cfgObTmp = cfgOb.exportConfig()
+    logger.info("Length of config object (%r)", len(cfgObTmp))
+    if len(cfgObTmp) == 0:
+        logger.error("Missing or access issue for config file %r", configPath)
+        raise ValueError("Missing or access issue for config file %r" % configPath)
+    else:
+        del cfgObTmp
+    #
+    # Do any additional argument checking
+    op = args.op
+    if not op:
+        raise ValueError("Must supply a value to '--op' argument")
+    #
+    cachePath = args.cache_path if args.cache_path else "."
+    cachePath = os.path.abspath(cachePath)
+
+    if args.db_type != "mongo":
+        logger.error("Unsupported database type %r (must be 'mongo')", args.db_type)
+        raise ValueError("Unsupported database type %r (must be 'mongo')" % args.db_type)
+
+    # Now collect arguments into dictionaries
+    commonD = {
+        "configPath": configPath,
+        "configName": configName,
+        "cachePath": cachePath,
+        "mockTopPath": mockTopPath,
+        "debugFlag": debugFlag,
+        "rebuildCache": args.rebuild_cache,
+        "providerTypeExcludeL": args.provider_types_exclude,
+    }
+    loadD = {
+        "loadType": args.load_type,
+        "numProc": int(args.num_proc),
+        "chunkSize": int(args.chunk_size),
+        "maxStepLength": int(args.max_step_length),
+        "dbType": args.db_type,
+        "documentLimit": int(args.document_limit) if args.document_limit else None,
+        "readBackCheck": not args.disable_read_back_check,
+        "rebuildSequenceCache": args.rebuild_sequence_cache,
+        "useFilteredLists": args.use_filtered_tax_list,
+        "refChunkSize": int(args.ref_chunk_size),
+        "minMissing": int(args.min_missing),
+        "minMatchPrimaryPercent": float(args.min_match_primary_percent) if args.min_match_primary_percent else None,
+        "testMode": args.test_mode,
+        "rebuildAllNeighborInteractions": args.rebuild_all_neighbor_interactions,
+        "ccFileNamePrefix": args.cc_file_prefix,
+        "ccUrlTarget": args.cc_url_target,
+        "birdUrlTarget": args.bird_url_target,
+        "ccOutputPath": args.cc_output_path,
+        "ccCachePath": args.cc_cache_path,
+        "licenseFilePath": args.oe_license_path,
+    }
+
+    return op, commonD, loadD
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        logger.exception("Run failed %s", str(e))
+        sys.exit(1)

rcsb/workflow/cli/ImgExec.py

@@ -0,0 +1,64 @@
+"""Cli parser for jpg generation. This is a portion of imgs workflow which takes bcif images and generates these jpgs as well as chem images."""
+##
+# File: ImgExec.py
+# Date: 11-Dec-2024  mjt
+#
+#  Execution wrapper  --  for jpg generation -
+#
+#  Updates:
+#
+##
+__docformat__ = "google en"
+__author__ = "Michael Trumbull"
+__email__ = "michael.trumbull@rcsb.org"
+__license__ = "Apache 2.0"
+
+import argparse
+from rcsb.workflow.wuw.PdbCsmImageWorkflow import PdbCsmImageWorkflow
+
+
+def main() -> None:
+    """Cli parser for images workflow."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--op",
+        required=True,
+        help="which function to call",
+        choices=[
+            "genJpgs",
+        ],
+    )
+
+    parser.add_argument("--baseDir", help="Base path for experimental or computational bcif file.")
+    parser.add_argument("--idListFilePath", help="List of ids to generate images for.")
+    parser.add_argument("--molrenderExe", default="/opt/modules/node_modules/molrender/build/bin/molrender.js", help="Molrender executable location.")
+    parser.add_argument("--jpgsOutDir", help="Where to put the complete jpg images.")
+    parser.add_argument("--jpgHeight", default=500, help="Height of generated jpg.")
+    parser.add_argument("--jpgWidth", default=500, help="Width of generated jpg.")
+    parser.add_argument("--jpgFormat", default="jpeg", help="Output image format.")
+    parser.add_argument("--jpgXvfbExecutable", default="/usr/bin/xvfb-run", help="Linux executable that allows this gui application to run in cli.")
+    parser.add_argument("--jpgScreen", default="1280x1024x24", help="Screen dimensions image will be rendered for.")
+    parser.add_argument("--jpgRender", default="all", help="Which elements of the potein do you want to render.")
+    parser.add_argument("--checkFileAppend", default="_model-1.jpeg", help="What jpg file do you want to check exists after the process runs.")
+    parser.add_argument("--numProcs", default=1, help="How many processors are available for parallel execution.")
+    parser.add_argument("--holdingsFilePath", help="Path (including filename) to holdings file .json file (csm, pdb, ihm)")
+    parser.add_argument("--targetFileSuffix", default="_model-1.jpeg", help="string that follows the ID in the jpg file name for comparing timestamps.")
+    parser.add_argument("--csmHoldingsFileSubstring", default="computed-models-holdings-list", help="substring in csm holdings file path to determine if we are working with CSMs.")
+    parser.add_argument("--baseUrl", help="Base url for model file. This will supercede any local bcif.")
+    parser.add_argument("--modelFileExtension", default=".bcif.gz", help="Model file type (.bcif.gz, .bcif, .cif).")
+    parser.add_argument("--tmpDir", default="/tmp", help="Temporary dir for downloading model file over url.")
+    args = vars(parser.parse_args())
+
+    imgWF = PdbCsmImageWorkflow()
+    if args["op"] == "genJpgs":
+        imgWF.imagesGenJpgs(**args)
+    else:
+        raise ValueError("CLI --op flag error: not available option %r" % args.op)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as err:
+        msg = f"Run failed {err}"
+        raise RuntimeError(msg) from err