looper 1.5.1__tar.gz → 1.6.0a1__tar.gz

{looper-1.5.1/looper.egg-info → looper-1.6.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: looper
-Version: 1.5.1
+Version: 1.6.0a1
 Summary: A pipeline submission engine that parses sample inputs and submits pipelines for each sample.
 Home-page: https://github.com/pepkit/looper
 Author: Nathan Sheffield, Vince Reuter, Michal Stolarczyk, Johanna Klughammer, Andre Rendeiro
@@ -15,6 +15,19 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
+Requires-Dist: colorama>=0.3.9
+Requires-Dist: divvy>=0.5.0
+Requires-Dist: eido>=0.2.1
+Requires-Dist: jinja2
+Requires-Dist: logmuse>=0.2.0
+Requires-Dist: pandas>=2.0.2
+Requires-Dist: pephubclient>=0.1.2
+Requires-Dist: peppy>=0.40.0.a4
+Requires-Dist: pipestat>=v0.6.0a9
+Requires-Dist: pyyaml>=3.12
+Requires-Dist: rich>=9.10.0
+Requires-Dist: ubiquerg>=0.5.2
+Requires-Dist: yacman>=0.9.2
 
 # <img src="docs/img/looper_logo.svg" alt="looper logo" height="70">
 

looper-1.6.0a1/looper/__init__.py

@@ -0,0 +1,44 @@
+"""Project configuration, particularly for logging.
+
+Project-scope constants may reside here, but more importantly, some setup here
+will provide a logging infrastructure for all of the project's modules.
+Individual modules and classes may provide separate configuration on a more
+local level, but this will at least provide a foundation.
+
+"""
+
+import logmuse
+
+logmuse.init_logger("looper")
+
+from .divvy import ComputingConfiguration, select_divvy_config
+from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME
+from .divvy import NEW_COMPUTE_KEY as COMPUTE_KEY
+
+from ._version import __version__
+from .conductor import (
+    SubmissionConductor,
+    write_submission_yaml,
+)
+from .plugins import (
+    write_sample_yaml,
+    write_sample_yaml_cwl,
+    write_sample_yaml_prj,
+    write_custom_template,
+)
+from .const import *
+from .pipeline_interface import PipelineInterface
+from .project import Project
+
+# Not used here, but make this the main import interface between peppy and
+# looper, so that other modules within this package need not worry about
+# the locations of some of the peppy declarations. Effectively, concentrate
+# the connection between peppy and looper here, to the extent possible.
+
+__all__ = [
+    "Project",
+    "PipelineInterface",
+    "SubmissionConductor",
+    "ComputingConfiguration",
+    "select_divvy_config",
+]

{looper-1.5.1 → looper-1.6.0a1}/looper/__main__.py

@@ -1,7 +1,7 @@
 import sys
 
-from .looper import main
-from .divvy import main as divvy_main
+from .cli_looper import main
+from .cli_divvy import main as divvy_main
 
 if __name__ == "__main__":
     try:

looper-1.6.0a1/looper/_version.py

@@ -0,0 +1 @@
+__version__ = "1.6.0a1"

looper-1.6.0a1/looper/cli_divvy.py

@@ -0,0 +1,182 @@
+import logmuse
+import os
+import sys
+import yaml
+from yaml import SafeLoader
+from ubiquerg import is_writable, VersionInHelpParser
+from .const import (
+    DEFAULT_COMPUTE_RESOURCES_NAME,
+    DEFAULT_CONFIG_FILEPATH,
+)
+from .divvy import select_divvy_config, ComputingConfiguration, divvy_init
+
+
+def build_argparser():
+    """
+    Builds argument parser.
+
+    :return argparse.ArgumentParser
+    """
+
+    banner = (
+        "%(prog)s - write compute job scripts that can be submitted to "
+        "any computing resource"
+    )
+    additional_description = "\nhttps://divvy.databio.org"
+
+    parser = VersionInHelpParser(
+        prog="divvy",
+        description=banner,
+        epilog=additional_description,
+        # version=__version__,
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    def add_subparser(cmd, description):
+        return subparsers.add_parser(cmd, description=description, help=description)
+
+    subparser_messages = {
+        "init": "Initialize a new divvy config file",
+        "list": "List available compute packages",
+        "write": "Write a job script",
+        "submit": "Write and then submit a job script",
+        "inspect": "Inspect compute package",
+    }
+
+    sps = {}
+    for cmd, desc in subparser_messages.items():
+        sps[cmd] = add_subparser(cmd, desc)
+        # sps[cmd].add_argument(
+        #     "config", nargs="?", default=None,
+        #     help="Divvy configuration file.")
+
+    for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
+        sp.add_argument(
+            "config", nargs="?", default=None, help="Divvy configuration file."
+        )
+
+    sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
+
+    for sp in [sps["inspect"]]:
+        sp.add_argument(
+            "-p",
+            "--package",
+            default=DEFAULT_COMPUTE_RESOURCES_NAME,
+            help="Select from available compute packages",
+        )
+
+    for sp in [sps["write"], sps["submit"]]:
+        sp.add_argument(
+            "-s",
+            "--settings",
+            help="YAML file with job settings to populate the template",
+        )
+
+        sp.add_argument(
+            "-p",
+            "--package",
+            default=DEFAULT_COMPUTE_RESOURCES_NAME,
+            help="Select from available compute packages",
+        )
+
+        sp.add_argument(
+            "-c",
+            "--compute",
+            nargs="+",
+            default=None,
+            help="Extra key=value variable pairs",
+        )
+
+        # sp.add_argument(
+        #         "-t", "--template",
+        #         help="Provide a template file (not yet implemented).")
+
+        sp.add_argument(
+            "-o", "--outfile", required=False, default=None, help="Output filepath"
+        )
+
+    return parser
+
+
+def main():
+    """Primary workflow for divvy CLI"""
+
+    parser = logmuse.add_logging_options(build_argparser())
+    # args, remaining_args = parser.parse_known_args()
+    args = parser.parse_args()
+
+    logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
+    logmuse.init_logger("yacman", **logger_kwargs)
+    global _LOGGER
+    _LOGGER = logmuse.logger_via_cli(args)
+
+    if not args.command:
+        parser.print_help()
+        _LOGGER.error("No command given")
+        sys.exit(1)
+
+    if args.command == "init":
+        divcfg = args.config
+        _LOGGER.debug("Initializing divvy configuration")
+        is_writable(os.path.dirname(divcfg), check_exist=False)
+        divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH)
+        sys.exit(0)
+
+    _LOGGER.debug("Divvy config: {}".format(args.config))
+    divcfg = select_divvy_config(args.config)
+    _LOGGER.info("Using divvy config: {}".format(divcfg))
+    dcc = ComputingConfiguration(filepath=divcfg)
+
+    if args.command == "list":
+        # Output header via logger and content via print so the user can
+        # redirect the list from stdout if desired without the header as clutter
+        _LOGGER.info("Available compute packages:\n")
+        print("{}".format("\n".join(dcc.list_compute_packages())))
+        sys.exit(1)
+
+    if args.command == "inspect":
+        # Output contents of selected compute package
+        _LOGGER.info("Your compute package template for: " + args.package + "\n")
+        found = False
+        for pkg_name, pkg in dcc.compute_packages.items():
+            if pkg_name == args.package:
+                found = True
+                with open(pkg.submission_template, "r") as f:
+                    print(f.read())
+                _LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
+                if pkg_name == "docker":
+                    print("Docker args are: " + pkg.docker_args)
+
+        if not found:
+            _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
+        sys.exit(1)
+
+    # Any non-divvy arguments will be passed along as key-value pairs
+    # that can be used to populate the template.
+    # keys = [str.replace(x, "--", "") for x in remaining_args[::2]]
+    # cli_vars = dict(zip(keys, remaining_args[1::2]))
+    if args.compute:
+        cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]}
+    else:
+        cli_vars = {}
+
+    if args.command == "write" or args.command == "submit":
+        try:
+            dcc.activate_package(args.package)
+        except AttributeError:
+            parser.print_help(sys.stderr)
+            sys.exit(1)
+
+        if args.settings:
+            _LOGGER.info("Loading settings file: %s", args.settings)
+            with open(args.settings, "r") as f:
+                vars_groups = [cli_vars, yaml.load(f, SafeLoader)]
+        else:
+            vars_groups = [cli_vars]
+
+        _LOGGER.debug(vars_groups)
+        if args.command == "write":
+            dcc.write_script(args.outfile, vars_groups)
+        elif args.command == "submit":
+            dcc.submit(args.outfile, vars_groups)

looper-1.5.1/looper/__init__.py → looper-1.6.0a1/looper/cli_looper.py

@@ -1,56 +1,31 @@
-"""Project configuration, particularly for logging.
-
-Project-scope constants may reside here, but more importantly, some setup here
-will provide a logging infrastructure for all of the project's modules.
-Individual modules and classes may provide separate configuration on a more
-local level, but this will at least provide a foundation.
-
-"""
-
-import logmuse
-
-logmuse.init_logger("looper")
-
 import argparse
-import logging
+import logmuse
 import os
-from typing import *
-from .divvy import ComputingConfiguration, select_divvy_config
-from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME
-from .divvy import NEW_COMPUTE_KEY as COMPUTE_KEY
+import sys
+import yaml
+
+from eido import inspect_project
+from pephubclient import PEPHubClient
+from typing import Tuple, List
 from ubiquerg import VersionInHelpParser
 
-from ._version import __version__
-from .conductor import (
-    SubmissionConductor,
-    write_sample_yaml,
-    write_sample_yaml_cwl,
-    write_sample_yaml_prj,
-    write_submission_yaml,
-    write_custom_template,
-)
+from . import __version__
 from .const import *
+from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
+from .exceptions import *
+from .looper import *
 from .parser_types import *
-from .pipeline_interface import PipelineInterface
-from .project import Project
-
-# Not used here, but make this the main import interface between peppy and
-# looper, so that other modules within this package need not worry about
-# the locations of some of the peppy declarations. Effectively, concentrate
-# the connection between peppy and looper here, to the extent possible.
-
-__all__ = [
-    "Project",
-    "PipelineInterface",
-    "SubmissionConductor",
-    "ComputingConfiguration",
-    "select_divvy_config",
-]
-
-
-SAMPLE_SELECTION_ATTRIBUTE_OPTNAME = "sel-attr"
-SAMPLE_EXCLUSION_OPTNAME = "sel-excl"
-SAMPLE_INCLUSION_OPTNAME = "sel-incl"
+from .project import Project, ProjectContext
+from .utils import (
+    dotfile_path,
+    enrich_args_via_cfg,
+    is_registry_path,
+    read_looper_dotfile,
+    read_looper_config_file,
+    read_yaml_file,
+    initiate_looper_config,
+    init_generic_pipeline,
+)
 
 
 class _StoreBoolActionType(argparse.Action):
@@ -77,21 +52,6 @@ class _StoreBoolActionType(argparse.Action):
         setattr(namespace, self.dest, self.const)
 
 
-MESSAGE_BY_SUBCOMMAND = {
-    "run": "Run or submit sample jobs.",
-    "rerun": "Resubmit sample jobs with failed flags.",
-    "runp": "Run or submit project jobs.",
-    "table": "Write summary stats table for project samples.",
-    "report": "Create browsable HTML report of project results.",
-    "destroy": "Remove output files of the project.",
-    "check": "Check flag status of current runs.",
-    "clean": "Run clean scripts of already processed jobs.",
-    "inspect": "Print information about a project.",
-    "init": "Initialize looper dotfile.",
-    "init-piface": "Initialize generic pipeline interface.",
-}
-
-
 def build_parser():
     """
     Building argument parser.
@@ -158,6 +118,7 @@ def build_parser():
         inspect_subparser = add_subparser("inspect")
         init_subparser = add_subparser("init")
         init_piface = add_subparser("init-piface")
+        link_subparser = add_subparser("link")
 
         # Flag arguments
         ####################################################################
@@ -308,7 +269,7 @@ def build_parser():
             )
 
         init_subparser.add_argument(
-            "config_file", help="Project configuration file (YAML)"
+            "pep_config", help="Project configuration file (PEP)"
         )
 
         init_subparser.add_argument(
@@ -366,6 +327,7 @@ def build_parser():
             clean_subparser,
             collate_subparser,
             inspect_subparser,
+            link_subparser,
         ]:
             subparser.add_argument(
                 "config_file",
@@ -442,6 +404,7 @@ def build_parser():
             clean_subparser,
             collate_subparser,
             inspect_subparser,
+            link_subparser,
         ]:
             fetch_samples_group = subparser.add_argument_group(
                 "sample selection arguments",
@@ -470,6 +433,7 @@ def build_parser():
                 metavar="ATTR",
                 help="Attribute for sample exclusion OR inclusion",
             )
+
             protocols = fetch_samples_group.add_mutually_exclusive_group()
             protocols.add_argument(
                 f"--{SAMPLE_EXCLUSION_OPTNAME}",
@@ -483,6 +447,22 @@ def build_parser():
                 metavar="I",
                 help="Include only samples with these values",
             )
+            fetch_samples_group.add_argument(
+                f"--{SAMPLE_SELECTION_FLAG_OPTNAME}",
+                default=None,
+                nargs="*",
+                metavar="SELFLAG",
+                help="Include samples with this flag status, e.g. completed",
+            )
+
+            fetch_samples_group.add_argument(
+                f"--{SAMPLE_EXCLUSION_FLAG_OPTNAME}",
+                default=None,
+                nargs="*",
+                metavar="EXCFLAG",
+                help="Exclude samples with this flag status, e.g. completed",
+            )
+
             subparser.add_argument(
                 "-a",
                 "--amend",
@@ -490,7 +470,13 @@ def build_parser():
                 metavar="A",
                 help="List of amendments to activate",
             )
-        for subparser in [report_subparser, table_subparser, check_subparser]:
+        for subparser in [
+            report_subparser,
+            table_subparser,
+            check_subparser,
+            destroy_subparser,
+            link_subparser,
+        ]:
             subparser.add_argument(
                 "--project",
                 help="Process project-level pipelines",
@@ -537,3 +523,254 @@ def validate_post_parse(args: argparse.Namespace) -> List[str]:
             f"Used multiple mutually exclusive options: {', '.join(used_exclusives)}"
         )
     return problems
+
+
+def _proc_resources_spec(args):
+    """
+    Process CLI-sources compute setting specification. There are two sources
+    of compute settings in the CLI alone:
+        * YAML file (--settings argument)
+        * itemized compute settings (--compute argument)
+
+    The itemized compute specification is given priority
+
+    :param argparse.Namespace: arguments namespace
+    :return Mapping[str, str]: binding between resource setting name and value
+    :raise ValueError: if interpretation of the given specification as encoding
+        of key-value pairs fails
+    """
+    spec = getattr(args, "compute", None)
+    try:
+        settings_data = read_yaml_file(args.settings) or {}
+    except yaml.YAMLError:
+        _LOGGER.warning(
+            "Settings file ({}) does not follow YAML format,"
+            " disregarding".format(args.settings)
+        )
+        settings_data = {}
+    if not spec:
+        return settings_data
+    pairs = [(kv, kv.split("=")) for kv in spec]
+    bads = []
+    for orig, pair in pairs:
+        try:
+            k, v = pair
+        except ValueError:
+            bads.append(orig)
+        else:
+            settings_data[k] = v
+    if bads:
+        raise ValueError(
+            "Could not correctly parse itemized compute specification. "
+            "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
+        )
+    return settings_data
+
+
+def main(test_args=None):
+    """Primary workflow"""
+    global _LOGGER
+
+    parser, aux_parser = build_parser()
+    aux_parser.suppress_defaults()
+
+    if test_args:
+        args, remaining_args = parser.parse_known_args(args=test_args)
+    else:
+        args, remaining_args = parser.parse_known_args()
+
+    cli_use_errors = validate_post_parse(args)
+    if cli_use_errors:
+        parser.print_help(sys.stderr)
+        parser.error(
+            f"{len(cli_use_errors)} CLI use problem(s): {', '.join(cli_use_errors)}"
+        )
+    if args.command is None:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+    if args.command == "init":
+        return int(
+            not initiate_looper_config(
+                dotfile_path(),
+                args.pep_config,
+                args.output_dir,
+                args.sample_pipeline_interfaces,
+                args.project_pipeline_interfaces,
+                args.force,
+            )
+        )
+
+    if args.command == "init-piface":
+        sys.exit(int(not init_generic_pipeline()))
+
+    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
+    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, args.command))
+
+    if "config_file" in vars(args):
+        if args.config_file is None:
+            looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+            try:
+                if args.looper_config:
+                    looper_config_dict = read_looper_config_file(args.looper_config)
+                else:
+                    looper_config_dict = read_looper_dotfile()
+                    _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+                for looper_config_key, looper_config_item in looper_config_dict.items():
+                    setattr(args, looper_config_key, looper_config_item)
+
+            except OSError:
+                parser.print_help(sys.stderr)
+                _LOGGER.warning(
+                    f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
+                )
+                sys.exit(1)
+        else:
+            _LOGGER.warning(
+                "This PEP configures looper through the project config. This approach is deprecated and will "
+                "be removed in future versions. Please use a looper config file. For more information see "
+                "looper.databio.org/en/latest/looper-config"
+            )
+
+    args = enrich_args_via_cfg(args, aux_parser, test_args)
+
+    # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
+    if vars(args)[PROJECT_PL_ARG]:
+        args.pipeline_interfaces = vars(args)[PROJECT_PL_ARG]
+
+    if len(remaining_args) > 0:
+        _LOGGER.warning(
+            "Unrecognized arguments: {}".format(
+                " ".join([str(x) for x in remaining_args])
+            )
+        )
+
+    divcfg = (
+        select_divvy_config(filepath=args.divvy) if hasattr(args, "divvy") else None
+    )
+
+    # Ignore flags if user is selecting or excluding on flags:
+    if args.sel_flag or args.exc_flag:
+        args.ignore_flags = True
+
+    # Initialize project
+    if is_registry_path(args.config_file):
+        if vars(args)[SAMPLE_PL_ARG]:
+            p = Project(
+                amendments=args.amend,
+                divcfg_path=divcfg,
+                runp=args.command == "runp",
+                project_dict=PEPHubClient()._load_raw_pep(
+                    registry_path=args.config_file
+                ),
+                **{
+                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                },
+            )
+        else:
+            raise MisconfigurationException(
+                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
+            )
+    else:
+        try:
+            p = Project(
+                cfg=args.config_file,
+                amendments=args.amend,
+                divcfg_path=divcfg,
+                runp=args.command == "runp",
+                **{
+                    attr: getattr(args, attr) for attr in CLI_PROJ_ATTRS if attr in args
+                },
+            )
+        except yaml.parser.ParserError as e:
+            _LOGGER.error(f"Project config parse failed -- {e}")
+            sys.exit(1)
+
+    selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
+    if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
+        _LOGGER.info(
+            "Failed to activate '{}' computing package. "
+            "Using the default one".format(selected_compute_pkg)
+        )
+
+    with ProjectContext(
+        prj=p,
+        selector_attribute=args.sel_attr,
+        selector_include=args.sel_incl,
+        selector_exclude=args.sel_excl,
+        selector_flag=args.sel_flag,
+        exclusion_flag=args.exc_flag,
+    ) as prj:
+        if args.command in ["run", "rerun"]:
+            run = Runner(prj)
+            try:
+                compute_kwargs = _proc_resources_spec(args)
+                return run(args, rerun=(args.command == "rerun"), **compute_kwargs)
+            except SampleFailedException:
+                sys.exit(1)
+            except IOError:
+                _LOGGER.error(
+                    "{} pipeline_interfaces: '{}'".format(
+                        prj.__class__.__name__, prj.pipeline_interface_sources
+                    )
+                )
+                raise
+
+        if args.command == "runp":
+            compute_kwargs = _proc_resources_spec(args)
+            collate = Collator(prj)
+            collate(args, **compute_kwargs)
+            return collate.debug
+
+        if args.command == "destroy":
+            return Destroyer(prj)(args)
+
+        # pipestat support introduces breaking changes and pipelines run
+        # with no pipestat reporting would not be compatible with
+        # commands: table, report and check. Therefore we plan maintain
+        # the old implementations for a couple of releases.
+        # if hasattr(args, "project"):
+        #     use_pipestat = (
+        #         prj.pipestat_configured_project
+        #         if args.project
+        #         else prj.pipestat_configured
+        #     )
+        use_pipestat = (
+            prj.pipestat_configured_project if args.project else prj.pipestat_configured
+        )
+        if args.command == "table":
+            if use_pipestat:
+                Tabulator(prj)(args)
+            else:
+                raise PipestatConfigurationException("table")
+
+        if args.command == "report":
+            if use_pipestat:
+                Reporter(prj)(args)
+            else:
+                raise PipestatConfigurationException("report")
+
+        if args.command == "link":
+            if use_pipestat:
+                Linker(prj)(args)
+            else:
+                raise PipestatConfigurationException("link")
+
+        if args.command == "check":
+            if use_pipestat:
+                return Checker(prj)(args)
+            else:
+                raise PipestatConfigurationException("check")
+
+        if args.command == "clean":
+            return Cleaner(prj)(args)
+
+        if args.command == "inspect":
+            inspect_project(p, args.sample_names, args.attr_limit)
+            from warnings import warn
+
+            warn(
+                "The inspect feature has moved to eido and will be removed in the future release of looper. "
+                "Use `eido inspect` from now on.",
+            )