looper 1.7.0a1__tar.gz → 2.0.0__tar.gz

{looper-1.7.0a1 → looper-2.0.0}/MANIFEST.in

@@ -6,3 +6,4 @@ include looper/default_config/*
 include looper/default_config/divvy_templates/*
 include looper/jinja_templates_old/*
 include looper/schemas/*
+include looper/command_models/*

{looper-1.7.0a1/looper.egg-info → looper-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: looper
-Version: 1.7.0a1
+Version: 2.0.0
 Summary: A pipeline submission engine that parses sample inputs and submits pipelines for each sample.
 Home-page: https://github.com/pepkit/looper
 Author: Nathan Sheffield, Vince Reuter, Michal Stolarczyk, Johanna Klughammer, Andre Rendeiro
@@ -16,18 +16,28 @@ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: colorama>=0.3.9
-Requires-Dist: divvy>=0.5.0
-Requires-Dist: eido>=0.2.1
+Requires-Dist: eido>=0.2.4
 Requires-Dist: jinja2
 Requires-Dist: logmuse>=0.2.0
 Requires-Dist: pandas>=2.0.2
-Requires-Dist: pephubclient>=0.1.2
-Requires-Dist: peppy>=0.40.0
-Requires-Dist: pipestat>=0.7.1a1
+Requires-Dist: pephubclient>=0.4.0
+Requires-Dist: pipestat>=0.12.0a1
+Requires-Dist: peppy>=0.40.6
 Requires-Dist: pyyaml>=3.12
 Requires-Dist: rich>=9.10.0
-Requires-Dist: ubiquerg>=0.5.2
-Requires-Dist: yacman>=0.9.2
+Requires-Dist: ubiquerg>=0.8.1a1
+Requires-Dist: yacman==0.9.3
+Requires-Dist: pydantic-argparse>=0.9.0
+Requires-Dist: psutil
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # <img src="docs/img/looper_logo.svg" alt="looper logo" height="70">
 

{looper-1.7.0a1 → looper-2.0.0}/looper/__main__.py

@@ -1,6 +1,6 @@
 import sys
 
-from .cli_looper import main
+from .cli_pydantic import main
 from .cli_divvy import main as divvy_main
 
 if __name__ == "__main__":

looper-2.0.0/looper/_version.py

@@ -0,0 +1,2 @@
+__version__ = "2.0.0"
+# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!

{looper-1.7.0a1 → looper-2.0.0}/looper/cli_divvy.py

@@ -53,10 +53,10 @@ def build_argparser():
 
     for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
         sp.add_argument(
-            "config", nargs="?", default=None, help="Divvy configuration file."
+            "--config", nargs="?", default=None, help="Divvy configuration file."
         )
 
-    sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
+    sps["init"].add_argument("--config", default=None, help="Divvy configuration file.")
 
     for sp in [sps["inspect"]]:
         sp.add_argument(
@@ -124,9 +124,11 @@ def main():
         sys.exit(0)
 
     _LOGGER.debug("Divvy config: {}".format(args.config))
+
     divcfg = select_divvy_config(args.config)
+
     _LOGGER.info("Using divvy config: {}".format(divcfg))
-    dcc = ComputingConfiguration(filepath=divcfg)
+    dcc = ComputingConfiguration.from_yaml_file(filepath=divcfg)
 
     if args.command == "list":
         # Output header via logger and content via print so the user can
@@ -142,11 +144,13 @@ def main():
         for pkg_name, pkg in dcc.compute_packages.items():
             if pkg_name == args.package:
                 found = True
-                with open(pkg.submission_template, "r") as f:
+                with open(pkg["submission_template"], "r") as f:
                     print(f.read())
-                _LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
+                _LOGGER.info(
+                    "Submission command is: " + pkg["submission_command"] + "\n"
+                )
                 if pkg_name == "docker":
-                    print("Docker args are: " + pkg.docker_args)
+                    print("Docker args are: " + pkg["docker_args"])
 
         if not found:
             _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")

looper-2.0.0/looper/cli_pydantic.py

@@ -0,0 +1,413 @@
+"""
+CLI script using `pydantic-argparse` for parsing of arguments
+
+Arguments / commands are defined in `command_models/` and are given, eventually, as
+`pydantic` models, allowing for type-checking and validation of arguments.
+
+Note: this is only a test script so far, and coexists next to the current CLI
+(`cli_looper.py`), which uses `argparse` directly. The goal is to eventually
+replace the current CLI with a CLI based on above-mentioned `pydantic` models,
+but whether this will happen with `pydantic-argparse` or another, possibly self-
+written library is not yet clear.
+It is well possible that this script will be removed again.
+"""
+
+# Note: The following import is used for forward annotations (Python 3.8)
+# to prevent potential 'TypeError' related to the use of the '|' operator
+# with types.
+from __future__ import annotations
+
+import sys
+
+import logmuse
+import pydantic_argparse
+import yaml
+from eido import inspect_project
+from pephubclient import PEPHubClient
+from pydantic_argparse.argparse.parser import ArgumentParser
+
+from . import __version__
+
+from .command_models.arguments import ArgumentEnum
+
+from .command_models.commands import (
+    SUPPORTED_COMMANDS,
+    TopLevelParser,
+    add_short_arguments,
+)
+from .const import *
+from .divvy import DEFAULT_COMPUTE_RESOURCES_NAME, select_divvy_config
+from .exceptions import *
+from .looper import *
+from .parser_types import *
+from .project import Project, ProjectContext
+from .utils import (
+    dotfile_path,
+    enrich_args_via_cfg,
+    is_pephub_registry_path,
+    read_looper_config_file,
+    read_looper_dotfile,
+    initiate_looper_config,
+    init_generic_pipeline,
+    read_yaml_file,
+    inspect_looper_config_file,
+    is_PEP_file_type,
+    looper_config_tutorial,
+)
+
+from typing import List, Tuple
+from rich.console import Console
+
+
+def opt_attr_pair(name: str) -> Tuple[str, str]:
+    """Takes argument as attribute and returns as tuple of top-level or subcommand used."""
+    return f"--{name}", name.replace("-", "_")
+
+
+def validate_post_parse(args: argparse.Namespace) -> List[str]:
+    """Checks if user is attempting to use mutually exclusive options."""
+    problems = []
+    used_exclusives = [
+        opt
+        for opt, attr in map(
+            opt_attr_pair,
+            [
+                "skip",
+                "limit",
+                SAMPLE_EXCLUSION_OPTNAME,
+                SAMPLE_INCLUSION_OPTNAME,
+            ],
+        )
+        # Depending on the subcommand used, the above options might either be in
+        # the top-level namespace or in the subcommand namespace (the latter due
+        # to a `modify_args_namespace()`)
+        if getattr(
+            args, attr, None
+        )  # or (getattr(args.run, attr, None) if hasattr(args, "run") else False)
+    ]
+    if len(used_exclusives) > 1:
+        problems.append(
+            f"Used multiple mutually exclusive options: {', '.join(used_exclusives)}"
+        )
+    return problems
+
+
+# TODO rename to run_looper_via_cli for running lloper as a python library:
+#  https://github.com/pepkit/looper/pull/472#discussion_r1521970763
+def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
+    # here comes adapted `cli_looper.py` code
+    global _LOGGER
+
+    _LOGGER = logmuse.logger_via_cli(args, make_root=True)
+
+    # Find out which subcommand was used
+    supported_command_names = [cmd.name for cmd in SUPPORTED_COMMANDS]
+    subcommand_valued_args = [
+        (arg, value)
+        for arg, value in vars(args).items()
+        if arg and arg in supported_command_names and value is not None
+    ]
+    # Only one subcommand argument will be not `None`, else we found a bug in `pydantic-argparse`
+    [(subcommand_name, subcommand_args)] = subcommand_valued_args
+
+    cli_use_errors = validate_post_parse(subcommand_args)
+    if cli_use_errors:
+        parser.print_help(sys.stderr)
+        parser.error(
+            f"{len(cli_use_errors)} CLI use problem(s): {', '.join(cli_use_errors)}"
+        )
+
+    if subcommand_name is None:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+    if subcommand_name == "init":
+
+        console = Console()
+        console.clear()
+        console.rule(f"\n[magenta]Looper initialization[/magenta]")
+        selection = subcommand_args.generic
+        if selection is True:
+            console.clear()
+            return int(
+                not initiate_looper_config(
+                    dotfile_path(),
+                    subcommand_args.pep_config,
+                    subcommand_args.output_dir,
+                    subcommand_args.sample_pipeline_interfaces,
+                    subcommand_args.project_pipeline_interfaces,
+                    subcommand_args.force_yes,
+                )
+            )
+        else:
+            console.clear()
+            return int(looper_config_tutorial())
+
+    if subcommand_name == "init_piface":
+        sys.exit(int(not init_generic_pipeline()))
+
+    _LOGGER.info("Looper version: {}\nCommand: {}".format(__version__, subcommand_name))
+
+    looper_cfg_path = os.path.relpath(dotfile_path(), start=os.curdir)
+    try:
+        if subcommand_args.config:
+            looper_config_dict = read_looper_config_file(subcommand_args.config)
+        else:
+            looper_config_dict = read_looper_dotfile()
+            _LOGGER.info(f"Using looper config ({looper_cfg_path}).")
+
+        cli_modifiers_dict = None
+        for looper_config_key, looper_config_item in looper_config_dict.items():
+            if looper_config_key == CLI_KEY:
+                cli_modifiers_dict = looper_config_item
+            else:
+                setattr(subcommand_args, looper_config_key, looper_config_item)
+
+    except OSError as e:
+        if subcommand_args.config:
+            _LOGGER.warning(
+                f"\nLooper config file does not exist at given path {subcommand_args.config}. Use looper init to create one at {looper_cfg_path}."
+            )
+        else:
+            _LOGGER.warning(e)
+
+        sys.exit(1)
+
+    subcommand_args = enrich_args_via_cfg(
+        subcommand_name,
+        subcommand_args,
+        parser,
+        test_args=test_args,
+        cli_modifiers=cli_modifiers_dict,
+    )
+
+    # If project pipeline interface defined in the cli, change name to: "pipeline_interface"
+    if vars(subcommand_args)[PROJECT_PL_ARG]:
+        subcommand_args.pipeline_interfaces = vars(subcommand_args)[PROJECT_PL_ARG]
+
+    divcfg = (
+        select_divvy_config(filepath=subcommand_args.divvy)
+        if hasattr(subcommand_args, "divvy")
+        else None
+    )
+    # Ignore flags if user is selecting or excluding on flags:
+    if subcommand_args.sel_flag or subcommand_args.exc_flag:
+        subcommand_args.ignore_flags = True
+
+    # Initialize project
+    if is_PEP_file_type(subcommand_args.pep_config) and os.path.exists(
+        subcommand_args.pep_config
+    ):
+        try:
+            p = Project(
+                cfg=subcommand_args.pep_config,
+                amendments=subcommand_args.amend,
+                divcfg_path=divcfg,
+                runp=subcommand_name == "runp",
+                **{
+                    attr: getattr(subcommand_args, attr)
+                    for attr in CLI_PROJ_ATTRS
+                    if attr in subcommand_args
+                },
+            )
+        except yaml.parser.ParserError as e:
+            _LOGGER.error(f"Project config parse failed -- {e}")
+            sys.exit(1)
+    elif is_pephub_registry_path(subcommand_args.pep_config):
+        if vars(subcommand_args)[SAMPLE_PL_ARG]:
+            p = Project(
+                amendments=subcommand_args.amend,
+                divcfg_path=divcfg,
+                runp=subcommand_name == "runp",
+                project_dict=PEPHubClient().load_raw_pep(
+                    registry_path=subcommand_args.pep_config
+                ),
+                **{
+                    attr: getattr(subcommand_args, attr)
+                    for attr in CLI_PROJ_ATTRS
+                    if attr in subcommand_args
+                },
+            )
+        else:
+            raise MisconfigurationException(
+                f"`sample_pipeline_interface` is missing. Provide it in the parameters."
+            )
+    else:
+        raise MisconfigurationException(
+            f"Cannot load PEP. Check file path or registry path to pep."
+        )
+
+    selected_compute_pkg = p.selected_compute_package or DEFAULT_COMPUTE_RESOURCES_NAME
+    if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
+        _LOGGER.info(
+            "Failed to activate '{}' computing package. "
+            "Using the default one".format(selected_compute_pkg)
+        )
+
+    with ProjectContext(
+        prj=p,
+        selector_attribute=subcommand_args.sel_attr,
+        selector_include=subcommand_args.sel_incl,
+        selector_exclude=subcommand_args.sel_excl,
+        selector_flag=subcommand_args.sel_flag,
+        exclusion_flag=subcommand_args.exc_flag,
+    ) as prj:
+
+        # Check at the beginning if user wants to use pipestat and pipestat is configurable
+        is_pipestat_configured = (
+            prj._check_if_pipestat_configured(pipeline_type=PipelineLevel.PROJECT.value)
+            if getattr(subcommand_args, "project", None) or subcommand_name == "runp"
+            else prj._check_if_pipestat_configured()
+        )
+
+        if subcommand_name in ["run", "rerun"]:
+            if getattr(subcommand_args, "project", None):
+                _LOGGER.warning(
+                    "Project flag set but 'run' command was used. Please use 'runp' to run at project-level."
+                )
+            rerun = subcommand_name == "rerun"
+            run = Runner(prj)
+            try:
+                # compute_kwargs = _proc_resources_spec(args)
+                compute_kwargs = _proc_resources_spec(subcommand_args)
+
+                # TODO Shouldn't top level args and subcommand args be accessible on the same object?
+                return run(
+                    subcommand_args, top_level_args=args, rerun=rerun, **compute_kwargs
+                )
+            except SampleFailedException:
+                sys.exit(1)
+            except IOError:
+                _LOGGER.error(
+                    "{} pipeline_interfaces: '{}'".format(
+                        prj.__class__.__name__, prj.pipeline_interface_sources
+                    )
+                )
+                raise
+
+        if subcommand_name == "runp":
+            compute_kwargs = _proc_resources_spec(subcommand_args)
+            collate = Collator(prj)
+            collate(subcommand_args, **compute_kwargs)
+            return collate.debug
+
+        if subcommand_name == "destroy":
+            return Destroyer(prj)(subcommand_args)
+
+        if subcommand_name == "table":
+            if is_pipestat_configured:
+                return Tabulator(prj)(subcommand_args)
+            else:
+                raise PipestatConfigurationException("table")
+
+        if subcommand_name == "report":
+            if is_pipestat_configured:
+                return Reporter(prj)(subcommand_args)
+            else:
+                raise PipestatConfigurationException("report")
+
+        if subcommand_name == "link":
+            if is_pipestat_configured:
+                Linker(prj)(subcommand_args)
+            else:
+                raise PipestatConfigurationException("link")
+
+        if subcommand_name == "check":
+            if is_pipestat_configured:
+                return Checker(prj)(subcommand_args)
+            else:
+                raise PipestatConfigurationException("check")
+
+        if subcommand_name == "clean":
+            return Cleaner(prj)(subcommand_args)
+
+        if subcommand_name == "inspect":
+            # Inspect PEP from Eido
+            sample_names = []
+            for sample in p.samples:
+                sample_names.append(sample["sample_name"])
+            inspect_project(p, sample_names)
+            # Inspect looper config file
+            if looper_config_dict:
+                inspect_looper_config_file(looper_config_dict)
+            else:
+                _LOGGER.warning("No looper configuration was supplied.")
+
+
+def main(test_args=None) -> dict:
+    parser = pydantic_argparse.ArgumentParser(
+        model=TopLevelParser,
+        prog="looper",
+        description="Looper: A job submitter for Portable Encapsulated Projects",
+        add_help=True,
+        version="2.0.0",
+    )
+
+    parser = add_short_arguments(parser, ArgumentEnum)
+
+    if test_args:
+        args = parser.parse_typed_args(args=test_args)
+    else:
+        args = parser.parse_typed_args()
+
+    return run_looper(args, parser, test_args=test_args)
+
+
+def main_cli() -> None:
+    main()
+
+
+def _proc_resources_spec(args):
+    """
+    Process CLI-sources compute setting specification. There are two sources
+    of compute settings in the CLI alone:
+        * YAML file (--settings argument)
+        * itemized compute settings (--compute argument)
+
+    The itemized compute specification is given priority
+
+    :param argparse.Namespace: arguments namespace
+    :return Mapping[str, str]: binding between resource setting name and value
+    :raise ValueError: if interpretation of the given specification as encoding
+        of key-value pairs fails
+    """
+    spec = getattr(args, "compute", None)
+    settings = args.settings
+    try:
+        settings_data = read_yaml_file(settings) or {}
+    except yaml.YAMLError:
+        _LOGGER.warning(
+            "Settings file ({}) does not follow YAML format,"
+            " disregarding".format(settings)
+        )
+        settings_data = {}
+    if not spec:
+        return settings_data
+    if isinstance(
+        spec, str
+    ):  # compute: "partition=standard time='01-00:00:00' cores='32' mem='32000'"
+        spec = spec.split(sep=" ")
+    if isinstance(spec, list):
+        pairs = [(kv, kv.split("=")) for kv in spec]
+        bads = []
+        for orig, pair in pairs:
+            try:
+                k, v = pair
+            except ValueError:
+                bads.append(orig)
+            else:
+                settings_data[k] = v
+        if bads:
+            raise ValueError(
+                "Could not correctly parse itemized compute specification. "
+                "Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
+            )
+    elif isinstance(spec, dict):
+        for key, value in spec.items():
+            settings_data[key] = value
+
+    return settings_data
+
+
+if __name__ == "__main__":
+    main()

looper-2.0.0/looper/command_models/DEVELOPER.md

@@ -0,0 +1,85 @@
+# Developer documentation
+
+## Adding new command models
+
+To add a new model (command) to the project, follow these steps:
+
+1. Add new arguments in `looper/command_models/arguments.py` if necessary.
+
+- Add a new entry for the `ArgumentEnum` class.
+- For example:
+
+```python
+# arguments.py
+
+class ArgumentEnum(enum.Enum):
+    ...
+
+    NEW_ARGUMENT = Argument(
+        name="new_argument",
+        default=(new_argument_type, "default_value"),
+        description="Description of the new argument",
+    )
+
+```
+
+2. Create a new command in the existing command creation logic in `looper/command_models/commands.py`.
+
+- Create a new `Command` instance.
+- Create a `pydantic` model for this new command.
+- Add the new `Command` instance to `SUPPORTED_COMMANDS`.
+- For example:
+
+```python
+NewCommandParser = Command(
+    "new_command",
+    MESSAGE_BY_SUBCOMMAND["new_command"],
+    [
+        ...
+        ArgumentEnum.NEW_ARGUMENT.value,
+        # Add more arguments as needed for the new command
+    ],
+)
+NewCommandParserModel = NewCommandParser.create_model()
+
+SUPPORTED_COMMANDS = [..., NewCommandParser]
+```
+
+3. Update the new argument(s) and command in `TopLevelParser` from `looper/command_models/commands.py`.
+
+- Add a new field for the new command.
+- Add a new field for the new argument(s).
+- For example:
+
+```python
+class TopLevelParser(pydantic.BaseModel):
+
+    # commands
+    ...
+    new_command: Optional[NewCommandParserModel] = pydantic.Field(description=NewCommandParser.description)
+
+    # arguments
+    ...
+    new_argument: Optional[new_argument_type] = ArgumentEnum.NEW_ARGUMENT.value.with_reduced_default()
+```
+
+## Special treatment for the `run` command
+
+The `run` command in our project requires special treatment to accommodate hierarchical namespaces
+and properly handle its unique characteristics. Several functions have been adapted to ensure the
+correct behavior of the run command, and similar adaptations may be necessary for other commands.
+
+For developers looking to understand the details of the special treatment given to the `run`
+command and its associated changes, we recommend to inspect the following functions / part of the
+code:
+- `looper/cli_looper.py`:
+  - `make_hierarchical_if_needed()`
+  - assignment of the `divcfg` variable
+  - assignment of the `project_args` variable
+  - `_proc_resources_spec()`
+  - `validate_post_parse()`
+- `looper/utils.py`:
+  - `enrich_args_via_cfg()`
+
+If you are adding new commands to the project / migrate existing commands to a `pydantic` model-based definition, adapt these parts of the codes with equivalent behavior for your new command.
+Likewise, adapt argument accessions in the corresponding executor in `looper/looper.py` to take into account the hierarchical organization of the command's arguments.

looper-2.0.0/looper/command_models/README.md

@@ -0,0 +1,4 @@
+# `pydantic`-based definitions of `looper` commands and their arguments
+
+With the goal of writing an HTTP API that is in sync with the `looper` CLI, this module defines `looper` commands as `pydantic` models and arguments as fields in there.
+These can then be used by the [`pydantic-argparse`](https://pydantic-argparse.supimdos.com/) library to create a type-validated CLI (see `../cli_pydantic.py`), and by the future HTTP API for validating `POST`ed JSON data. Eventually, the `pydantic-argparse`-based CLI will replace the existing `argparse`-based CLI defined in `../cli_looper.py`.

looper-2.0.0/looper/command_models/__init__.py

@@ -0,0 +1,6 @@
+"""
+This package holds `pydantic` models that describe commands and their arguments.
+
+These can be used either by an HTTP API or with the `pydantic-argparse` 
+library to build a CLI.
+"""