looper 1.7.0a1__py3-none-any.whl → 2.0.0__py3-none-any.whl

looper/command_models/arguments.py

@@ -0,0 +1,293 @@
+"""
+Argument definitions via a thin wrapper around `pydantic.fields.FieldInfo`
+"""
+
+import enum
+import os
+from copy import copy
+from typing import Any, List
+
+import pydantic.v1 as pydantic
+
+
+class Argument(pydantic.fields.FieldInfo):
+    """
+    CLI argument / flag definition
+
+    This class is designed to define CLI arguments or flags. It leverages
+    Pydantic for data validation and serves as a source of truth for multiple
+    interfaces, including a CLI.
+
+    Naively, one would think one could just subclass `pydantic.Field`,
+    but actually `pydantic.Field` is a function, and not a class.
+    `pydantic.Field()` returns a validated `FieldInfo` instance,
+    so we instead subclass `FieldInfo` directly and validate it in the
+    constructor.
+
+    :param str name: argument name, e.g. "ignore-args"
+    :param Any default: a tuple of the form (type, default_value). If the
+        default value is `...` (Ellipsis), then the argument is required.
+    :param str description: argument description, which will appear as the
+        help text for this argument
+    :param dict kwargs: additional keyword arguments supported by
+        `FieldInfo`. These are passed along as they are.
+    """
+
+    def __init__(
+        self, name: str, default: Any, description: str, alias: str = None, **kwargs
+    ) -> None:
+        self._name = name
+        super().__init__(
+            default=default, description=description, alias=alias, **kwargs
+        )
+        self._validate()
+
+    @property
+    def name(self):
+        """
+        Argument name as used in the CLI, e.g. "ignore-args"
+        """
+        return self._name
+
+    def with_reduced_default(self) -> pydantic.fields.FieldInfo:
+        """
+        Convert to a `FieldInfo` instance with reduced default value
+
+        Returns a copy of an instance, but with the `default` attribute
+        replaced by only the default value, without the type information.
+        This is required when using an instance in a direct `pydantic`
+        model definition, instead of creating a model dynamically using
+        `pydantic.create_model`.
+
+        TODO: this is due to this issue:
+        https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
+        and it's a bit tedious.
+
+        """
+        c = copy(self)
+        _, default_value = self.default
+        c.default = default_value
+        return c
+
+
+class ArgumentEnum(enum.Enum):
+    """
+    Lists all available arguments
+
+    Having a single "repository" of arguments allows us to re-use them easily across different commands.
+
+    TODO: not sure whether an enum is the ideal data structure for that
+    """
+
+    IGNORE_FLAGS = Argument(
+        name="ignore_flags",
+        alias="-i",
+        default=(bool, False),
+        description="Ignore run status flags",
+    )
+    FORCE_YES = Argument(
+        name="force_yes",
+        alias="-f",
+        default=(bool, False),
+        description="Provide upfront confirmation of destruction intent, to skip console query. Default=False",
+    )
+
+    DESCRIBE_CODES = Argument(
+        name="describe_codes",
+        default=(bool, False),
+        description="Show status codes description. Default=False",
+    )
+
+    ITEMIZED = Argument(
+        name="itemized",
+        default=(bool, False),
+        description="Show detailed overview of sample statuses. Default=False",
+    )
+
+    FLAGS = Argument(
+        name="flags",
+        alias="-f",
+        default=(List, []),
+        description="Only check samples based on these status flags.",
+    )
+
+    TIME_DELAY = Argument(
+        name="time_delay",
+        alias="-t",
+        default=(int, 0),
+        description="Time delay in seconds between job submissions (min: 0, max: 30)",
+    )
+    DRY_RUN = Argument(
+        name="dry_run",
+        alias="-d",
+        default=(bool, False),
+        description="Don't actually submit jobs",
+    )
+    COMMAND_EXTRA = Argument(
+        name="command_extra",
+        alias="-x",
+        default=(str, ""),
+        description="String to append to every command",
+    )
+    COMMAND_EXTRA_OVERRIDE = Argument(
+        name="command_extra_override",
+        alias="-y",
+        default=(str, ""),
+        description="Same as command-extra, but overrides values in PEP",
+    )
+    LUMP = Argument(
+        name="lump",
+        alias="-u",
+        default=(float, None),
+        description="Total input file size (GB) to batch into one job",
+    )
+    LUMPN = Argument(
+        name="lump_n",
+        alias="-n",
+        default=(int, None),
+        description="Number of commands to batch into one job",
+    )
+    LUMPJ = Argument(
+        name="lump_j",
+        alias="-j",
+        default=(int, None),
+        description="Lump samples into number of jobs.",
+    )
+    LIMIT = Argument(
+        name="limit", alias="-l", default=(int, None), description="Limit to n samples"
+    )
+    SKIP = Argument(
+        name="skip",
+        alias="-k",
+        default=(int, None),
+        description="Skip samples by numerical index",
+    )
+    CONFIG = Argument(
+        name="config",
+        alias="-c",
+        default=(str, None),
+        description="Looper configuration file (YAML)",
+    )
+    SETTINGS = Argument(
+        name="settings",
+        default=(str, ""),
+        description="Path to a YAML settings file with compute settings",
+    )
+    PEP_CONFIG = Argument(
+        name="pep_config",
+        default=(str, None),
+        description="PEP configuration file",
+    )
+    OUTPUT_DIR = Argument(
+        name="output_dir",
+        alias="-o",
+        default=(str, None),
+        description="Output directory",
+    )
+    REPORT_OUTPUT_DIR = Argument(
+        name="report_dir",
+        alias="-r",
+        default=(str, None),
+        description="Set location for looper report and looper table outputs",
+    )
+
+    GENERIC = Argument(
+        name="generic",
+        alias="-g",
+        default=(bool, False),
+        description="Use generic looper config?",
+    )
+
+    SAMPLE_PIPELINE_INTERFACES = Argument(
+        name="sample_pipeline_interfaces",
+        alias="-S",
+        default=(List, []),
+        description="Paths to looper sample pipeline interfaces",
+    )
+    PROJECT_PIPELINE_INTERFACES = Argument(
+        name="project_pipeline_interfaces",
+        alias="-P",
+        default=(List, []),
+        description="Paths to looper project pipeline interfaces",
+    )
+    AMEND = Argument(
+        name="amend", default=(List, []), description="List of amendments to activate"
+    )
+    SEL_ATTR = Argument(
+        name="sel_attr",
+        default=(str, "toggle"),
+        description="Attribute for sample exclusion OR inclusion",
+    )
+    SEL_INCL = Argument(
+        name="sel_incl",
+        default=(List, []),
+        description="Include only samples with these values",
+    )
+    SEL_EXCL = Argument(
+        name="sel_excl",
+        default=(str, ""),
+        description="Exclude samples with these values",
+    )
+    SEL_FLAG = Argument(
+        name="sel_flag", default=(List, []), description="Sample selection flag"
+    )
+    EXC_FLAG = Argument(
+        name="exc_flag", default=(List, []), description="Sample exclusion flag"
+    )
+    SKIP_FILE_CHECKS = Argument(
+        name="skip_file_checks",
+        alias="-f",
+        default=(bool, False),
+        description="Do not perform input file checks",
+    )
+    PACKAGE = Argument(
+        name="package",
+        alias="-p",
+        default=(str, None),
+        description="Name of computing resource package to use",
+    )
+    COMPUTE = Argument(
+        name="compute",
+        default=(List, []),
+        description="List of key-value pairs (k1=v1)",
+    )
+    DIVVY = Argument(
+        name="divvy",
+        default=(str, os.getenv("DIVCFG", None)),
+        description=(
+            "Path to divvy configuration file. Default=$DIVCFG env "
+            "variable. Currently: {}".format(os.getenv("DIVCFG") or "not set")
+        ),
+    )
+    # Arguments for logger compatible with logmuse
+    SILENT = Argument(
+        name="silent", default=(bool, False), description="Whether to silence logging"
+    )
+    VERBOSITY = Argument(
+        name="verbosity",
+        default=(int, None),
+        description="Alternate mode of expression for logging level that better "
+        "accords with intuition about how to convey this.",
+    )
+    LOGDEV = Argument(
+        name="logdev",
+        default=(bool, False),
+        description="Whether to log in development mode; possibly among other "
+        "behavioral changes to logs handling, use a more information-rich "
+        "message format template.",
+    )
+    PIPESTAT = Argument(
+        name="pipestat",
+        default=(str, None),
+        description="Path to pipestat files.",
+    )
+    PORTABLE = Argument(
+        name="portable",
+        default=(bool, False),
+        description="Makes html report portable.",
+    )
+    PROJECT_LEVEL = Argument(
+        name="project",
+        default=(bool, False),
+        description="Is this command executed for project-level?",
+    )

looper/command_models/commands.py

@@ -0,0 +1,335 @@
+"""
+`pydantic` models for `looper` commands and a wrapper class.
+"""
+
+from dataclasses import dataclass
+from typing import List, Optional, Type, Union
+
+import pydantic.v1 as pydantic
+
+from ..const import MESSAGE_BY_SUBCOMMAND
+from .arguments import Argument, ArgumentEnum
+from pydantic_argparse import ArgumentParser
+
+
+@dataclass
+class Command:
+    """
+    Representation of a command
+
+    :param str name: command name
+    :param str description: command description
+    :param list[Argument] arguments: list of arguments supported by this command
+    """
+
+    name: str
+    description: str
+    arguments: List[Argument]
+
+    def create_model(self) -> Type[pydantic.BaseModel]:
+        """
+        Creates a `pydantic` model for this command
+        """
+        arguments = dict()
+        for arg in self.arguments:
+            # These gymnastics are necessary because of
+            # https://github.com/pydantic/pydantic/issues/2248#issuecomment-757448447
+            arg_type, arg_default_value = arg.default
+            arguments[arg.name] = (
+                arg_type,
+                pydantic.Field(arg_default_value, description=arg.description),
+            )
+        return pydantic.create_model(self.name, **arguments)
+
+
+SHARED_ARGUMENTS = [
+    ArgumentEnum.SETTINGS.value,
+    ArgumentEnum.EXC_FLAG.value,
+    ArgumentEnum.SEL_FLAG.value,
+    ArgumentEnum.SEL_ATTR.value,
+    ArgumentEnum.SEL_INCL.value,
+    ArgumentEnum.SEL_EXCL.value,
+    ArgumentEnum.LIMIT.value,
+    ArgumentEnum.SKIP.value,
+    ArgumentEnum.PEP_CONFIG.value,
+    ArgumentEnum.OUTPUT_DIR.value,
+    ArgumentEnum.CONFIG.value,
+    ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+    ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+    ArgumentEnum.PIPESTAT.value,
+    ArgumentEnum.SETTINGS.value,
+    ArgumentEnum.AMEND.value,
+    ArgumentEnum.PROJECT_LEVEL.value,
+]
+
+RunParser = Command(
+    "run",
+    MESSAGE_BY_SUBCOMMAND["run"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LUMPJ.value,
+        ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value,
+    ],
+)
+
+# RERUN
+RerunParser = Command(
+    "rerun",
+    MESSAGE_BY_SUBCOMMAND["rerun"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.LUMPJ.value,
+        ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value,
+    ],
+)
+
+# RUNP
+RunProjectParser = Command(
+    "runp",
+    MESSAGE_BY_SUBCOMMAND["runp"],
+    [
+        ArgumentEnum.IGNORE_FLAGS.value,
+        ArgumentEnum.TIME_DELAY.value,
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.COMMAND_EXTRA.value,
+        ArgumentEnum.COMMAND_EXTRA_OVERRIDE.value,
+        ArgumentEnum.LUMP.value,
+        ArgumentEnum.LUMPN.value,
+        ArgumentEnum.DIVVY.value,
+        ArgumentEnum.SKIP_FILE_CHECKS.value,
+        ArgumentEnum.COMPUTE.value,
+        ArgumentEnum.PACKAGE.value,
+    ],
+)
+
+# TABLE
+TableParser = Command(
+    "table",
+    MESSAGE_BY_SUBCOMMAND["table"],
+    [
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
+    ],
+)
+
+
+# REPORT
+ReportParser = Command(
+    "report",
+    MESSAGE_BY_SUBCOMMAND["report"],
+    [
+        ArgumentEnum.PORTABLE.value,
+        ArgumentEnum.REPORT_OUTPUT_DIR.value,
+    ],
+)
+
+# DESTROY
+DestroyParser = Command(
+    "destroy",
+    MESSAGE_BY_SUBCOMMAND["destroy"],
+    [
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.FORCE_YES.value,
+    ],
+)
+
+# CHECK
+CheckParser = Command(
+    "check",
+    MESSAGE_BY_SUBCOMMAND["check"],
+    [
+        ArgumentEnum.DESCRIBE_CODES.value,
+        ArgumentEnum.ITEMIZED.value,
+        ArgumentEnum.FLAGS.value,
+    ],
+)
+
+# CLEAN
+CleanParser = Command(
+    "clean",
+    MESSAGE_BY_SUBCOMMAND["clean"],
+    [
+        ArgumentEnum.DRY_RUN.value,
+        ArgumentEnum.FORCE_YES.value,
+    ],
+)
+
+# INSPECT
+InspectParser = Command(
+    "inspect",
+    MESSAGE_BY_SUBCOMMAND["inspect"],
+    [],
+)
+
+
+# INIT
+InitParser = Command(
+    "init",
+    MESSAGE_BY_SUBCOMMAND["init"],
+    [
+        # Original command has force flag which is technically a different flag, but we should just use FORCE_YES
+        ArgumentEnum.FORCE_YES.value,
+        ArgumentEnum.OUTPUT_DIR.value,
+        ArgumentEnum.PEP_CONFIG.value,
+        ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
+        ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
+        ArgumentEnum.GENERIC.value,
+    ],
+)
+
+
+# INIT-PIFACE
+InitPifaceParser = Command(
+    "init_piface",
+    MESSAGE_BY_SUBCOMMAND["init-piface"],
+    [],
+)
+
+
+# LINK
+LinkParser = Command(
+    "link",
+    MESSAGE_BY_SUBCOMMAND["link"],
+    [],
+)
+
+
+# Add shared arguments for all commands that use them
+for arg in SHARED_ARGUMENTS:
+    RunParser.arguments.append(arg)
+    RerunParser.arguments.append(arg)
+    RunProjectParser.arguments.append(arg)
+    ReportParser.arguments.append(arg)
+    DestroyParser.arguments.append(arg)
+    CheckParser.arguments.append(arg)
+    CleanParser.arguments.append(arg)
+    TableParser.arguments.append(arg)
+    LinkParser.arguments.append(arg)
+    InspectParser.arguments.append(arg)
+
+# Create all Models
+RunParserModel = RunParser.create_model()
+RerunParserModel = RerunParser.create_model()
+RunProjectParserModel = RunProjectParser.create_model()
+ReportParserModel = ReportParser.create_model()
+DestroyParserModel = DestroyParser.create_model()
+CheckParserModel = CheckParser.create_model()
+CleanParserModel = CleanParser.create_model()
+TableParserModel = TableParser.create_model()
+LinkParserModel = LinkParser.create_model()
+InspectParserModel = InspectParser.create_model()
+InitParserModel = InitParser.create_model()
+InitPifaceParserModel = InitPifaceParser.create_model()
+
+
+def add_short_arguments(
+    parser: ArgumentParser, argument_enums: Type[ArgumentEnum]
+) -> ArgumentParser:
+    """
+    This function takes a parser object created under pydantic argparse and adds the short arguments AFTER the initial creation.
+    This is a workaround as pydantic-argparse does not currently support this during initial parser creation.
+
+    :param ArgumentParser parser: parser before adding short arguments
+    :param Type[ArgumentEnum] argument_enums:  enumeration of arguments that contain names and aliases
+    :return ArgumentParser parser: parser after short arguments have been added
+    """
+
+    for cmd in parser._subcommands.choices.keys():
+
+        for argument_enum in list(argument_enums):
+            # First check there is an alias for the argument otherwise skip
+            if argument_enum.value.alias:
+                short_key = argument_enum.value.alias
+                long_key = "--" + argument_enum.value.name.replace(
+                    "_", "-"
+                )  # We must do this because the ArgumentEnum names are transformed during parser creation
+                if long_key in parser._subcommands.choices[cmd]._option_string_actions:
+                    argument = parser._subcommands.choices[cmd]._option_string_actions[
+                        long_key
+                    ]
+                    argument.option_strings = (short_key, long_key)
+                    parser._subcommands.choices[cmd]._option_string_actions[
+                        short_key
+                    ] = argument
+
+    return parser
+
+
+SUPPORTED_COMMANDS = [
+    RunParser,
+    RerunParser,
+    RunProjectParser,
+    TableParser,
+    ReportParser,
+    DestroyParser,
+    CheckParser,
+    CleanParser,
+    InitParser,
+    InitPifaceParser,
+    LinkParser,
+    InspectParser,
+]
+
+
+class TopLevelParser(pydantic.BaseModel):
+    """
+    Top level parser that takes
+    - commands (run, runp, check...)
+    - arguments that are required no matter the subcommand
+    """
+
+    # commands
+    run: Optional[RunParserModel] = pydantic.Field(description=RunParser.description)
+    rerun: Optional[RerunParserModel] = pydantic.Field(
+        description=RerunParser.description
+    )
+    runp: Optional[RunProjectParserModel] = pydantic.Field(
+        description=RunProjectParser.description
+    )
+    table: Optional[TableParserModel] = pydantic.Field(
+        description=TableParser.description
+    )
+    report: Optional[ReportParserModel] = pydantic.Field(
+        description=ReportParser.description
+    )
+    destroy: Optional[DestroyParserModel] = pydantic.Field(
+        description=DestroyParser.description
+    )
+    check: Optional[CheckParserModel] = pydantic.Field(
+        description=CheckParser.description
+    )
+    clean: Optional[CleanParserModel] = pydantic.Field(
+        description=CleanParser.description
+    )
+    init: Optional[InitParserModel] = pydantic.Field(description=InitParser.description)
+    init_piface: Optional[InitPifaceParserModel] = pydantic.Field(
+        description=InitPifaceParser.description
+    )
+    link: Optional[LinkParserModel] = pydantic.Field(description=LinkParser.description)
+
+    inspect: Optional[InspectParserModel] = pydantic.Field(
+        description=InspectParser.description
+    )
+
+    # Additional arguments for logging, added to ALL commands
+    # These must be used before the command
+    silent: Optional[bool] = ArgumentEnum.SILENT.value.with_reduced_default()
+    verbosity: Optional[int] = ArgumentEnum.VERBOSITY.value.with_reduced_default()
+    logdev: Optional[bool] = ArgumentEnum.LOGDEV.value.with_reduced_default()