dso-core 0.8.2.post2__py3-none-any.whl

dso/__init__.py

@@ -0,0 +1,62 @@
+import logging
+import os
+
+import rich_click as click
+
+from ._logging import log
+from ._metadata import __version__
+from .compile_config import cli as compile_config_cli
+from .create import cli as create_cli
+from .exec import cli as exec_cli
+from .get_config import cli as get_config_cli
+from .init import cli as init_cli
+from .lint import cli as lint_cli
+from .repro import cli as repro_cli
+from .watermark import cli as watermark_cli
+
+click.rich_click.USE_MARKDOWN = True
+
+
+@click.group()
+@click.option(
+    "-q",
+    "--quiet",
+    count=True,
+    help=(
+        "Reduce verbosity. `-q` disables info messages, `-qq` disables warnings. Errors messages cannot be disabled. "
+        "The same can be achieved by setting the env var `DSO_QUIET=1` or `DSO_QUIET=2`, respectively."
+    ),
+    default=int(os.environ.get("DSO_QUIET", 0)),
+)
+@click.option(
+    "-v",
+    "--verbose",
+    help=(
+        "Increase logging verbosity to include debug messages. "
+        "The same can be achieved by setting the env var `DSO_VERBOSE=1`."
+    ),
+    default=bool(int(os.environ.get("DSO_VERBOSE", 0))),
+    is_flag=True,
+)
+@click.version_option(version=__version__, prog_name="dso")
+def cli(quiet: int, verbose: bool):
+    """Root command"""
+    if quiet >= 2:
+        log.setLevel(logging.ERROR)
+        os.environ["DSO_QUIET"] = "2"
+    elif quiet == 1:
+        log.setLevel(logging.WARNING)
+        os.environ["DSO_QUIET"] = "1"
+    elif verbose:
+        log.setLevel(logging.DEBUG)
+        os.environ["DSO_VERBOSE"] = "1"
+
+
+cli.add_command(create_cli)
+cli.add_command(init_cli)
+cli.add_command(compile_config_cli)
+cli.add_command(repro_cli)
+cli.add_command(exec_cli)
+cli.add_command(lint_cli)
+cli.add_command(get_config_cli)
+cli.add_command(watermark_cli)

dso/_logging.py

@@ -0,0 +1,15 @@
+from logging import basicConfig, getLogger
+
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.traceback import install
+
+console_stderr = Console(stderr=True)
+console = Console(stderr=False)
+log = getLogger("dso")
+basicConfig(
+    level="INFO",
+    format="%(message)s",
+    handlers=[RichHandler(markup=True, console=console_stderr, show_path=False, show_time=True)],
+)
+install(show_locals=True)

dso/_metadata.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("dso-core")

dso/_util.py

@@ -0,0 +1,226 @@
+import importlib
+import json
+import subprocess
+import sys
+from collections.abc import Sequence
+from functools import cache
+from importlib import resources
+from importlib.abc import Traversable
+from pathlib import Path
+from typing import Literal
+
+from git.repo import Repo
+from jinja2 import StrictUndefined, Template
+from rich.prompt import Confirm
+
+from dso._logging import console, log
+
+DEFAULT_BRANCH = "master"
+
+
+def check_project_roots(paths: Sequence[Path]) -> Path:
+    """Check project roots for multiple paths and raise an error if they are ambiguous"""
+    try:
+        tmp_project_roots = {get_project_root(p) for p in paths}
+    except FileNotFoundError:
+        log.error("Not in a dso project (no .git directory found)")
+        sys.exit(1)
+    if len(tmp_project_roots) != 1:
+        log.error("Specified paths point to an ambiguous project root.")
+        sys.exit(1)
+    return tmp_project_roots.pop()
+
+
+def _find_in_parent(start_directory: Path, file_or_folder: str, recurse_barrier: Path | None = None) -> Path | None:
+    """
+    Recursively walk up to the folder directory until we either find `file_or_folder` or reach the root.
+
+    If recurse_barrier is specified, we don't recurse past this level.
+
+    By using @cache this is efficient, even when called repeatedly.
+
+    If the root is reached without finding the file, None is returned.
+    """
+    return _find_in_parent_abs(
+        start_directory.absolute(), file_or_folder, recurse_barrier.absolute() if recurse_barrier is not None else None
+    )
+
+
+@cache
+def _find_in_parent_abs(start_directory: Path, file_or_folder: str, recurse_barrier: Path | None = None) -> Path | None:
+    """
+    Implementation of `_find_in_parent`, work only with absolute paths here.
+
+    This is to ensure @cache doesn't lead to wrong results when calling this from different working directories.
+    """
+    if start_directory == Path("/"):
+        return None
+    if recurse_barrier is not None:
+        if not start_directory.is_relative_to(recurse_barrier):
+            return None
+    if start_directory.is_file():
+        return _find_in_parent_abs(start_directory.parent, file_or_folder, recurse_barrier)
+    if (start_directory / file_or_folder).exists():
+        return start_directory / file_or_folder
+    else:
+        return _find_in_parent_abs(start_directory.parent, file_or_folder, recurse_barrier)
+
+
+def get_project_root(start_directory: Path) -> Path:
+    """
+    Find the dso project root.
+
+    This is defined as the next parent directory that contains a `.git` directory.
+
+    Parameters
+    ----------
+    start_directory : Path
+        The directory to start the search from.
+
+    Returns
+    -------
+    The project root
+
+    Raises
+    ------
+    FileNotFoundError
+        If the .git folder is not found.
+    """
+    proj_root = _find_in_parent(start_directory, ".git")
+    if proj_root is None:
+        raise FileNotFoundError("Not within a dso project (No .git directory found)")
+    else:
+        # .parent, because proj_root points to the git directory
+        return proj_root.parent
+
+
+def _get_template_path(template_type: Literal["init", "folder", "stage"], template_name: str) -> Traversable:
+    template_module = importlib.import_module(f"dso.templates.{template_type}")
+    return resources.files(template_module) / template_name
+
+
+def _copy_with_render(source: Traversable, destination: Path, params: dict):
+    """Fill all placeholders in a file with jinja2 and save file to destination"""
+    with source.open() as f:
+        template = Template(f.read(), undefined=StrictUndefined)
+    rendered_content = template.render(params)
+    with destination.open("w") as file:
+        file.write(rendered_content)
+        # Non-empty files should have a terminal new-line to make the pre-commit hooks happy
+        if len(rendered_content):
+            file.write("\n")
+
+
+def _instantiate_template(template_path: Traversable, target_dir: Path | str, **params) -> None:
+    """Copy a template folder to a target directory, filling all placeholder values."""
+    target_dir = Path(target_dir)
+
+    def _traverse_template(curr_path, subdir):
+        for p in curr_path.iterdir():
+            if p.is_file():
+                name_rendered = Template(p.name).render(params)
+                target_file = target_dir / subdir / name_rendered
+                if not target_file.exists():
+                    _copy_with_render(p, target_file, params)
+            else:
+                (target_dir / subdir / p.name).mkdir(exist_ok=True)
+                _traverse_template(p, subdir / p.name)
+
+    _traverse_template(template_path, Path("."))
+
+
+def _instantiate_with_repo(template: Traversable, target_dir: Path | str, **params) -> None:
+    """Create a git repo in a directory and render a template inside.
+
+    Creates an initial commit.
+    """
+    target_dir = Path(target_dir)
+    target_dir.mkdir(exist_ok=True)
+    log.info("Created project directory.")
+
+    _instantiate_template(template, target_dir, **params)
+    log.info("Created folder structure from template.")
+
+    if not (target_dir / ".git").exists():
+        Repo.init(target_dir)
+        repo = Repo(target_dir)
+        # set main as default branch
+        repo.git.checkout("-b", DEFAULT_BRANCH)
+        repo.git.symbolic_ref("HEAD", f"refs/heads/{DEFAULT_BRANCH}")
+        repo.git.add(A=True)
+        repo.index.commit("Initial commit generated by dso CLI tool.")
+        log.info("Initalized local git repo.")
+
+
+def _git_list_files(dir: Path) -> list[Path]:
+    """
+    Recursively list all files in `dir` that are not .gitignored.
+
+    This lists both files that are tracked and untracked by git.
+    Source: https://stackoverflow.com/a/77197460/2340703
+    """
+    res = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard"], cwd=dir, capture_output=True
+    )
+    if res.returncode:
+        sys.exit(res.returncode)
+    return [dir / Path(p) for p in res.stdout.decode("utf-8").strip().split("\n")]
+
+
+def _read_dot_dso_json(dir: Path):
+    """
+    Read .dso.json from the project directory
+
+    The `.dso.json` file is file that can store project-specific settings and data generated by the dso CLI.
+    It is not intended to be edited by the user.
+
+    If the file doesn not exist it just means that is has not been generated yet. We return an empty dict in such a case.
+    """
+    project_root = get_project_root(dir)
+    dot_dso_json = project_root / ".dso.json"
+    if (dot_dso_json).exists():
+        with dot_dso_json.open("rb") as f:
+            return json.load(f)
+    else:
+        return {}
+
+
+def _update_dot_dso_json(dir: Path, update_dict: dict):
+    """
+    Update the .dso.json with `update_dict`.
+
+    Only keys present in `update_dict` will be updated. All other keys will be left as they are.
+    """
+    project_root = get_project_root(dir)
+    dot_dso_json = project_root / ".dso.json"
+    config = _read_dot_dso_json(dir)
+    config.update(update_dict)
+
+    with dot_dso_json.open("w") as f:
+        json.dump(config, f)
+
+
+def check_ask_pre_commit(dir: Path):
+    """
+    Check if pre-commit hooks are installed and asks to install them
+
+    If the user declines, info will be written to `.dso.json` to not ask again in the future.
+    """
+    config = _read_dot_dso_json(dir)
+    if config.get("check_ask_pre_commit", True):
+        project_root = get_project_root(dir)
+        hook = project_root / ".git" / "hooks" / "pre-commit"
+        if not hook.is_file() or "pre-commit" not in hook.read_text():
+            console.print("Pre-commit hooks are not installed in this project.")
+            console.print(
+                "This hooks will take care of running consistency checks and automatically syncing [bold]dvc."
+            )
+            if Confirm.ask("[bold]Do you want to install the pre-commit hooks now?"):
+                res = subprocess.run(["pre-commit", "install"], cwd=project_root)
+                if res.returncode:
+                    log.error("Failed to install pre-commit hooks")
+                    sys.exit(res.returncode)
+                else:
+                    log.info("Pre-commit hooks installed successfully")
+            else:
+                _update_dot_dso_json(dir, {"check_ask_pre_commit": False})

dso/compile_config.py

@@ -0,0 +1,181 @@
+import filecmp
+import os.path
+import shutil
+import tempfile
+from collections.abc import Collection, Sequence
+from functools import partial
+from io import TextIOWrapper
+from pathlib import Path
+from textwrap import dedent
+
+import rich_click as click
+from ruamel.yaml import YAML, yaml_object
+
+from dso import hiyapyco
+
+from ._logging import log
+from ._util import _find_in_parent, check_project_roots
+
+PARAMS_YAML_DISCLAIMER = dedent(
+    """\
+    ################################# !!! WARNING !!! #############################################
+    #                                                                                             #
+    # params.yaml is AUTOMATICALLY GENERATED. DO NOT EDIT BY HAND or your changes might be lost.  #
+    #                                                                                             #
+    # Instead, edit `params.in.yaml` and compile the changes using `dso compile-config`.          #
+    # If you do not wish to use this feature, simply delete `params.in.yaml`                      #
+    # and remove this notice                                                                      #
+    ###############################################################################################
+    """
+)
+
+
+def _load_yaml_with_auto_adjusting_paths(yaml_stream: TextIOWrapper, destination: Path):
+    """
+    Load a yaml file and adjust paths for all !path objects based on a destination file
+
+    Parameters
+    ----------
+    yaml_path
+        Path of the yaml file to load
+    destination
+        Path to which the file shall be adjusted
+
+    Returns
+    -------
+    The output of ruamel.yaml.YAML.load_all after registering a class for !path
+    """
+    ruamel = YAML()
+
+    # The folder of the source config file
+    source = Path(yaml_stream.name).parent
+    if not destination.is_relative_to(source):
+        raise ValueError("Destination path can be the same as source, or a child thereof.")
+
+    @yaml_object(ruamel)
+    class AutoAdjustingPathWithLocation:
+        yaml_tag = "!path"
+
+        def __init__(self, path: str):
+            self.path = Path(path)
+            if not (source / self.path).exists():
+                # Warn, but do not fail (it could also be an output path to be populated by a dvc stage)
+                log.warning(f"Path {self.path} does not exist!")
+
+        def get_adjusted(self):
+            # not possible with pathlib, because pathlib requires the paths to be subpaths of each other
+            diff = Path(os.path.relpath(source / self.path, destination))
+            return diff
+
+        @classmethod
+        def to_yaml(cls, representer, node):
+            return representer.represent_str(str(node.get_adjusted()))
+
+        @classmethod
+        def from_yaml(cls, constructor, node):
+            return cls(node.value)
+
+    return ruamel.load_all(yaml_stream)
+
+
+def _get_list_of_configs_to_compile(paths: Sequence[Path], project_root: Path):
+    """Find all files named params.in.yaml in `dir` and all subdirectories"""
+    # Get all configs that are children of the current working directory
+    all_configs = {x for dir in paths for x in dir.glob("**/params.in.yaml")}
+    for c in all_configs:
+        assert c.is_relative_to(project_root), "Config file not relative to project root"
+
+    # Now we still need to find all config.in.yaml files in any parent directory (to enable the hierarchical compilation).
+    # We can start with the input paths, as they are per definition a parent of all config files found.
+    for tmp_path in paths:
+        # Check each parent directory if it contains a "params.in.yaml" - If yes, add it to the list of all configs.
+        # We don't need to re-check the parents of added items, because their parent is per definition also a parent
+        # of a config that was already part of the list.
+        while (tmp_path := _find_in_parent(tmp_path.parent, "params.in.yaml", project_root)) is not None:
+            all_configs.add(tmp_path)
+            # we don't want to find the current config again, therefore .parent
+            tmp_path = tmp_path.parent
+
+    return all_configs
+
+
+def _get_parent_configs(current_config: Path, all_configs: Collection[Path]) -> list[Path]:
+    """For a particular config file, find all config files that are parent to it in a list of config files.
+
+    The files are sorted from parent to child. The current_config is always the last item in the list.
+    """
+    parent_configs = []
+    for tmp_cfg in all_configs:
+        if current_config.is_relative_to(tmp_cfg.parent):
+            parent_configs.append(tmp_cfg)
+
+    # sort from parent to child (based on the number of path parts)
+    return sorted(parent_configs, key=lambda x: len(x.parts))
+
+
+def compile_all_configs(paths: Sequence[Path]):
+    """Compile params.in.yaml into params.yaml using Jinja2 templating and resolving recursive templates.
+
+    paths:
+        One or multiple locations within the project. Can be files or directories -- instead of files, their
+        parent directory will be used. Will compile all params.in.yaml files in child directories
+        and the respective parent config files.
+    """
+    # If files are specified, use the respective parent dir
+    paths = [p.parent.resolve() if p.is_file() else p.resolve() for p in paths]
+
+    project_root = check_project_roots(paths)
+    log.info(f"Detected {project_root} as project root.")
+
+    all_configs = _get_list_of_configs_to_compile(paths, project_root)
+    log.info(f"Compiling a total of {len(all_configs)} config files.")
+
+    for config in all_configs:
+        # sorted sorts path from parent to child. This is what we want as hyapyco gives precedence to configs
+        # later in the list.
+        configs_to_merge = _get_parent_configs(config, all_configs)
+        conf = hiyapyco.load(
+            *[str(x) for x in configs_to_merge],
+            method=hiyapyco.METHOD_MERGE,
+            interpolate=True,
+            loader_callback=partial(_load_yaml_with_auto_adjusting_paths, destination=config.parent),
+        )
+        # an empty configuration should actually be an empty dictionary.
+        if conf is None:
+            conf = {}
+        # write config to "params.yaml" in same directory
+        out_file = config.parent / "params.yaml"
+
+        # Write to temporary file first and compare to previous params.yaml
+        # Only ask for confirmation, overwrite, and show log if they are different
+        with tempfile.NamedTemporaryFile() as tmpfile:
+            # dump to tempfile
+            with open(tmpfile.name, "w") as f:
+                f.write(PARAMS_YAML_DISCLAIMER)
+                f.write("\n")
+                ruamel = YAML()
+                ruamel.dump(conf, f)
+            # check for equivalience
+            if not out_file.exists() or not filecmp.cmp(f.name, out_file, shallow=False):
+                shutil.copy(tmpfile.name, out_file)
+                log.debug(f"Compiled ./{config.relative_to(project_root)} to {out_file.name}")
+            else:
+                log.debug(f"./{config.relative_to(project_root)} [green]is already up-to-date!")
+
+    log.info("[green]Configuration compiled successfully.")
+
+
+@click.command(name="compile-config")
+@click.argument("args", nargs=-1)
+def cli(args):
+    """Compile params.in.yaml into params.yaml using Jinja2 templating and resolving recursive templates.
+
+    If passing no arguments, configs will be resolved for the current working directory (i.e. all parent configs,
+    and all configs in child directories). Alternatively a list of paths can be specified. In that case, all configs
+    related to these paths will be compiled (useful for using with pre-commit).
+    """
+    if not len(args):
+        paths = [Path.cwd()]
+    else:
+        paths = [Path(x) for x in args]
+    compile_all_configs(paths)

dso/create.py

@@ -0,0 +1,108 @@
+"""Creates a project folder structure"""
+
+import sys
+from os import getcwd
+from pathlib import Path
+from textwrap import dedent, indent
+
+import questionary
+import rich_click as click
+from rich.prompt import Confirm, Prompt
+
+from dso._logging import log
+from dso._util import _get_template_path, _instantiate_template, get_project_root
+from dso.compile_config import compile_all_configs
+
+DEFAULT_BRANCH = "master"
+# list of stage template with description - can be later populated also from external directories
+STAGE_TEMPLATES = {
+    "bash": "Execute a simple bash snippet or call an external script (e.g. nextflow)",
+    "quarto": "Generate a report using quarto",
+}
+# Create help text for CLI listing all templates
+STAGE_TEMPLATE_TEXT = "\n".join(f" * __{name}__: {description}" for name, description in STAGE_TEMPLATES.items())
+CREATE_STAGE_HELP_TEXT = dedent(
+    f"""\
+    Create a new stage.
+
+    A stage can be in any subfolder of the projects. Stages shall not be nested.
+
+    Available templates: \n{indent(STAGE_TEMPLATE_TEXT, " " * 6)}
+    """
+)
+
+
+@click.option("--description")
+@click.option("--template", type=click.Choice(list(STAGE_TEMPLATES)))
+@click.argument("name", required=False)
+@click.command("stage", help=CREATE_STAGE_HELP_TEXT)
+def create_stage(name: str | None = None, template: str | None = None, description: str | None = None):
+    """Create a new stage."""
+    if template is None:
+        template = str(questionary.select("Choose a template:", choices=list(STAGE_TEMPLATES)).ask())
+
+    if name is None:
+        name = Prompt.ask('[bold]Please enter the name of the stage, e.g. "01_preprocessing"')
+
+    if description is None:
+        description = Prompt.ask("[bold]Please add a short description of the stage")
+
+    target_dir = Path(getcwd()) / name
+    if target_dir.exists():
+        log.error(f"[red]Couldn't create stage: Folder with name {target_dir} already exists!")
+        sys.exit(1)
+    target_dir.mkdir()
+
+    # stage dir, relative to project root
+    project_root = get_project_root(target_dir)
+    stage_path = target_dir.relative_to(project_root)
+
+    _instantiate_template(
+        _get_template_path("stage", template),
+        target_dir,
+        stage_name=name,
+        stage_description=description,
+        stage_path=stage_path,
+    )
+    log.info("[green]Stage created successfully.")
+    compile_all_configs([target_dir])
+
+
+@click.argument("name", required=False)
+@click.command("folder")
+def create_folder(name: str | None = None):
+    """Create a new folder. A folder can contain subfolders or stages.
+
+    Technically, nothing prevents you from just using `mkdir`. This command additionally adds some default
+    files that might be useful, e.g. an empty `dvc.yaml`.
+
+    You can specify a path to an existing folder. In that case all template files that do not exist will
+    be copied to the folder. Existing files will never be overwritten.
+    """
+    # currently there's only one template for folders
+    template = "default"
+
+    if name is None:
+        name = Prompt.ask('[bold]Please enter the name of the folder, e.g. "RNAseq"')
+
+    target_dir = Path(getcwd()) / name
+
+    if target_dir.exists():
+        if not Confirm.ask("[bold]Directory already exists. Do you want to copy template files to existing folder?"):
+            sys.exit(1)
+
+    target_dir.mkdir(exist_ok=True)
+
+    _instantiate_template(_get_template_path("folder", template), target_dir, stage_name=name)
+    log.info("[green]Folder created successfully.")
+    compile_all_configs([target_dir])
+
+
+@click.group(name="create")
+def cli():
+    """Create stage folder structure subcommand."""
+    pass
+
+
+cli.add_command(create_stage)
+cli.add_command(create_folder)