housekeeper 4.13.5__py3-none-any.whl

README.md

@@ -0,0 +1,91 @@
+# Housekeeper
+![Housekeeper tests][github-url] [![Coverage Status][coveralls-image]][coveralls-url] [![CodeFactor][codefactor-image]][codefactor-url] [![Code style: black][black-image]][black-url]
+
+### Store, tag, fetch, and archive files with ease 🗃
+
+**Housekeeper** is a tool that aims to provide:
+
+- a backend for storing versioned bundles of files
+- different interfaces (Python, CLI, REST) for fetching files based on tags
+- a way to backup and retrieve bundles from long-term storage
+
+## Installation
+
+Housekeeper written in Python 3.6+ and is available on the [Python Package Index][pypi] (PyPI).
+
+```bash
+poetry install
+```
+
+If you would like to install the latest development version:
+
+```bash
+git clone https://github.com/Clinical-Genomics/housekeeper
+cd housekeeper
+poetry install
+```
+
+## Contributing
+
+Housekeeper is using GitHub flow branching model as described in our [development manual][development manual].
+
+## Documentation
+
+### Command line interface
+
+#### Config file
+
+Housekeeper supports a basic YAML config. The following options are supported:
+
+```yaml
+---
+database: mysql+pymysql://userName:passWord@domain.com/database
+root: /path/to/root/dir
+```
+
+The `root` option is used to store files within the Housekeeper context.
+
+#### Command: `init`
+
+Setup (or reset) the database. It will simply setup all the tables in the database. You can reset an existing database by using the `--reset` option.
+
+```bash
+housekeeper --database "sqlite:///hk.sqlite3" init
+Success! New tables: bundle, file, file_tag_link, tag, version
+```
+
+#### Command: `include`
+
+Include (hard-link) all files of an existing bundle version into Housekeeper and the `root` path.
+
+```bash
+housekeeper myBundle
+```
+
+This will only work if the bundle only has a single version which can be "imported". If you want to import a specific version of a bundle you can use the `--version` option.
+
+#### Command: `delete files`
+
+Delete files that are not on disk anymore like his:
+`housekeeper delete files --tag fastq --notondisk`
+
+Remove all bam files before a certain date:
+`housekeeper delete files --tag bam --before 2017-06-15`
+
+Remove fastq files from a flowcell:
+`housekeeper delete files --tag fastq --tag H0HKKALXX`
+
+It'll always ask for confirmation, unless you add --yes:
+`housekeeper delete files --bundle sillyfish --yes`
+
+If you do not provide a --tag or --bundle, essentially deleting everything, the function will not let you do that.
+
+[pypi]: https://pypi.python.org/pypi/housekeeper/
+[coveralls-url]: https://coveralls.io/r/Clinical-Genomics/housekeeper
+[coveralls-image]: https://img.shields.io/coveralls/Clinical-Genomics/housekeeper.svg?style=flat-square
+[github-url]: https://github.com/Clinical-Genomics/housekeeper/workflows/Housekeeper%20tests/badge.svg
+[development manual]: http://www.clinicalgenomics.se/development/dev/githubflow/
+[codefactor-image]: https://www.codefactor.io/repository/github/clinical-genomics/housekeeper/badge
+[codefactor-url]: https://www.codefactor.io/repository/github/clinical-genomics/housekeeper
+[black-image]: https://img.shields.io/badge/code%20style-black-000000.svg
+[black-url]: https://github.com/psf/black

housekeeper/__init__.py

@@ -0,0 +1,2 @@
+__title__ = "housekeeper"
+__version__ = "4.13.5"

housekeeper/__main__.py

@@ -0,0 +1,16 @@
+"""
+housekeeper.__main__
+
+The main entry point for the command line interface.
+
+Invoke as `housekeeper` (if installed)
+or `python -m housekeeper` (no install required).
+"""
+
+import sys
+
+from housekeeper.cli.core import base
+
+if __name__ == "__main__":
+    # exit using whatever exit code the CLI returned
+    sys.exit(base())

housekeeper/cli/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from .core import base

housekeeper/cli/add.py

@@ -0,0 +1,226 @@
+"""Module for adding via CLI"""
+
+import datetime as dt
+import logging
+from logging import Logger
+from pathlib import Path
+from typing import Generator
+
+import click
+
+from housekeeper.constants import ROOT
+from housekeeper.date import get_date
+from housekeeper.exc import VersionIncludedError
+from housekeeper.files import load_json, validate_input
+from housekeeper.include import include_version, link_to_relative_path, relative_path
+from housekeeper.store.models import Bundle, Tag, Version
+from housekeeper.store.store import Store
+
+LOG: Logger = logging.getLogger(__name__)
+
+
+def validate_args(arg: str, json: str, arg_name: str) -> None:
+    """Check if input is valid
+
+    One of the arguments has to be specified. Both arguments are not allowed
+    """
+    if not (arg or json):
+        LOG.error("Please input json or %s", arg_name)
+        raise click.Abort
+
+    if arg and json:
+        LOG.warning("Can not input both json and %s", arg_name)
+        raise click.Abort
+
+
+@click.group()
+def add():
+    """Add things to the store."""
+
+
+@add.command("bundle")
+@click.argument("bundle_name", required=False)
+@click.option("-j", "--json", help="Input json string")
+@click.option("-e", "--exclude", help="Use to not include bundle when adding it.", is_flag=True)
+@click.pass_context
+def bundle_cmd(context: click.Context, bundle_name: str, json: str, exclude: bool):
+    """Add a new bundle."""
+    LOG.info("Running add bundle")
+    store: Store = context.obj["store"]
+
+    validate_args(arg=bundle_name, json=json, arg_name="bundle_name")
+
+    data: dict = {"name": bundle_name, "created_at": str(dt.datetime.now())}
+    # This is to preserve the behaviour of adding a bundle without providing all information
+    if json:
+        data: dict = load_json(json)
+
+    bundle_name = data["name"]
+    if store.get_bundle_by_name(bundle_name=bundle_name):
+        LOG.warning("bundle name %s already exists", bundle_name)
+        raise click.Abort
+
+    validate_input(data, input_type="bundle")
+    data["created_at"] = get_date(data.get("created_at"))
+
+    if "files" not in data:
+        data["files"] = []
+
+    try:
+        new_bundle, new_version = store.add_bundle(data)
+    except FileNotFoundError as err:
+        LOG.warning("File %s does not exist", err)
+        raise click.Abort
+    store.session.add(new_bundle)
+    new_version.bundle: Bundle = new_bundle
+    store.session.add(new_version)
+    if not exclude:
+        try:
+            context_root_dir: str = context.obj[ROOT]
+            include_version(global_root=context_root_dir, version_obj=new_version)
+        except VersionIncludedError as error:
+            LOG.error(error.message)
+            raise click.Abort
+        new_version.included_at = dt.datetime.now()
+    store.session.commit()
+    LOG.info("new bundle added: %s (%s)", new_bundle.name, new_bundle.id)
+
+
+@add.command("file")
+@click.option("-t", "--tag", "tags", multiple=True, help="tag to associate the file by")
+@click.option("-b", "--bundle-name", help="name of bundle that file should be added to")
+@click.option("-j", "--json", help="json formatted input")
+@click.option(
+    "-kip",
+    "--keep-input-path",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Flag to use the input path in Housekeeper",
+)
+@click.argument("path", required=False)
+@click.pass_context
+def file_cmd(
+    context: click.Context,
+    tags: list[str],
+    bundle_name: str,
+    json: str,
+    keep_input_path: bool,
+    path: str,
+):
+    """Add a file to the latest version of a bundle."""
+    LOG.info("Running add file")
+    store: Store = context.obj["store"]
+    validate_args(arg=path, json=json, arg_name="path")
+
+    data: dict = {}
+    if json:
+        data: dict = load_json(json)
+        validate_input(data, input_type="file")
+
+    file_path: Path = Path(data.get("path", path))
+    if not file_path.exists():
+        LOG.warning("File: %s does not exist", file_path)
+        raise click.Abort
+
+    bundle_name = data.get("bundle", bundle_name)
+    bundle = store.get_bundle_by_name(bundle_name=bundle_name)
+
+    if bundle is None:
+        LOG.warning("unknown bundle: %s", bundle_name)
+        raise click.Abort
+
+    tags = data.get("tags", tags)
+    if not keep_input_path:
+        version: Version = bundle.versions[0]
+        link_to_relative_path(version=version, file_path=file_path, root_path=context.obj[ROOT])
+        file_path: Path = relative_path(version=version, file=file_path)
+    new_file = store.add_file(file_path=file_path, bundle=bundle, tags=tags)
+    store.session.add(new_file)
+    store.session.commit()
+    LOG.info("new file added: %s (%s)", new_file.path, new_file.id)
+
+
+@add.command("version")
+@click.option("-j", "--json", help="Input in json format")
+@click.option("--created-at", help="Date when created")
+@click.argument("bundle_name", required=False)
+@click.pass_context
+def version_cmd(context: click.Context, bundle_name: str, created_at: str, json: str):
+    """Add a new version to a bundle."""
+    LOG.info("Running add version")
+    store: Store = context.obj["store"]
+
+    validate_args(arg=bundle_name, json=json, arg_name="bundle_name")
+
+    data: dict = {"bundle_name": bundle_name, "created_at": created_at}
+    if json:
+        data: dict = load_json(json)
+        bundle_name = data["bundle_name"]
+
+    data["created_at"] = data.get("created_at") or str(dt.datetime.now())
+    validate_input(data, input_type="version")
+
+    bundle = store.get_bundle_by_name(bundle_name=bundle_name)
+    if bundle is None:
+        LOG.warning("unknown bundle: %s", bundle_name)
+        raise click.Abort
+
+    data["created_at"] = get_date(data.get("created_at"))
+    new_version = store.add_version(data, bundle)
+
+    if not new_version:
+        LOG.warning("Seems like version already exists for the bundle")
+        raise click.Abort
+
+    store.session.add(new_version)
+    store.session.commit()
+    LOG.info("new version (%s) added to bundle %s", new_version.id, bundle.name)
+
+
+@add.command("tag")
+@click.argument("tags", nargs=-1)
+@click.option("-f", "--file-id", type=int)
+@click.pass_context
+def tag_cmd(context: click.Context, tags: list[str], file_id: int):
+    """Add tags to housekeeper. Use `--file-id` to add tags to existing file"""
+    LOG.info("Running add tag")
+    store: Store = context.obj["store"]
+    file = None
+    if not tags:
+        LOG.warning("No tags provided")
+        raise click.Abort
+
+    if file_id:
+        file = store.get_file_by_id(file_id=file_id)
+
+        if not file:
+            LOG.warning(f"unable to find file with id: {file_id}")
+            raise click.Abort
+
+    tag_name: str
+    for tag_name in tags:
+        tag: Tag = store.get_tag(tag_name)
+
+        if not tag:
+            LOG.info("%s: tag created", tag_name)
+            tag: Tag = store.new_tag(tag_name)
+            store.session.add(tag)
+            store.session.commit()
+
+        if not file:
+            continue
+
+        if tag in file.tags:
+            LOG.info("%s: tag already added", tag_name)
+            continue
+
+        file.tags.append(tag)
+
+    store.session.commit()
+
+    if not file:
+        return
+
+    all_tags: Generator = (tag.name for tag in file.tags)
+    LOG.info("file tags: %s", ", ".join(all_tags))

housekeeper/cli/core.py

@@ -0,0 +1,62 @@
+"""Module for base CLI"""
+
+import logging
+
+import click
+import coloredlogs
+import yaml
+
+import housekeeper
+from housekeeper.constants import ROOT
+from housekeeper.services.file_service.file_service import FileService
+from housekeeper.services.file_report_service.file_report_service import FileReportService
+from housekeeper.store.database import initialize_database
+from housekeeper.store.store import Store
+
+from . import add, delete, get, include, init
+
+LOG = logging.getLogger(__name__)
+
+
+@click.group()
+@click.option("-c", "--config", type=click.File())
+@click.option("-d", "--database", help="path/URI of the SQL database")
+@click.option("-r", "--root", type=click.Path(exists=True), help="Housekeeper root dir")
+@click.option("-l", "--log-level", default="INFO")
+@click.version_option(housekeeper.__version__, prog_name=housekeeper.__title__)
+@click.pass_context
+def base(
+    context: click.Context,
+    config: click.File,
+    database: str | None,
+    root: str | None,
+    log_level: str,
+):
+    """Housekeeper - Access your files!"""
+    coloredlogs.install(level=log_level)
+    config_values = {}
+    if config:
+        config_values = yaml.full_load(config)
+    context.obj = config_values
+    db_path = database or context.obj.get("database")
+    root_path = context.obj[ROOT] = root or context.obj.get(ROOT)
+    if not db_path:
+        LOG.error("Please point to a database")
+        raise click.Abort
+    if not root_path:
+        LOG.error("Please specify a root dir")
+        raise click.Abort
+    context.obj["database"] = db_path
+    LOG.info("Use root path %s", root_path)
+    initialize_database(db_path)
+    store = Store(root=root_path)
+    context.obj["store"] = store
+    context.obj["file_service"] = FileService(store)
+    context.obj["file_report_service"] = FileReportService()
+
+
+base.add_command(init.init)
+base.add_command(add.add)
+base.add_command(include.include)
+base.add_command(get.get)
+base.add_command(delete.delete)

housekeeper/cli/delete.py

@@ -0,0 +1,243 @@
+"""Module for deleting via CLI"""
+
+import logging
+import shutil
+from pathlib import Path
+
+import click
+
+from housekeeper.date import get_date
+from housekeeper.store.models import File, Tag
+from housekeeper.store.store import Store
+
+LOG = logging.getLogger(__name__)
+
+
+@click.group()
+def delete():
+    """Delete things in the database."""
+
+
+@delete.command("bundle")
+@click.option("-y", "--yes", is_flag=True, help="skip checks")
+@click.argument("bundle_name")
+@click.pass_context
+def bundle_cmd(context, yes, bundle_name):
+    """Delete a empty bundle, that is a bundle without versions."""
+    store: Store = context.obj["store"]
+    bundle = store.get_bundle_by_name(bundle_name=bundle_name)
+    if bundle is None:
+        LOG.warning("bundle %s not found", bundle_name)
+        raise click.Abort
+
+    if bundle.versions:
+        LOG.warning("Can not delete bundle, please remove all versions first")
+        raise click.Abort
+
+    question = f"Remove bundle {bundle.name} from database?"
+    if not (yes or click.confirm(question)):
+        raise click.Abort
+
+    store.session.delete(bundle)
+    store.session.commit()
+    LOG.info(f"Deleted bundle {bundle.name}")
+
+
+@delete.command("version")
+@click.option("-b", "--bundle-name", help="Fetch all versions from a bundle")
+@click.option("-i", "--version-id", type=int, help="Fetch a specific version")
+@click.option("-y", "--yes", is_flag=True, help="skip checks")
+@click.pass_context
+def version_cmd(context, bundle_name, version_id, yes):
+    """Delete a version from database."""
+    store: Store = context.obj["store"]
+    if not (bundle_name or version_id):
+        LOG.info("Please select a bundle or a version")
+        raise click.Abort
+
+    if bundle_name:
+        bundle = store.get_bundle_by_name(bundle_name=bundle_name)
+        if not bundle:
+            LOG.info(f"Could not find bundle {bundle_name}")
+            return
+        if len(bundle.versions) == 0:
+            LOG.warning(f"Could not find versions for bundle {bundle_name}")
+            return
+        LOG.info(f"Deleting the latest version of bundle {bundle_name}")
+        version_obj = bundle.versions[0]
+
+    if version_id:
+        version = store.get_version_by_id(version_id=version_id)
+        if not version:
+            LOG.warning(f"Could not find version {version_id}")
+            raise click.Abort
+        bundle = store.get_bundle_by_id(bundle_id=version.bundle_id)
+        for ver in bundle.versions:
+            if ver.id == version_id:
+                version_obj = ver
+
+    if version_obj.included_at:
+        question = f"Remove bundle version {version_obj.full_path} from file system and database?"
+    else:
+        question = f"Remove bundle version {version_obj.created_at.date()} from database?"
+
+    if not (yes or click.confirm(question)):
+        raise click.Abort
+
+    if version_obj.included_at:
+        shutil.rmtree(version_obj.full_path, ignore_errors=True)
+
+    store.session.delete(version_obj)
+    store.session.commit()
+    LOG.info(f"version deleted: {version_obj.full_path}")
+
+
+@delete.command("files")
+@click.option("--yes", is_flag=True, help="skip checks")
+@click.option("-t", "--tag", multiple=True, help="file tag")
+@click.option("-b", "--bundle-name", help="bundle name")
+@click.option("-a", "--before", help="version created before...")
+@click.option("-n", "--notondisk", is_flag=True, help="rm db entry from files not on disk")
+@click.option("-l", "--list-files", is_flag=True, help="lists files that will be deleted")
+@click.option("-L", "--list-files-verbose", is_flag=True, help="lists additional information")
+@click.pass_context
+def files_cmd(context, yes, tag, bundle_name, before, notondisk, list_files, list_files_verbose):
+    """Delete files based on tags."""
+
+    validate_delete_options(tag=tag, bundle_name=bundle_name)
+    before_date = parse_date(before) if before else None
+    store: Store = context.obj["store"]
+
+    if bundle_name:
+        validate_bundle_exists(store=store, bundle_name=bundle_name)
+
+    files = store.get_files_before(bundle_name=bundle_name, tag_names=tag, before_date=before_date)
+
+    if notondisk:
+        files = store.get_files_not_on_disk(files=files)
+
+    if not files:
+        LOG.warning("No files found")
+        raise click.Abort
+
+    if list_files_verbose:
+        list_files_verbosely(files=files)
+
+    elif list_files:
+        list_files_with_full_path(files=files)
+
+    if not (yes or click.confirm(f"Are you sure you want to delete {len(files)} files?")):
+        raise click.Abort
+
+    for file in files:
+        if file.archive:
+            LOG.warning(
+                f"File {file.path} is archived, please delete it with 'cg archive delete-file' instead. Skipping."
+            )
+            continue
+        if yes or click.confirm(f"Remove file from disk and database: {file.full_path}?"):
+            delete_file(file=file, store=store)
+
+
+@delete.command("tag")
+@click.option("-n", "--name", help="Name of the tag to delete")
+@click.option("-y", "--yes", is_flag=True, help="skip checks")
+@click.pass_context
+def tag_cmd(context, yes, name: str):
+    """Delete a tag from the database.
+    Entries in the file_tag_link table associated with the tag will be removed."""
+    store: Store = context.obj["store"]
+    tag: Tag | None = store.get_tag(tag_name=name)
+    if not tag:
+        LOG.warning(f"Tag {name} not found")
+        raise click.Abort
+
+    tag_file_count: int = len(tag.files) if tag.files else 0
+    question = f"Delete tag {name} with {tag_file_count} associated files?"
+
+    if yes or click.confirm(question):
+        store.session.delete(tag)
+        store.session.commit()
+        LOG.info(f"Tag {name} deleted")
+
+
+def validate_delete_options(tag: str, bundle_name: str):
+    """Validate delete options."""
+    if not (tag or bundle_name):
+        LOG.info("Please specify a bundle or a tag")
+        raise click.Abort
+
+
+def validate_bundle_exists(store: Store, bundle_name: str):
+    """Validate bundle exists."""
+    if not store.get_bundle_by_name(bundle_name=bundle_name):
+        LOG.warning(f"Bundle {bundle_name} not found")
+        raise click.Abort
+
+
+def list_files_verbosely(files):
+    """List files verbosely."""
+    for file in files:
+        tags = ", ".join(tag.name for tag in file.tags)
+        click.echo(
+            f"{click.style(str(file.id), fg='blue')} | {file.full_path} | "
+            f"{click.style(tags, fg='yellow')}"
+        )
+
+
+def list_files_with_full_path(files):
+    for file in files:
+        click.echo(file.full_path)
+
+
+def delete_file(file: File, store: Store):
+    file_path = Path(file.full_path)
+    if file_should_be_unlinked(file):
+        file_path.unlink()
+
+    store.session.delete(file)
+    store.session.commit()
+    LOG.info(f"{file.full_path} deleted")
+
+
+def file_should_be_unlinked(file: File):
+    """Check if file should be unlinked."""
+    file_path = Path(file.full_path)
+    return file.is_included and (file_path.exists() or file_path.is_symlink())
+
+
+def parse_date(date: str):
+    """Attempt to parse date in two different formats."""
+    try:
+        return get_date(date=date)
+    except ValueError:
+        return get_date(date=date, date_format="%Y-%m-%d %H:%M:%S")
+
+
+@delete.command("file")
+@click.option("-y", "--yes", is_flag=True, help="skip checks")
+@click.argument("file_id", type=int)
+@click.pass_context
+def file_cmd(context, yes, file_id):
+    """Delete a file."""
+    store: Store = context.obj["store"]
+    file = store.get_file_by_id(file_id=file_id)
+    if not file:
+        LOG.info("file not found")
+        raise click.Abort
+    if file.archive:
+        LOG.warning("File is archived, please delete it with 'cg archive delete-file' instead")
+        raise click.Abort
+
+    if file.is_included:
+        question = f"Remove file {file.full_path} from file system and database?"
+    else:
+        question = f"Remove file {file.full_path} from database?"
+
+    if yes or click.confirm(question):
+        if file.is_included and Path(file.full_path).exists():
+            Path(file.full_path).unlink()
+
+        store.session.delete(file)
+        store.session.commit()
+        LOG.info("file deleted")