divbase-cli 0.1.0.dev0__py3-none-any.whl

divbase_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0.dev0"

divbase_cli/cli_commands/__init__.py

@@ -0,0 +1,4 @@
+# File: /divbase/divbase/packages/divbase-cli/src/divbase_cli/commands/__init__.py
+"""
+This file initializes the commands subpackage for the divbase-cli package.
+"""

divbase_cli/cli_commands/auth_cli.py

@@ -0,0 +1,92 @@
+"""
+CLI subcommand for managing user auth with DivBase server.
+"""
+
+from datetime import datetime
+from pathlib import Path
+
+import typer
+from pydantic import SecretStr
+from typing_extensions import Annotated
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_config import cli_settings
+from divbase_cli.cli_exceptions import AuthenticationError
+from divbase_cli.user_auth import (
+    check_existing_session,
+    login_to_divbase,
+    logout_of_divbase,
+    make_authenticated_request,
+)
+from divbase_cli.user_config import load_user_config
+
+auth_app = typer.Typer(
+    no_args_is_help=True, help="Login/logout of DivBase server. To register, visit https://divbase.scilifelab.se/."
+)
+
+
+@auth_app.command("login")
+def login(
+    email: str,
+    password: Annotated[str, typer.Option(prompt=True, hide_input=True)],
+    divbase_url: str = typer.Option(cli_settings.DIVBASE_API_URL, help="DivBase server URL to connect to."),
+    config_file: Path = CONFIG_FILE_OPTION,
+    force: bool = typer.Option(False, "--force", "-f", help="Force login again even if already logged in"),
+):
+    """
+    Log in to the DivBase server.
+
+    TODO - think abit more about already logged in validation and UX.
+    One thing to consider would be use case of very close to refresh token expiry, that could be bad UX.
+    (But that is dependent on whether we will allow renewal of refresh tokens...)
+    """
+    secret_password = SecretStr(password)
+    del password  # avoid user passwords showing up in error messages etc...
+
+    config = load_user_config(config_file)
+
+    if not force:
+        session_expires_at = check_existing_session(divbase_url=divbase_url, config=config)
+        if session_expires_at:
+            print(f"Already logged in to {divbase_url}")
+            print(f"Session expires: {datetime.fromtimestamp(session_expires_at)}")
+
+            if not typer.confirm("Do you want to login again? This will replace your current session."):
+                print("Login cancelled.")
+                return
+
+    login_to_divbase(email=email, password=secret_password, divbase_url=divbase_url, config_path=config_file)
+    print(f"Logged in successfully as: {email}")
+
+
+@auth_app.command("logout")
+def logout():
+    """
+    Log out of the DivBase server.
+    """
+    logout_of_divbase()
+    print("Logged out successfully.")
+
+
+@auth_app.command("whoami")
+def whoami(
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Return information about the currently logged-in user.
+    """
+    config = load_user_config(config_file)
+    logged_in_url = config.logged_in_url
+
+    # TODO - move logged in check to the make_authenticated_request function?
+    if not logged_in_url:
+        raise AuthenticationError("You are not logged in. Please log in with 'divbase-cli auth login [EMAIL]'.")
+
+    request = make_authenticated_request(
+        method="GET",
+        divbase_base_url=logged_in_url,
+        api_route="v1/auth/whoami",
+    )
+
+    current_user = request.json()
+    print(f"Currently logged in as: {current_user['email']} (Name: {current_user['name']})")

divbase_cli/cli_commands/dimensions_cli.py

@@ -0,0 +1,143 @@
+import logging
+from pathlib import Path
+
+import typer
+import yaml
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
+from divbase_cli.config_resolver import resolve_project
+from divbase_cli.user_auth import make_authenticated_request
+from divbase_lib.api_schemas.vcf_dimensions import DimensionsShowResult
+
+logger = logging.getLogger(__name__)
+
+
+dimensions_app = typer.Typer(
+    no_args_is_help=True,
+    help="Create and inspect dimensions (number of samples, number of variants, scaffold names) of the VCF files in a project",
+)
+
+
+@dimensions_app.command("update")
+def update_dimensions_index(
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """Calculate and add the dimensions of a VCF file to the dimensions index file in the project."""
+
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    response = make_authenticated_request(
+        method="PUT",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/vcf-dimensions/projects/{project_config.name}",
+    )
+
+    task_id = response.json()
+    print(f"Job submitted successfully with task id: {task_id}")
+
+
+@dimensions_app.command("show")
+def show_dimensions_index(
+    filename: str = typer.Option(
+        None,
+        "--filename",
+        help="If set, will show only the entry for this VCF filename.",
+    ),
+    unique_scaffolds: bool = (
+        typer.Option(
+            False,
+            "--unique-scaffolds",
+            help="If set, will show all unique scaffold names found across all the VCF files in the project.",
+        )
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """
+    Show the dimensions index file for a project.
+    When running --unique-scaffolds, the sorting separates between numeric and non-numeric scaffold names.
+    """
+
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    response = make_authenticated_request(
+        method="GET",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/vcf-dimensions/projects/{project_config.name}",
+    )
+    vcf_dimensions_data = DimensionsShowResult(**response.json())
+
+    dimensions_info = _format_api_response_for_display_in_terminal(vcf_dimensions_data)
+
+    if filename:
+        record = None
+        for entry in dimensions_info.get("indexed_files", []):
+            if entry.get("filename") == filename:
+                record = entry
+                break
+        if record:
+            print(yaml.safe_dump(record, sort_keys=False))
+        else:
+            print(
+                f"No entry found for filename: {filename}. Please check that the filename is correct and that it is a VCF file (extension: .vcf or .vcf.gz)."
+                "\nHint: use 'divbase-cli files list' to view all files in the project."
+            )
+        return
+
+    if unique_scaffolds:
+        unique_scaffold_names = set()
+        for entry in dimensions_info.get("indexed_files", []):
+            unique_scaffold_names.update(entry.get("dimensions", {}).get("scaffolds", []))
+
+        numeric_scaffold_names = []
+        non_numeric_scaffold_names = []
+        for scaffold in unique_scaffold_names:
+            if scaffold.isdigit():
+                numeric_scaffold_names.append(int(scaffold))
+            else:
+                non_numeric_scaffold_names.append(scaffold)
+
+        unique_scaffold_names_sorted = [str(n) for n in sorted(numeric_scaffold_names)] + sorted(
+            non_numeric_scaffold_names
+        )
+
+        print(f"Unique scaffold names found across all the VCF files in the project:\n{unique_scaffold_names_sorted}")
+        return
+
+    print(yaml.safe_dump(dimensions_info, sort_keys=False))
+
+
+def _format_api_response_for_display_in_terminal(api_response: DimensionsShowResult) -> dict:
+    """
+    Convert the API response to a YAML-like format for display in the user's terminal.
+    """
+    dimensions_list = []
+    for entry in api_response.vcf_files:
+        dimensions_entry = {
+            "filename": entry["vcf_file_s3_key"],
+            "file_version_ID_in_bucket": entry["s3_version_id"],
+            "last_updated": entry.get("updated_at"),
+            "dimensions": {
+                "scaffolds": entry.get("scaffolds", []),
+                "sample_count": entry.get("sample_count", 0),
+                "sample_names": entry.get("samples", []),
+                "variants": entry.get("variant_count", 0),
+            },
+        }
+        dimensions_list.append(dimensions_entry)
+
+    skipped_list = []
+    for entry in api_response.skipped_files:
+        skipped_entry = {
+            "filename": entry["vcf_file_s3_key"],
+            "file_version_ID_in_bucket": entry["s3_version_id"],
+            "skip_reason": entry.get("skip_reason", "unknown"),
+        }
+        skipped_list.append(skipped_entry)
+
+    return {
+        "indexed_files": dimensions_list,
+        "skipped_files": skipped_list,
+    }

divbase_cli/cli_commands/file_cli.py

@@ -0,0 +1,245 @@
+"""
+Command line interface for managing files in a DivBase project's store on DivBase.
+
+TODO - support for specifying versions of files when downloading files?
+TODO - Download all files option.
+TODO - skip checked option (aka skip files that already exist in same local dir with correct checksum).
+"""
+
+from pathlib import Path
+
+import typer
+from rich import print
+from typing_extensions import Annotated
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
+from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
+from divbase_cli.services import (
+    download_files_command,
+    list_files_command,
+    soft_delete_objects_command,
+    upload_files_command,
+)
+
+file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
+
+
+@file_app.command("list")
+def list_files(
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    list all files in the project's DivBase store.
+
+    To see files at a user specified project version (controlled by the 'divbase-cli version' subcommand),
+    you can instead use the 'divbase-cli version info [VERSION_NAME]' command.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    files = list_files_command(divbase_base_url=logged_in_url, project_name=project_config.name)
+    if not files:
+        print("No files found in the project's store on DivBase.")
+    else:
+        print(f"Files in the project '{project_config.name}':")
+        for file in files:
+            print(f"- '{file}'")
+
+
+@file_app.command("download")
+def download_files(
+    files: list[str] = typer.Argument(
+        None, help="Space separated list of files/objects to download from the project's store on DivBase."
+    ),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    download_dir: str = typer.Option(
+        None,
+        help="""Directory to download the files to. 
+            If not provided, defaults to what you specified in your user config. 
+            If also not specified in your user config, downloads to the current directory.
+            You can also specify "." to download to the current directory.""",
+    ),
+    disable_verify_checksums: Annotated[
+        bool,
+        typer.Option(
+            "--disable-verify-checksums",
+            help="Turn off checksum verification which is on by default. "
+            "Checksum verification means all downloaded files are verified against their MD5 checksums."
+            "It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
+        ),
+    ] = False,
+    project_version: str = typer.Option(
+        default=None,
+        help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Download files from the project's store on DivBase. This can be done by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a directory to download the files to.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+    download_dir_path = resolve_download_dir(download_dir=download_dir, config_path=config_file)
+
+    if bool(files) + bool(file_list) > 1:
+        print("Please specify only one of --files or --file-list.")
+        raise typer.Exit(1)
+
+    all_files: set[str] = set()
+    if files:
+        all_files.update(files)
+    if file_list:
+        with open(file_list) as f:
+            for object_name in f:
+                all_files.add(object_name.strip())
+
+    if not all_files:
+        print("No files specified for download.")
+        raise typer.Exit(1)
+
+    download_results = download_files_command(
+        divbase_base_url=logged_in_url,
+        project_name=project_config.name,
+        all_files=list(all_files),
+        download_dir=download_dir_path,
+        verify_checksums=not disable_verify_checksums,
+        project_version=project_version,
+    )
+
+    if download_results.successful:
+        print("[green bold]Successfully downloaded the following files:[/green bold]")
+        for success in download_results.successful:
+            print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
+    if download_results.failed:
+        print("[red bold]ERROR: Failed to download the following files:[/red bold]")
+        for failed in download_results.failed:
+            print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
+
+        raise typer.Exit(1)
+
+
+@file_app.command("upload")
+def upload_files(
+    files: list[Path] | None = typer.Argument(None, help="Space seperated list of files to upload."),
+    upload_dir: Path | None = typer.Option(None, "--upload-dir", help="Directory to upload all files from."),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    disable_safe_mode: Annotated[
+        bool,
+        typer.Option(
+            "--disable-safe-mode",
+            help="Turn off safe mode which is on by default. Safe mode adds 2 extra bits of security by first calculating the MD5 checksum of each file that you're about to upload:"
+            "(1) Checks if any of the files you're about to upload already exist (by comparing name and checksum) and if so stops the upload process."
+            "(2) Sends the file's checksum when the file is uploaded so the server can verify the upload was successful (by calculating and comparing the checksums)."
+            "It is recommended to leave safe mode enabled unless you have a specific reason to disable it.",
+        ),
+    ] = False,
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Upload files to your project's store on DivBase by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a directory to upload
+        3. providing a text file with or a file list.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    if bool(files) + bool(upload_dir) + bool(file_list) > 1:
+        print("Please specify only one of --files, --upload_dir, or --file-list.")
+        raise typer.Exit(1)
+
+    all_files = set()
+    if files:
+        all_files.update(files)
+    if upload_dir:
+        all_files.update([p for p in upload_dir.iterdir() if p.is_file()])
+    if file_list:
+        with open(file_list) as f:
+            for line in f:
+                path = Path(line.strip())
+                if path.is_file():
+                    all_files.add(path)
+
+    if not all_files:
+        print("No files specified for upload.")
+        raise typer.Exit(1)
+
+    uploaded_results = upload_files_command(
+        project_name=project_config.name,
+        divbase_base_url=logged_in_url,
+        all_files=list(all_files),
+        safe_mode=not disable_safe_mode,
+    )
+
+    if uploaded_results.successful:
+        print("[green bold] The following files were successfully uploaded: [/green bold]")
+        for object in uploaded_results.successful:
+            print(f"- '{object.object_name}' created from file at: '{object.file_path.resolve()}'")
+
+    if uploaded_results.failed:
+        print("[red bold]ERROR: Failed to upload the following files:[/red bold]")
+        for failed in uploaded_results.failed:
+            print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
+
+        raise typer.Exit(1)
+
+
+@file_app.command("remove")
+def remove_files(
+    files: list[str] | None = typer.Argument(
+        None, help="Space seperated list of files/objects in the project's store on DivBase to delete."
+    ),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="If set, will not actually delete the files, just print what would be deleted."
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Remove files from the project's store on DivBase by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a text file with or a file list.
+
+    'dry_run' mode will not actually delete the files, just print what would be deleted.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    if bool(files) + bool(file_list) > 1:
+        print("Please specify only one of --files or --file-list.")
+        raise typer.Exit(1)
+
+    all_files = set()
+
+    if files:
+        all_files.update(files)
+    if file_list:
+        with open(file_list) as f:
+            for line in f:
+                all_files.add(line.strip())
+
+    if dry_run:
+        print("Dry run mode enabled. The following files would have been deleted:")
+        for file in all_files:
+            print(f"- '{file}'")
+        return
+
+    deleted_files = soft_delete_objects_command(
+        divbase_base_url=logged_in_url,
+        project_name=project_config.name,
+        all_files=list(all_files),
+    )
+
+    if deleted_files:
+        print("Deleted files:")
+        for file in deleted_files:
+            print(f"- '{file}'")
+    else:
+        print("No files were deleted.")

divbase_cli/cli_commands/query_cli.py

@@ -0,0 +1,144 @@
+"""
+Query subcommand for the DivBase CLI.
+
+Submits queries (sample metadata and/or bcftools) to the DivBase API.
+
+If sample metadata query:
+    results are printed to the console.
+
+If bcftools query:
+    a task id is returned which can be used to check the status of the job.
+    After task completed, a merged VCF file will be added to the project's storage bucket which can be downloaded by the user.
+
+
+TODO:
+- Ability to download results file given task id with the file cli?
+-
+"""
+
+import logging
+from pathlib import Path
+
+import typer
+from rich import print
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
+from divbase_cli.cli_config import cli_settings
+from divbase_cli.config_resolver import resolve_project
+from divbase_cli.user_auth import make_authenticated_request
+from divbase_lib.api_schemas.queries import (
+    BcftoolsQueryRequest,
+    SampleMetadataQueryRequest,
+    SampleMetadataQueryTaskResult,
+)
+
+logger = logging.getLogger(__name__)
+
+
+METADATA_TSV_ARGUMENT = typer.Option(
+    cli_settings.METADATA_TSV_NAME, help="Name of the sample metadata TSV file in the project's data store on DivBase."
+)
+
+BCFTOOLS_ARGUMENT = typer.Option(
+    ...,
+    help="""
+        String consisting of the bcftools command to run on the files returned by the tsv query.
+        """,
+)
+
+# In 1 command this is required, other optional, hence only defining the text up here.
+TSV_FILTER_HELP_TEXT = """String consisting of keys:values in the tsv file to filter on.
+    The syntax is 'Key1:Value1,Value2;Key2:Value3,Value4', where the key
+    are the column header names in the tsv, and values are the column values. 
+    Multiple values for a key are separated by commas, and multiple keys are 
+    separated by semicolons. When multple keys are provided, an intersect query 
+    will be performed. E.g. 'Area:West of Ireland,Northern Portugal;Sex:F'.
+    """
+
+
+query_app = typer.Typer(
+    help="Run queries on the VCF files stored in the project's data store on DivBase. Queries are run on the DivBase API",
+    no_args_is_help=True,
+)
+
+
+@query_app.command("tsv")
+def sample_metadata_query(
+    filter: str = typer.Argument(
+        ...,
+        help=TSV_FILTER_HELP_TEXT,
+    ),
+    show_sample_results: bool = typer.Option(
+        default=False,
+        help="Print sample_ID and Filename results from the query.",
+    ),
+    metadata_tsv_name: str = METADATA_TSV_ARGUMENT,
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """
+    Query the tsv sidecar metadata file for the VCF files in the project's data store on DivBase.
+    Returns the sample IDs and filenames that match the query.
+
+    TODO: it perhaps be useful to set the default download_dir in the config so that we can
+    look for files there? For now this code just uses file.parent as the download directory.
+    TODO: handle when the name of the sample column is something other than Sample_ID
+    """
+
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    request_data = SampleMetadataQueryRequest(tsv_filter=filter, metadata_tsv_name=metadata_tsv_name)
+
+    response = make_authenticated_request(
+        method="POST",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/query/sample-metadata/projects/{project_config.name}",
+        json=request_data.model_dump(),
+        timeout=20,  # This is longer than default (5), as api call response is query result, not a task-id.
+    )
+
+    results = SampleMetadataQueryTaskResult(**response.json())
+
+    if show_sample_results:
+        print("[bright_blue]Name and file for each sample in query results:[/bright_blue]")
+        for sample in results.sample_and_filename_subset:
+            print(f"Sample ID: '{sample['Sample_ID']}', Filename: '{sample['Filename']}'")
+
+    print(f"The results for the query ([bright_blue]{results.query_message}[/bright_blue]):")
+    print(f"Unique Sample IDs: {results.unique_sample_ids}")
+    print(f"Unique filenames: {results.unique_filenames}\n")
+
+
+@query_app.command("bcftools-pipe")
+def pipe_query(
+    tsv_filter: str = typer.Option(None, help=TSV_FILTER_HELP_TEXT),
+    command: str = BCFTOOLS_ARGUMENT,
+    metadata_tsv_name: str = METADATA_TSV_ARGUMENT,
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """
+    Submit a query to run on the DivBase API. A single, merged VCF file will be added to the project on success.
+
+    TODO Error handling for subprocess calls.
+    TODO: handle case empty results are returned from tsv_query()
+    TODO what if the user just want to run bcftools on existing files in the bucket, without a tsv file query first?
+    TODO what if a job fails and the user wants to re-run it? do we store temp files?
+    TODO be consistent about input argument and options. when are they optional, how is that indicated in docstring? etc.
+    TODO consider handling the bcftools command whitelist checks also on the CLI level since the error messages are nicer looking?
+    TODO consider moving downloading of missing files elsewhere, since this is now done before the celery task
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    request_data = BcftoolsQueryRequest(tsv_filter=tsv_filter, command=command, metadata_tsv_name=metadata_tsv_name)
+
+    response = make_authenticated_request(
+        method="POST",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/query/bcftools-pipe/projects/{project_config.name}",
+        json=request_data.model_dump(),
+    )
+
+    task_id = response.json()
+    print(f"Job submitted successfully with task id: {task_id}")