divbase-cli 0.1.0.dev0__tar.gz

divbase_cli-0.1.0.dev0/.gitignore

@@ -0,0 +1,38 @@
+# repo secrets 
+.env 
+.env*
+
+# python stuff 
+.venv
+.ruff_cache/
+__pycache__/
+*.py[cod]
+*$py.class
+
+.vscode/
+
+# project specific files
+/sample_metadata.tsv
+sample_metadata_*.tsv
+*.vcf
+*.vcf.gz
+*.vcf.gz.csi
+!tests/fixtures/*.vcf.gz
+tests/fixtures/temp*
+tests/fixtures/merged*
+
+# query job config files
+bcftools_divbase_job_config.json
+
+# benchmarking files
+vcf_dimensions.tsv
+mock*.tsv
+task_records*.json
+
+#MacOS artifacts
+.DS_Store
+
+# mkdocs build cache
+.cache/
+# pypi 
+dist/

divbase_cli-0.1.0.dev0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: divbase-cli
+Version: 0.1.0.dev0
+Summary: Command Line Interface for Divbase
+Project-URL: Homepage, https://divbase.scilifelab.se
+Project-URL: Documentation, https://scilifelabdatacentre.github.io/divbase
+Project-URL: Repository, https://github.com/ScilifelabDataCentre/divbase
+Project-URL: Issues, https://github.com/ScilifelabDataCentre/divbase/issues
+Author-email: SciLifeLab Data Centre <datacentre@scilifelab.se>
+License-Expression: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.12
+Requires-Dist: divbase-lib==0.1.0.dev0
+Requires-Dist: httpx<2,>=0.28.1
+Requires-Dist: typer<1,>=0.21.1
+Description-Content-Type: text/markdown
+
+# divbase-cli
+
+Command Line Interface for Divbase. 
+
+## Installation
+
+```bash
+pipx install divbase-cli
+```
+
+## Usage
+
+```bash
+divbase --help # can also run divbase-cli --help
+```
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_cli-0.1.0.dev0/README.md

@@ -0,0 +1,19 @@
+# divbase-cli
+
+Command Line Interface for Divbase. 
+
+## Installation
+
+```bash
+pipx install divbase-cli
+```
+
+## Usage
+
+```bash
+divbase --help # can also run divbase-cli --help
+```
+
+## Development
+
+This package is developed in the [DivBase repository](https://github.com/ScilifelabDataCentre/divbase) by Scilifelab Data Centre.

divbase_cli-0.1.0.dev0/pyproject.toml

@@ -0,0 +1,47 @@
+[project]
+name = "divbase-cli"
+description = "Command Line Interface for Divbase"
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "SciLifeLab Data Centre", email = "datacentre@scilifelab.se" },
+]
+requires-python = ">=3.12"
+dependencies = [
+    "divbase-lib==0.1.0.dev0",
+    "typer>=0.21.1,<1",
+    "httpx>=0.28.1,<2",
+]
+dynamic = ["version"]
+
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+
+[project.urls]
+Homepage = "https://divbase.scilifelab.se"
+Documentation = "https://scilifelabdatacentre.github.io/divbase"
+Repository = "https://github.com/ScilifelabDataCentre/divbase"
+Issues = "https://github.com/ScilifelabDataCentre/divbase/issues"
+
+[project.scripts]
+divbase = "divbase_cli.divbase_cli:main"
+divbase-cli = "divbase_cli.divbase_cli:main"  # TODO choose 1 
+
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/divbase_cli/__init__.py"
+
+[tool.uv.sources]
+divbase-lib = { workspace = true }

divbase_cli-0.1.0.dev0/src/divbase_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0.dev0"

divbase_cli-0.1.0.dev0/src/divbase_cli/cli_commands/__init__.py

@@ -0,0 +1,4 @@
+# File: /divbase/divbase/packages/divbase-cli/src/divbase_cli/commands/__init__.py
+"""
+This file initializes the commands subpackage for the divbase-cli package.
+"""

divbase_cli-0.1.0.dev0/src/divbase_cli/cli_commands/auth_cli.py

@@ -0,0 +1,92 @@
+"""
+CLI subcommand for managing user auth with DivBase server.
+"""
+
+from datetime import datetime
+from pathlib import Path
+
+import typer
+from pydantic import SecretStr
+from typing_extensions import Annotated
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_config import cli_settings
+from divbase_cli.cli_exceptions import AuthenticationError
+from divbase_cli.user_auth import (
+    check_existing_session,
+    login_to_divbase,
+    logout_of_divbase,
+    make_authenticated_request,
+)
+from divbase_cli.user_config import load_user_config
+
+auth_app = typer.Typer(
+    no_args_is_help=True, help="Login/logout of DivBase server. To register, visit https://divbase.scilifelab.se/."
+)
+
+
+@auth_app.command("login")
+def login(
+    email: str,
+    password: Annotated[str, typer.Option(prompt=True, hide_input=True)],
+    divbase_url: str = typer.Option(cli_settings.DIVBASE_API_URL, help="DivBase server URL to connect to."),
+    config_file: Path = CONFIG_FILE_OPTION,
+    force: bool = typer.Option(False, "--force", "-f", help="Force login again even if already logged in"),
+):
+    """
+    Log in to the DivBase server.
+
+    TODO - think abit more about already logged in validation and UX.
+    One thing to consider would be use case of very close to refresh token expiry, that could be bad UX.
+    (But that is dependent on whether we will allow renewal of refresh tokens...)
+    """
+    secret_password = SecretStr(password)
+    del password  # avoid user passwords showing up in error messages etc...
+
+    config = load_user_config(config_file)
+
+    if not force:
+        session_expires_at = check_existing_session(divbase_url=divbase_url, config=config)
+        if session_expires_at:
+            print(f"Already logged in to {divbase_url}")
+            print(f"Session expires: {datetime.fromtimestamp(session_expires_at)}")
+
+            if not typer.confirm("Do you want to login again? This will replace your current session."):
+                print("Login cancelled.")
+                return
+
+    login_to_divbase(email=email, password=secret_password, divbase_url=divbase_url, config_path=config_file)
+    print(f"Logged in successfully as: {email}")
+
+
+@auth_app.command("logout")
+def logout():
+    """
+    Log out of the DivBase server.
+    """
+    logout_of_divbase()
+    print("Logged out successfully.")
+
+
+@auth_app.command("whoami")
+def whoami(
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Return information about the currently logged-in user.
+    """
+    config = load_user_config(config_file)
+    logged_in_url = config.logged_in_url
+
+    # TODO - move logged in check to the make_authenticated_request function?
+    if not logged_in_url:
+        raise AuthenticationError("You are not logged in. Please log in with 'divbase-cli auth login [EMAIL]'.")
+
+    request = make_authenticated_request(
+        method="GET",
+        divbase_base_url=logged_in_url,
+        api_route="v1/auth/whoami",
+    )
+
+    current_user = request.json()
+    print(f"Currently logged in as: {current_user['email']} (Name: {current_user['name']})")

divbase_cli-0.1.0.dev0/src/divbase_cli/cli_commands/dimensions_cli.py

@@ -0,0 +1,143 @@
+import logging
+from pathlib import Path
+
+import typer
+import yaml
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
+from divbase_cli.config_resolver import resolve_project
+from divbase_cli.user_auth import make_authenticated_request
+from divbase_lib.api_schemas.vcf_dimensions import DimensionsShowResult
+
+logger = logging.getLogger(__name__)
+
+
+dimensions_app = typer.Typer(
+    no_args_is_help=True,
+    help="Create and inspect dimensions (number of samples, number of variants, scaffold names) of the VCF files in a project",
+)
+
+
+@dimensions_app.command("update")
+def update_dimensions_index(
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """Calculate and add the dimensions of a VCF file to the dimensions index file in the project."""
+
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    response = make_authenticated_request(
+        method="PUT",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/vcf-dimensions/projects/{project_config.name}",
+    )
+
+    task_id = response.json()
+    print(f"Job submitted successfully with task id: {task_id}")
+
+
+@dimensions_app.command("show")
+def show_dimensions_index(
+    filename: str = typer.Option(
+        None,
+        "--filename",
+        help="If set, will show only the entry for this VCF filename.",
+    ),
+    unique_scaffolds: bool = (
+        typer.Option(
+            False,
+            "--unique-scaffolds",
+            help="If set, will show all unique scaffold names found across all the VCF files in the project.",
+        )
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+) -> None:
+    """
+    Show the dimensions index file for a project.
+    When running --unique-scaffolds, the sorting separates between numeric and non-numeric scaffold names.
+    """
+
+    project_config = resolve_project(project_name=project, config_path=config_file)
+
+    response = make_authenticated_request(
+        method="GET",
+        divbase_base_url=project_config.divbase_url,
+        api_route=f"v1/vcf-dimensions/projects/{project_config.name}",
+    )
+    vcf_dimensions_data = DimensionsShowResult(**response.json())
+
+    dimensions_info = _format_api_response_for_display_in_terminal(vcf_dimensions_data)
+
+    if filename:
+        record = None
+        for entry in dimensions_info.get("indexed_files", []):
+            if entry.get("filename") == filename:
+                record = entry
+                break
+        if record:
+            print(yaml.safe_dump(record, sort_keys=False))
+        else:
+            print(
+                f"No entry found for filename: {filename}. Please check that the filename is correct and that it is a VCF file (extension: .vcf or .vcf.gz)."
+                "\nHint: use 'divbase-cli files list' to view all files in the project."
+            )
+        return
+
+    if unique_scaffolds:
+        unique_scaffold_names = set()
+        for entry in dimensions_info.get("indexed_files", []):
+            unique_scaffold_names.update(entry.get("dimensions", {}).get("scaffolds", []))
+
+        numeric_scaffold_names = []
+        non_numeric_scaffold_names = []
+        for scaffold in unique_scaffold_names:
+            if scaffold.isdigit():
+                numeric_scaffold_names.append(int(scaffold))
+            else:
+                non_numeric_scaffold_names.append(scaffold)
+
+        unique_scaffold_names_sorted = [str(n) for n in sorted(numeric_scaffold_names)] + sorted(
+            non_numeric_scaffold_names
+        )
+
+        print(f"Unique scaffold names found across all the VCF files in the project:\n{unique_scaffold_names_sorted}")
+        return
+
+    print(yaml.safe_dump(dimensions_info, sort_keys=False))
+
+
+def _format_api_response_for_display_in_terminal(api_response: DimensionsShowResult) -> dict:
+    """
+    Convert the API response to a YAML-like format for display in the user's terminal.
+    """
+    dimensions_list = []
+    for entry in api_response.vcf_files:
+        dimensions_entry = {
+            "filename": entry["vcf_file_s3_key"],
+            "file_version_ID_in_bucket": entry["s3_version_id"],
+            "last_updated": entry.get("updated_at"),
+            "dimensions": {
+                "scaffolds": entry.get("scaffolds", []),
+                "sample_count": entry.get("sample_count", 0),
+                "sample_names": entry.get("samples", []),
+                "variants": entry.get("variant_count", 0),
+            },
+        }
+        dimensions_list.append(dimensions_entry)
+
+    skipped_list = []
+    for entry in api_response.skipped_files:
+        skipped_entry = {
+            "filename": entry["vcf_file_s3_key"],
+            "file_version_ID_in_bucket": entry["s3_version_id"],
+            "skip_reason": entry.get("skip_reason", "unknown"),
+        }
+        skipped_list.append(skipped_entry)
+
+    return {
+        "indexed_files": dimensions_list,
+        "skipped_files": skipped_list,
+    }

divbase_cli-0.1.0.dev0/src/divbase_cli/cli_commands/file_cli.py

@@ -0,0 +1,245 @@
+"""
+Command line interface for managing files in a DivBase project's store on DivBase.
+
+TODO - support for specifying versions of files when downloading files?
+TODO - Download all files option.
+TODO - skip checked option (aka skip files that already exist in same local dir with correct checksum).
+"""
+
+from pathlib import Path
+
+import typer
+from rich import print
+from typing_extensions import Annotated
+
+from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
+from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
+from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
+from divbase_cli.services import (
+    download_files_command,
+    list_files_command,
+    soft_delete_objects_command,
+    upload_files_command,
+)
+
+file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
+
+
+@file_app.command("list")
+def list_files(
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    list all files in the project's DivBase store.
+
+    To see files at a user specified project version (controlled by the 'divbase-cli version' subcommand),
+    you can instead use the 'divbase-cli version info [VERSION_NAME]' command.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    files = list_files_command(divbase_base_url=logged_in_url, project_name=project_config.name)
+    if not files:
+        print("No files found in the project's store on DivBase.")
+    else:
+        print(f"Files in the project '{project_config.name}':")
+        for file in files:
+            print(f"- '{file}'")
+
+
+@file_app.command("download")
+def download_files(
+    files: list[str] = typer.Argument(
+        None, help="Space separated list of files/objects to download from the project's store on DivBase."
+    ),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    download_dir: str = typer.Option(
+        None,
+        help="""Directory to download the files to. 
+            If not provided, defaults to what you specified in your user config. 
+            If also not specified in your user config, downloads to the current directory.
+            You can also specify "." to download to the current directory.""",
+    ),
+    disable_verify_checksums: Annotated[
+        bool,
+        typer.Option(
+            "--disable-verify-checksums",
+            help="Turn off checksum verification which is on by default. "
+            "Checksum verification means all downloaded files are verified against their MD5 checksums."
+            "It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
+        ),
+    ] = False,
+    project_version: str = typer.Option(
+        default=None,
+        help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Download files from the project's store on DivBase. This can be done by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a directory to download the files to.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+    download_dir_path = resolve_download_dir(download_dir=download_dir, config_path=config_file)
+
+    if bool(files) + bool(file_list) > 1:
+        print("Please specify only one of --files or --file-list.")
+        raise typer.Exit(1)
+
+    all_files: set[str] = set()
+    if files:
+        all_files.update(files)
+    if file_list:
+        with open(file_list) as f:
+            for object_name in f:
+                all_files.add(object_name.strip())
+
+    if not all_files:
+        print("No files specified for download.")
+        raise typer.Exit(1)
+
+    download_results = download_files_command(
+        divbase_base_url=logged_in_url,
+        project_name=project_config.name,
+        all_files=list(all_files),
+        download_dir=download_dir_path,
+        verify_checksums=not disable_verify_checksums,
+        project_version=project_version,
+    )
+
+    if download_results.successful:
+        print("[green bold]Successfully downloaded the following files:[/green bold]")
+        for success in download_results.successful:
+            print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
+    if download_results.failed:
+        print("[red bold]ERROR: Failed to download the following files:[/red bold]")
+        for failed in download_results.failed:
+            print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
+
+        raise typer.Exit(1)
+
+
+@file_app.command("upload")
+def upload_files(
+    files: list[Path] | None = typer.Argument(None, help="Space seperated list of files to upload."),
+    upload_dir: Path | None = typer.Option(None, "--upload-dir", help="Directory to upload all files from."),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    disable_safe_mode: Annotated[
+        bool,
+        typer.Option(
+            "--disable-safe-mode",
+            help="Turn off safe mode which is on by default. Safe mode adds 2 extra bits of security by first calculating the MD5 checksum of each file that you're about to upload:"
+            "(1) Checks if any of the files you're about to upload already exist (by comparing name and checksum) and if so stops the upload process."
+            "(2) Sends the file's checksum when the file is uploaded so the server can verify the upload was successful (by calculating and comparing the checksums)."
+            "It is recommended to leave safe mode enabled unless you have a specific reason to disable it.",
+        ),
+    ] = False,
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Upload files to your project's store on DivBase by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a directory to upload
+        3. providing a text file with or a file list.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    if bool(files) + bool(upload_dir) + bool(file_list) > 1:
+        print("Please specify only one of --files, --upload_dir, or --file-list.")
+        raise typer.Exit(1)
+
+    all_files = set()
+    if files:
+        all_files.update(files)
+    if upload_dir:
+        all_files.update([p for p in upload_dir.iterdir() if p.is_file()])
+    if file_list:
+        with open(file_list) as f:
+            for line in f:
+                path = Path(line.strip())
+                if path.is_file():
+                    all_files.add(path)
+
+    if not all_files:
+        print("No files specified for upload.")
+        raise typer.Exit(1)
+
+    uploaded_results = upload_files_command(
+        project_name=project_config.name,
+        divbase_base_url=logged_in_url,
+        all_files=list(all_files),
+        safe_mode=not disable_safe_mode,
+    )
+
+    if uploaded_results.successful:
+        print("[green bold] The following files were successfully uploaded: [/green bold]")
+        for object in uploaded_results.successful:
+            print(f"- '{object.object_name}' created from file at: '{object.file_path.resolve()}'")
+
+    if uploaded_results.failed:
+        print("[red bold]ERROR: Failed to upload the following files:[/red bold]")
+        for failed in uploaded_results.failed:
+            print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
+
+        raise typer.Exit(1)
+
+
+@file_app.command("remove")
+def remove_files(
+    files: list[str] | None = typer.Argument(
+        None, help="Space seperated list of files/objects in the project's store on DivBase to delete."
+    ),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="If set, will not actually delete the files, just print what would be deleted."
+    ),
+    project: str | None = PROJECT_NAME_OPTION,
+    config_file: Path = CONFIG_FILE_OPTION,
+):
+    """
+    Remove files from the project's store on DivBase by either:
+        1. providing a list of files paths directly in the command line
+        2. providing a text file with or a file list.
+
+    'dry_run' mode will not actually delete the files, just print what would be deleted.
+    """
+    project_config = resolve_project(project_name=project, config_path=config_file)
+    logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
+
+    if bool(files) + bool(file_list) > 1:
+        print("Please specify only one of --files or --file-list.")
+        raise typer.Exit(1)
+
+    all_files = set()
+
+    if files:
+        all_files.update(files)
+    if file_list:
+        with open(file_list) as f:
+            for line in f:
+                all_files.add(line.strip())
+
+    if dry_run:
+        print("Dry run mode enabled. The following files would have been deleted:")
+        for file in all_files:
+            print(f"- '{file}'")
+        return
+
+    deleted_files = soft_delete_objects_command(
+        divbase_base_url=logged_in_url,
+        project_name=project_config.name,
+        all_files=list(all_files),
+    )
+
+    if deleted_files:
+        print("Deleted files:")
+        for file in deleted_files:
+            print(f"- '{file}'")
+    else:
+        print("No files were deleted.")