divbase-cli 0.1.0a2__tar.gz → 0.1.0a5__tar.gz

{divbase_cli-0.1.0a2 → divbase_cli-0.1.0a5}/.gitignore

@@ -22,6 +22,9 @@ __pycache__/
 tests/fixtures/temp*
 tests/fixtures/merged*
 divbase_metadata_template*.tsv
+# query results files
+result_of_job_*.vcf.gz
+result_of_job_*.log
 
 # query job config files
 bcftools_divbase_job_config.json
@@ -40,6 +43,10 @@ scripts/benchmarking/results
 htmlcov/
 docker/coverage-data/
 
+# log files written by docker compose, if env var set to do so (by default no)
+docker/logs/*
+!docker/logs/.gitkeep
+
 #MacOS artifacts
 .DS_Store
 
@@ -47,6 +54,7 @@ docker/coverage-data/
 .cache/
 # mkdocs auto generated files
 docs/cli/_auto_generated/*.md
+docs/releases/index.md
 
 # pypi 
 dist/

{divbase_cli-0.1.0a2 → divbase_cli-0.1.0a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: divbase-cli
-Version: 0.1.0a2
+Version: 0.1.0a5
 Summary: Command Line Interface for Divbase
 Project-URL: Homepage, https://divbase.scilifelab-2-prod.sys.kth.se
 Project-URL: Documentation, https://scilifelabdatacentre.github.io/divbase
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.12
-Requires-Dist: divbase-lib==0.1.0a2
+Requires-Dist: divbase-lib==0.1.0a5
 Requires-Dist: keyring>=25.7.0
 Requires-Dist: typer==0.24.1
 Description-Content-Type: text/markdown

{divbase_cli-0.1.0a2 → divbase_cli-0.1.0a5}/pyproject.toml

@@ -10,7 +10,7 @@ requires-python = ">=3.12"
 dependencies = [
     "typer==0.24.1",
     "keyring>=25.7.0",
-    "divbase-lib==0.1.0a2",
+    "divbase-lib==0.1.0a5",
 ]
 dynamic = ["version"]
 

divbase_cli-0.1.0a5/src/divbase_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0a5"

divbase_cli-0.1.0a5/src/divbase_cli/cli_commands/auth_cli.py

@@ -0,0 +1,184 @@
+"""
+CLI subcommand for managing user auth with DivBase server.
+"""
+
+import logging
+import time
+from datetime import datetime
+
+import typer
+from pydantic import SecretStr
+from rich import print
+
+from divbase_cli.cli_config import cli_settings
+from divbase_cli.cli_exceptions import DivBaseAPIConnectionError, DivBaseAPIError
+from divbase_cli.config_resolver import resolve_url_for_non_project_specific_commands
+from divbase_cli.services.announcements import get_and_display_announcements
+from divbase_cli.user_auth import (
+    PERSONAL_ACCESS_TOKEN_EXPIRED_MESSAGE,
+    PATData,
+    check_existing_session,
+    delete_stored_pat,
+    load_stored_user_pat,
+    login_to_divbase,
+    logout_of_divbase,
+    make_authenticated_request,
+)
+from divbase_cli.user_config import load_user_config
+from divbase_lib.divbase_constants import PAT_TOKEN_PREFIX
+
+logger = logging.getLogger(__name__)
+
+divbase_home_url = cli_settings.DIVBASE_API_URL.replace("/api", "")
+
+auth_app = typer.Typer(
+    no_args_is_help=True,
+    help=f"Login/logout of DivBase server. To register, visit {divbase_home_url}.",
+)
+
+
+@auth_app.command("login")
+def login(
+    email: str,
+    divbase_url: str = typer.Option(cli_settings.DIVBASE_API_URL, help="DivBase server URL to connect to."),
+    force: bool = typer.Option(False, "--force", "-f", help="Force login again even if already logged in"),
+):
+    """
+    Log in to the DivBase server.
+
+    You'll be prompted for your password after running the command.
+    """
+    password: str = typer.prompt("please enter your password", hide_input=True, confirmation_prompt=False)
+    secret_password = SecretStr(password)
+    del password  # avoid user passwords showing up in error messages etc...
+
+    config = load_user_config()
+
+    if not force:
+        session_expires_at = check_existing_session(divbase_url=divbase_url, config=config)
+        if session_expires_at:
+            print(f"Already logged in to {divbase_url} with email: {config.logged_in_email}.")
+            print(f"Session expires: {datetime.fromtimestamp(session_expires_at)}")
+
+            if not typer.confirm("Do you want to login again? This will replace your current session."):
+                print("Login cancelled.")
+                return
+
+    login_to_divbase(email=email, password=secret_password, divbase_url=divbase_url)
+
+    try:
+        get_and_display_announcements(divbase_base_url=divbase_url)
+    except (DivBaseAPIError, DivBaseAPIConnectionError):
+        # lets not fail the login process if announcements are not working.
+        logger.info("Failed to get announcements after login. Error was: ", exc_info=True)
+
+    print(f"Logged in successfully as: {email}")
+
+
+@auth_app.command("logout")
+def logout():
+    """
+    Log out of the DivBase server.
+
+    Removes your locally stored credentials and invalidates the server-side refresh token.
+    """
+    logout_of_divbase()
+    print("Logged out successfully.")
+
+
+@auth_app.command("add-pat")
+def add_pat(
+    name: str = typer.Argument(..., help="Name of the personal access token (PAT). E.g. 'work-laptop-pat'"),
+    expires_unix_timestamp: int | None = typer.Option(
+        None,
+        "--expires",
+        "-e",
+        help="When the personal access token (PAT) expires (if it does), as a unix timestamp.",
+        min=1,
+    ),
+    overwrite_existing: bool = typer.Option(
+        False, "--overwrite-existing", "-o", help="Overwrite the existing stored PAT if one already exists."
+    ),
+):
+    """
+    Add a personal access token (PAT) to your device for authentication with DivBase.
+
+    PATs are used for authenticating with the DivBase API instead of using your password, and are recommended for use in scripts and pipelines.
+    See https://scilifelabdatacentre.github.io/divbase/user-guides/using-divbase-programmatically/ for more details on how to create a personal access token.
+    You can only store one personal access token at a time.
+    """
+    existing = load_stored_user_pat()
+    if existing and not overwrite_existing:
+        print(
+            f"A personal access token named '{existing.name}' is already stored on your device \n"
+            f"It is due to expire on: {existing.pat_expiry_formatted()} \n"
+            "Append the flag --overwrite-existing (-o) to this command if you want to replace it. \n"
+            "You can only store one PAT at a time."
+        )
+        raise typer.Exit(code=1)
+
+    if expires_unix_timestamp and expires_unix_timestamp < time.time():
+        print("The expiry time you entered is in the past. Please enter a valid expiry time for the PAT.")
+        raise typer.Exit(code=1)
+
+    pat = SecretStr(
+        typer.prompt("please paste your personal access token now", hide_input=True, confirmation_prompt=False)
+    )
+    if not pat.get_secret_value().startswith(PAT_TOKEN_PREFIX):
+        print(
+            "It looks like the token you entered is not a valid personal access token. "
+            f"Please make sure you copied the entire token, including the '{PAT_TOKEN_PREFIX}' prefix."
+        )
+        raise typer.Exit(code=1)
+
+    pat_data = PATData(name=name, pat=pat, pat_expires_at=expires_unix_timestamp)
+    pat_data.dump_pat_data()
+    print(f"Personal access token '{name}' stored successfully.")
+
+
+@auth_app.command("rm-pat")
+def rm_pat():
+    """
+    Remove the stored personal access token from your device.
+
+    If no such token exists the command will still succeed.
+    """
+    delete_stored_pat()
+    print("If a personal access token was stored, it has now been removed.")
+
+
+@auth_app.command("pat-info")
+def pat_info():
+    """
+    Display information about the personal access token stored on this device, if any.
+
+    The token will not be displayed for security reasons, but the name and expiry time will be shown.
+    """
+    pat_data = load_stored_user_pat()
+    if not pat_data:
+        print("No personal access token stored on this device.")
+        return
+
+    print(f"Name: {pat_data.name}")
+    print(f"Expires: {pat_data.pat_expiry_formatted()}")
+
+    if pat_data.is_pat_expired():
+        print("[red bold]Warning [/red bold]")
+        print(PERSONAL_ACCESS_TOKEN_EXPIRED_MESSAGE)
+
+
+@auth_app.command("whoami")
+def whoami():
+    """
+    Return information about the currently logged-in user.
+    """
+    divbase_url = resolve_url_for_non_project_specific_commands()
+
+    request = make_authenticated_request(
+        method="GET",
+        divbase_base_url=divbase_url,
+        api_route="v1/auth/whoami",
+    )
+
+    current_user = request.json()
+    print(f"Currently logged in as: {current_user['email']} (Name: {current_user['name']})")

{divbase_cli-0.1.0a2 → divbase_cli-0.1.0a5}/src/divbase_cli/cli_commands/dimensions_cli.py

@@ -32,7 +32,7 @@ dimensions_app = typer.Typer(
 def update_dimensions_index(
     project: str | None = PROJECT_NAME_OPTION,
 ) -> None:
-    """Calculate and add the dimensions of a VCF file to the dimensions index file in the project."""
+    """Calculate and add the dimensions of all VCF files in the project to the dimensions index in the project."""
 
     project_config = resolve_project(project_name=project)
 
@@ -279,7 +279,7 @@ def create_metadata_template_with_project_samples_names(
     project: str | None = PROJECT_NAME_OPTION,
 ) -> None:
     """
-    Create a template sample metadata file (TSV format) pre-filled with the sample names from the project's VCF files based on the information stored in the project's VCF dimensions cache. Tip: run 'divbase-cli dimensions update' first to ensure that the VCF dimensions areup-to-date.
+    Create a template sample metadata file (TSV format) pre-filled with the sample names from the project's VCF files based on the information stored in the project's VCF dimensions cache. Tip: run 'divbase-cli dimensions update' first to ensure that the VCF dimensions are up-to-date.
     """
     project_config = resolve_project(project_name=project)
     logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)

{divbase_cli-0.1.0a2 → divbase_cli-0.1.0a5}/src/divbase_cli/cli_commands/file_cli.py

@@ -2,6 +2,7 @@
 Command line interface for managing files in a DivBase project's store on DivBase.
 """
 
+from glob import glob
 from pathlib import Path
 from zoneinfo import ZoneInfo
 
@@ -10,8 +11,7 @@ from rich import print
 from rich.table import Table
 from typing_extensions import Annotated
 
-from divbase_cli.cli_commands.shared_args_options import FORMAT_AS_TSV_OPTION, PROJECT_NAME_OPTION
-from divbase_cli.cli_exceptions import UnsupportedFileNameError, UnsupportedFileTypeError
+from divbase_cli.cli_commands.shared_args_options import DOWNLOAD_DIR_OPTION, FORMAT_AS_TSV_OPTION, PROJECT_NAME_OPTION
 from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
 from divbase_cli.services.project_versions import get_version_details_command
 from divbase_cli.services.s3_files import (
@@ -33,22 +33,14 @@ from divbase_lib.utils import format_file_size
 file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
 
 NO_FILES_SPECIFIED_MSG = "No files specified for the command, exiting..."
+NO_UPLOAD_MATCHES_MSG = "Error: The following file paths or glob patterns did not match any existing files:"
 
-DOWNLOAD_DIR_OPTION = typer.Option(
-    None,
-    "--download-dir",
-    "-d",
-    help="""Directory to download the files to. 
-        If not provided, defaults to what you specified in your user config. 
-        If also not specified in your user config, downloads to the current directory.
-        You can also specify "." to download to the current directory.""",
-)
 DISABLE_VERIFY_CHECKSUMS_OPTION = typer.Option(
     False,
     "--disable-verify-checksums",
     "-nc",
     help="Turn off checksum verification which is on by default. "
-    "Checksum verification means all downloaded files are verified against their MD5 checksums."
+    "Checksum verification means all downloaded files are verified against their MD5 checksums. "
     "It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
 )
 PROJECT_VERSION_OPTION = typer.Option(
@@ -89,7 +81,7 @@ def list_files(
     project: str | None = PROJECT_NAME_OPTION,
 ):
     """
-    list all currently available files in the project's DivBase store.
+    List all currently available files in the project's DivBase store.
 
     You can optionally filter the listed files by providing a prefix.
     By default, DivBase query results files are hidden from the listing. Use the --include-results-files option to include them.
@@ -219,7 +211,7 @@ def download_files(
     files: list[str] = typer.Argument(
         None, help="Space separated list of files/objects to download from the project's store on DivBase."
     ),
-    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to download."),
     download_dir: str = DOWNLOAD_DIR_OPTION,
     dry_run: bool = DRY_RUN_OPTION,
     disable_verify_checksums: bool = DISABLE_VERIFY_CHECKSUMS_OPTION,
@@ -235,7 +227,7 @@ def download_files(
 
     To download the latest version of a file, just provide its name. "file1" "file2" etc.
     To download a specific/older version of a file, use the format: "file_name:version_id"
-    You can get a file's version id using the 'divbase-cli file info [FILE_NAME]' command.
+    You can get a file's version id using the 'divbase-cli files info [FILE_NAME]' command.
     You can mix and match latest and specific versions in the same command.
     E.g. to download the latest version of file1 and version "3xcdsdsdiw829x"
     of file2: 'divbase-cli files download file1 file2:3xcdsdsdiw829x'
@@ -385,7 +377,7 @@ def stream_file(
         default=None,
         help="Specify this if you want to look at an older/specific version of the file. "
         "If not provided, the latest version of the file is used. "
-        "To get a file's version ids, use the 'divbase-cli file info [FILE_NAME]' command.",
+        "To get a file's version ids, use the 'divbase-cli files info [FILE_NAME]' command.",
     ),
     project: str | None = PROJECT_NAME_OPTION,
 ):
@@ -412,9 +404,26 @@ def stream_file(
 
 @file_app.command("upload")
 def upload_files(
-    files: list[Path] | None = typer.Argument(None, help="Space separated list of files to upload."),
-    upload_dir: Path | None = typer.Option(None, "--upload-dir", help="Directory to upload all files from."),
-    file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
+    files: list[str] | None = typer.Argument(None, help="Space separated list of files or glob patterns to upload."),
+    file_list: Path | None = typer.Option(None, "--file-list", "-l", help="Text file with list of files to upload."),
+    skip_existing: bool = typer.Option(
+        False,
+        "--skip-existing",
+        "-s",
+        help="If set, will skip already uploaded files, from the files you provided in the command. "
+        "Already uploaded files are determined by checking if a file with the same name and MD5 checksum already exists in the project's store on DivBase. ",
+    ),
+    recursive: bool = typer.Option(
+        False,
+        "--recursive",
+        "-r",
+        "-R",
+        help="If set, recursively include subdirectories contents when uploading (i.e. '**' is expanded). "
+        "Without this flag, patterns only match files in the specified directory.",
+    ),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", "-n", help="If set, will show what files would be uploaded, but not actually upload them."
+    ),
     disable_safe_mode: Annotated[
         bool,
         typer.Option(
@@ -428,56 +437,116 @@ def upload_files(
     project: str | None = PROJECT_NAME_OPTION,
 ):
     """
-    Upload files to your project's store on DivBase:
+    Upload files to your project's store on DivBase.
+    NOTE that directory structure is not preserved on upload, only the file name is used as the object name.
+
+    You can specify files directly as arguments including glob patterns (aka * or **) or use the --file-list option.
+    Use the flag --recursive / -R to expand glob patterns into subdirectories.
+    We recommend wrapping the glob pattern in quotes when using --recursive to ensure the correct behavior across different shells.
 
-    To provide files to upload you can either:
-        1. provide a list of files paths directly in the command line
-        2. provide a directory to upload
-        3. provide a text file with or a file list.
+    Examples:
+        # Upload multiple files by specifying them one after another
+        divbase-cli files upload file1.vcf.gz file2.tsv
+
+        # Upload all .vcf.gz files in the current directory
+        divbase-cli files upload "*.vcf.gz"
+
+        # Upload all files in a directory (non-recursive)
+        divbase-cli files upload "/path/to/data/*"
+
+        # Upload all files in a directory and its subdirectories
+        divbase-cli files upload --recursive "/path/to/data/**"
+
+        # Upload from a text file list (one file path per line)
+        divbase-cli files upload --file-list files_to_upload.txt
     """
     project_config = resolve_project(project_name=project)
     logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
 
-    if bool(files) + bool(upload_dir) + bool(file_list) > 1:
-        print("Please specify only one of --files, --upload_dir, or --file-list.")
+    if bool(files) + bool(file_list) > 1:
+        print("Please specify only space separated files or provide a --file-list.")
+        raise typer.Exit(1)
+
+    if skip_existing and disable_safe_mode:
+        print(
+            "The --skip-existing and --disable-safe-mode options cannot be used together.\n"
+            "Safe mode calculates file checksums, "
+            "and these checksums are needed for --skip-existing to know what files to skip uploading."
+        )
         raise typer.Exit(1)
 
-    all_files: set[Path] = set()
+    # (to preserve order of files provided by user, but ensure no duplicates)
+    all_files: list[Path] = []
+    missing_files_or_patterns: list[str] = []
+    _seen: set[Path] = set()
     if files:
-        all_files.update(files)
-    if upload_dir:
-        all_files.update([p for p in upload_dir.iterdir() if p.is_file()])
+        for pattern in files:
+            matched = [Path(p) for p in glob(pattern, recursive=recursive) if Path(p).is_file()]
+            if not matched:
+                missing_files_or_patterns.append(pattern)
+            for file in matched:
+                if file not in _seen:
+                    _seen.add(file)
+                    all_files.append(file)
+        if missing_files_or_patterns:
+            print(f"[red bold]{NO_UPLOAD_MATCHES_MSG}[/red bold]")
+            for path in missing_files_or_patterns:
+                print(f"[red]- '{path}'[/red]")
+            raise typer.Exit(1)
     if file_list:
+        missing_files = []
         with open(file_list) as f:
             for line in f:
                 path = Path(line.strip())
                 if path.is_file():
-                    all_files.add(path)
+                    if path not in _seen:
+                        _seen.add(path)
+                        all_files.append(path)
+                else:
+                    missing_files.append(path)
+        if missing_files:
+            print(
+                "[red bold]Error: The following file paths provided in the --file-list do not exist or are not files:[/red bold]"
+            )
+            for path in missing_files:
+                print(f"[red]- '{path}'[/red]")
+            raise typer.Exit(1)
 
     if not all_files:
         print(NO_FILES_SPECIFIED_MSG)
         raise typer.Exit(1)
 
+    _check_for_duplicate_file_names(all_files)
     _check_for_unsupported_files(all_files)
 
     uploaded_results = upload_files_command(
         project_name=project_config.name,
         divbase_base_url=logged_in_url,
-        all_files=list(all_files),
+        all_files=all_files,
         safe_mode=not disable_safe_mode,
+        skip_existing=skip_existing,
+        dry_run=dry_run,
     )
 
+    if uploaded_results.skipped:
+        print("[yellow bold]\nThe following files were skipped: [/yellow bold]")
+        for file in uploaded_results.skipped:
+            print(f"[yellow]- '{file.object_name}' reason: {file.reason}[/yellow]")
+
     if uploaded_results.successful:
         print("[green bold]\nThe following files were successfully uploaded: [/green bold]")
-        for object in uploaded_results.successful:
-            print(f"- '{object.object_name}' created from file at: '{object.file_path.resolve()}'")
+        for file in uploaded_results.successful:
+            print(f"[green]- '{file.object_name}' created from file at: '{file.file_path.resolve()}'[/green]")
 
     if uploaded_results.failed:
         print("[red bold]\nERROR: Failed to upload the following files:[/red bold]")
         for failed in uploaded_results.failed:
             print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
-
         raise typer.Exit(1)
+    elif not uploaded_results.successful:
+        print("\nNo files were uploaded.")
+    else:
+        print("\n[green bold]All files uploaded successfully![/green bold]")
 
 
 @file_app.command("rm")
@@ -591,7 +660,32 @@ def _resolve_file_inputs(files: list[str] | None, file_list: Path | None) -> lis
     return list(all_files)
 
 
-def _check_for_unsupported_files(all_files: set[Path]) -> None:
+def _check_for_duplicate_file_names(all_files: list[Path]) -> None:
+    """
+    Helper fn to check if any two files in the upload set share the same file name.
+    Since directory structure is not preserved on upload, duplicate names would overwrite each other.
+    """
+    name_to_paths: dict[str, list[Path]] = {}
+    for f in all_files:
+        if f.name not in name_to_paths:
+            name_to_paths[f.name] = []
+        name_to_paths[f.name].append(f)
+
+    duplicates = {name: paths for name, paths in name_to_paths.items() if len(paths) > 1}
+    if duplicates:
+        print(
+            "[red bold]Error: The following file names appear more than once in the upload list.[/red bold]\n"
+            "[red]Since directory structure is not preserved on upload, these files would collide:[/red]\n"
+        )
+        for name, paths in duplicates.items():
+            print(f"[red bold]'{name}':[/red bold]")
+            for p in paths:
+                print(f"[red]- {p.resolve()}[/red]")
+        print("\nPlease rename the files so each has a unique name before uploading.")
+        raise typer.Exit(1)
+
+
+def _check_for_unsupported_files(all_files: list[Path]) -> None:
     """
     Helper fn to check if any of the files to be uploaded are not supported by DivBase.
     Raises error if so.
@@ -612,10 +706,24 @@ def _check_for_unsupported_files(all_files: set[Path]) -> None:
             unsupported_chars.append(file_path)
 
     if unsupported_file_types:
-        raise UnsupportedFileTypeError(unsupported_files=unsupported_file_types)
+        print(
+            f"[red bold]Error: The following files' types are not supported by DivBase and therefore cannot be uploaded: [/red bold] \n"
+            f"[red]{'\n'.join(str(file) for file in unsupported_file_types)}\n\n[/red]"
+            f"DivBase currently supports the following file types: '{', '.join(SUPPORTED_DIVBASE_FILE_TYPES)}'\n"
+            "Note that while you can upload '.tbi' and '.csi' files they are not used by DivBase in queries, we create our own index files instead. \n"
+            "If you want us to support another file type, please let us know.",
+        )
 
     if unsupported_chars:
-        raise UnsupportedFileNameError(unsupported_files=unsupported_chars)
+        print(
+            f"[red bold]Error: The following file(s) have unsupported characters in their filenames and therefore cannot be uploaded: [/red bold]\n"
+            f"[red]{'\n'.join(str(file) for file in unsupported_chars)}\n\n[/red]"
+            f"Filenames cannot contain any of the following characters: [green]{' '.join(UNSUPPORTED_CHARACTERS_IN_FILENAMES)} [/green]\n"
+            "Please rename the files and try again.",
+        )
+
+    if unsupported_file_types or unsupported_chars:
+        raise typer.Exit(1)
 
 
 def _pretty_print_download_results(download_results):