divbase-cli 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- divbase_cli/__init__.py +1 -1
- divbase_cli/cli_commands/auth_cli.py +4 -9
- divbase_cli/cli_commands/dimensions_cli.py +4 -8
- divbase_cli/cli_commands/file_cli.py +284 -70
- divbase_cli/cli_commands/query_cli.py +3 -7
- divbase_cli/cli_commands/shared_args_options.py +20 -0
- divbase_cli/cli_commands/task_history_cli.py +3 -8
- divbase_cli/cli_commands/user_config_cli.py +14 -44
- divbase_cli/cli_commands/version_cli.py +16 -24
- divbase_cli/cli_config.py +18 -7
- divbase_cli/cli_exceptions.py +37 -22
- divbase_cli/config_resolver.py +10 -10
- divbase_cli/divbase_cli.py +1 -1
- divbase_cli/retries.py +34 -0
- divbase_cli/services/__init__.py +0 -0
- divbase_cli/services/pre_signed_urls.py +446 -0
- divbase_cli/services/project_versions.py +77 -0
- divbase_cli/services/s3_files.py +355 -0
- divbase_cli/user_auth.py +26 -13
- divbase_cli/user_config.py +20 -9
- divbase_cli/utils.py +47 -0
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev3.dist-info}/METADATA +4 -3
- divbase_cli-0.1.0.dev3.dist-info/RECORD +27 -0
- divbase_cli/pre_signed_urls.py +0 -169
- divbase_cli/services.py +0 -219
- divbase_cli-0.1.0.dev2.dist-info/RECORD +0 -22
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev3.dist-info}/WHEEL +0 -0
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev3.dist-info}/entry_points.txt +0 -0
divbase_cli/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.1.0.
|
|
1
|
+
__version__ = "0.1.0.dev3"
|
|
@@ -3,13 +3,11 @@ CLI subcommand for managing user auth with DivBase server.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
from datetime import datetime
|
|
6
|
-
from pathlib import Path
|
|
7
6
|
|
|
8
7
|
import typer
|
|
9
8
|
from pydantic import SecretStr
|
|
10
9
|
from typing_extensions import Annotated
|
|
11
10
|
|
|
12
|
-
from divbase_cli.cli_commands.user_config_cli import CONFIG_FILE_OPTION
|
|
13
11
|
from divbase_cli.cli_config import cli_settings
|
|
14
12
|
from divbase_cli.cli_exceptions import AuthenticationError
|
|
15
13
|
from divbase_cli.user_auth import (
|
|
@@ -30,7 +28,6 @@ def login(
|
|
|
30
28
|
email: str,
|
|
31
29
|
password: Annotated[str, typer.Option(prompt=True, hide_input=True)],
|
|
32
30
|
divbase_url: str = typer.Option(cli_settings.DIVBASE_API_URL, help="DivBase server URL to connect to."),
|
|
33
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
34
31
|
force: bool = typer.Option(False, "--force", "-f", help="Force login again even if already logged in"),
|
|
35
32
|
):
|
|
36
33
|
"""
|
|
@@ -43,7 +40,7 @@ def login(
|
|
|
43
40
|
secret_password = SecretStr(password)
|
|
44
41
|
del password # avoid user passwords showing up in error messages etc...
|
|
45
42
|
|
|
46
|
-
config = load_user_config(
|
|
43
|
+
config = load_user_config()
|
|
47
44
|
|
|
48
45
|
if not force:
|
|
49
46
|
session_expires_at = check_existing_session(divbase_url=divbase_url, config=config)
|
|
@@ -55,7 +52,7 @@ def login(
|
|
|
55
52
|
print("Login cancelled.")
|
|
56
53
|
return
|
|
57
54
|
|
|
58
|
-
login_to_divbase(email=email, password=secret_password, divbase_url=divbase_url
|
|
55
|
+
login_to_divbase(email=email, password=secret_password, divbase_url=divbase_url)
|
|
59
56
|
print(f"Logged in successfully as: {email}")
|
|
60
57
|
|
|
61
58
|
|
|
@@ -69,13 +66,11 @@ def logout():
|
|
|
69
66
|
|
|
70
67
|
|
|
71
68
|
@auth_app.command("whoami")
|
|
72
|
-
def whoami(
|
|
73
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
74
|
-
):
|
|
69
|
+
def whoami():
|
|
75
70
|
"""
|
|
76
71
|
Return information about the currently logged-in user.
|
|
77
72
|
"""
|
|
78
|
-
config = load_user_config(
|
|
73
|
+
config = load_user_config()
|
|
79
74
|
logged_in_url = config.logged_in_url
|
|
80
75
|
|
|
81
76
|
# TODO - move logged in check to the make_authenticated_request function?
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from pathlib import Path
|
|
3
2
|
|
|
4
3
|
import typer
|
|
5
4
|
import yaml
|
|
6
5
|
|
|
7
|
-
from divbase_cli.cli_commands.
|
|
8
|
-
from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
|
|
6
|
+
from divbase_cli.cli_commands.shared_args_options import PROJECT_NAME_OPTION
|
|
9
7
|
from divbase_cli.config_resolver import resolve_project
|
|
10
8
|
from divbase_cli.user_auth import make_authenticated_request
|
|
11
9
|
from divbase_lib.api_schemas.vcf_dimensions import DimensionsShowResult
|
|
@@ -22,11 +20,10 @@ dimensions_app = typer.Typer(
|
|
|
22
20
|
@dimensions_app.command("update")
|
|
23
21
|
def update_dimensions_index(
|
|
24
22
|
project: str | None = PROJECT_NAME_OPTION,
|
|
25
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
26
23
|
) -> None:
|
|
27
24
|
"""Calculate and add the dimensions of a VCF file to the dimensions index file in the project."""
|
|
28
25
|
|
|
29
|
-
project_config = resolve_project(project_name=project
|
|
26
|
+
project_config = resolve_project(project_name=project)
|
|
30
27
|
|
|
31
28
|
response = make_authenticated_request(
|
|
32
29
|
method="PUT",
|
|
@@ -53,14 +50,13 @@ def show_dimensions_index(
|
|
|
53
50
|
)
|
|
54
51
|
),
|
|
55
52
|
project: str | None = PROJECT_NAME_OPTION,
|
|
56
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
57
53
|
) -> None:
|
|
58
54
|
"""
|
|
59
55
|
Show the dimensions index file for a project.
|
|
60
56
|
When running --unique-scaffolds, the sorting separates between numeric and non-numeric scaffold names.
|
|
61
57
|
"""
|
|
62
58
|
|
|
63
|
-
project_config = resolve_project(project_name=project
|
|
59
|
+
project_config = resolve_project(project_name=project)
|
|
64
60
|
|
|
65
61
|
response = make_authenticated_request(
|
|
66
62
|
method="GET",
|
|
@@ -82,7 +78,7 @@ def show_dimensions_index(
|
|
|
82
78
|
else:
|
|
83
79
|
print(
|
|
84
80
|
f"No entry found for filename: {filename}. Please check that the filename is correct and that it is a VCF file (extension: .vcf or .vcf.gz)."
|
|
85
|
-
"\nHint: use 'divbase-cli files
|
|
81
|
+
"\nHint: use 'divbase-cli files ls' to view all files in the project."
|
|
86
82
|
)
|
|
87
83
|
return
|
|
88
84
|
|
|
@@ -1,51 +1,153 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Command line interface for managing files in a DivBase project's store on DivBase.
|
|
3
3
|
|
|
4
|
-
TODO - support for specifying versions of files when downloading files?
|
|
5
4
|
TODO - Download all files option.
|
|
6
5
|
TODO - skip checked option (aka skip files that already exist in same local dir with correct checksum).
|
|
7
6
|
"""
|
|
8
7
|
|
|
9
8
|
from pathlib import Path
|
|
9
|
+
from zoneinfo import ZoneInfo
|
|
10
10
|
|
|
11
11
|
import typer
|
|
12
12
|
from rich import print
|
|
13
|
+
from rich.table import Table
|
|
13
14
|
from typing_extensions import Annotated
|
|
14
15
|
|
|
15
|
-
from divbase_cli.cli_commands.
|
|
16
|
-
from divbase_cli.
|
|
16
|
+
from divbase_cli.cli_commands.shared_args_options import FORMAT_AS_TSV_OPTION, PROJECT_NAME_OPTION
|
|
17
|
+
from divbase_cli.cli_exceptions import UnsupportedFileNameError, UnsupportedFileTypeError
|
|
17
18
|
from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
|
|
18
|
-
from divbase_cli.services import (
|
|
19
|
+
from divbase_cli.services.s3_files import (
|
|
19
20
|
download_files_command,
|
|
21
|
+
get_file_info_command,
|
|
20
22
|
list_files_command,
|
|
23
|
+
restore_objects_command,
|
|
21
24
|
soft_delete_objects_command,
|
|
25
|
+
stream_file_command,
|
|
22
26
|
upload_files_command,
|
|
23
27
|
)
|
|
28
|
+
from divbase_cli.utils import format_file_size, print_rich_table_as_tsv
|
|
29
|
+
from divbase_lib.divbase_constants import SUPPORTED_DIVBASE_FILE_TYPES, UNSUPPORTED_CHARACTERS_IN_FILENAMES
|
|
24
30
|
|
|
25
31
|
file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
|
|
26
32
|
|
|
33
|
+
NO_FILES_SPECIFIED_MSG = "No files specified for the command, exiting..."
|
|
27
34
|
|
|
28
|
-
|
|
35
|
+
|
|
36
|
+
@file_app.command("ls")
|
|
29
37
|
def list_files(
|
|
38
|
+
format_output_as_tsv: bool = FORMAT_AS_TSV_OPTION,
|
|
39
|
+
prefix_filter: str | None = typer.Option(
|
|
40
|
+
None,
|
|
41
|
+
"--prefix",
|
|
42
|
+
"-p",
|
|
43
|
+
help="Optional prefix to filter the listed files by name (only list files starting with this prefix).",
|
|
44
|
+
),
|
|
45
|
+
include_results_files: bool = typer.Option(
|
|
46
|
+
False,
|
|
47
|
+
"--include-results-files",
|
|
48
|
+
"-r",
|
|
49
|
+
help="If set, will also show DivBase query results files which are hidden by default.",
|
|
50
|
+
),
|
|
30
51
|
project: str | None = PROJECT_NAME_OPTION,
|
|
31
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
32
52
|
):
|
|
33
53
|
"""
|
|
34
|
-
list all files in the project's DivBase store.
|
|
54
|
+
list all currently available files in the project's DivBase store.
|
|
35
55
|
|
|
36
|
-
|
|
37
|
-
|
|
56
|
+
You can optionally filter the listed files by providing a prefix.
|
|
57
|
+
By default, DivBase query results files are hidden from the listing. Use the --include-results-files option to include them.
|
|
58
|
+
To see information about the versions of each file, use the 'divbase-cli files info [FILE_NAME]' command instead
|
|
38
59
|
"""
|
|
39
|
-
project_config = resolve_project(project_name=project
|
|
40
|
-
logged_in_url = ensure_logged_in(
|
|
60
|
+
project_config = resolve_project(project_name=project)
|
|
61
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
62
|
+
|
|
63
|
+
files = list_files_command(
|
|
64
|
+
divbase_base_url=logged_in_url,
|
|
65
|
+
project_name=project_config.name,
|
|
66
|
+
prefix_filter=prefix_filter,
|
|
67
|
+
include_results_files=include_results_files,
|
|
68
|
+
)
|
|
41
69
|
|
|
42
|
-
files = list_files_command(divbase_base_url=logged_in_url, project_name=project_config.name)
|
|
43
70
|
if not files:
|
|
44
71
|
print("No files found in the project's store on DivBase.")
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
table = Table(title=f"Files in [bold]{project_config.name}'s [/bold] DivBase Store:")
|
|
75
|
+
table.add_column("Name", justify="left", style="cyan")
|
|
76
|
+
table.add_column("File size", justify="left", style="magenta", no_wrap=True)
|
|
77
|
+
table.add_column("Upload date (CET)", justify="left", style="green", no_wrap=True)
|
|
78
|
+
table.add_column("MD5 checksum", justify="left", style="yellow")
|
|
79
|
+
|
|
80
|
+
for file_details in files:
|
|
81
|
+
cet_timestamp = file_details.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
82
|
+
file_size = format_file_size(size_bytes=file_details.size)
|
|
83
|
+
table.add_row(
|
|
84
|
+
file_details.name,
|
|
85
|
+
file_size,
|
|
86
|
+
cet_timestamp,
|
|
87
|
+
file_details.etag,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
if not format_output_as_tsv:
|
|
91
|
+
print(table)
|
|
45
92
|
else:
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
93
|
+
print_rich_table_as_tsv(table=table)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@file_app.command("info")
|
|
97
|
+
def file_info(
|
|
98
|
+
file_name: str = typer.Argument(..., help="Name of the file to get information about."),
|
|
99
|
+
format_output_as_tsv: bool = FORMAT_AS_TSV_OPTION,
|
|
100
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
101
|
+
):
|
|
102
|
+
"""
|
|
103
|
+
Get detailed information about a specific file in the project's DivBase store.
|
|
104
|
+
|
|
105
|
+
This includes all versions of the file and whether the file is currently marked as soft deleted.
|
|
106
|
+
"""
|
|
107
|
+
project_config = resolve_project(project_name=project)
|
|
108
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
109
|
+
|
|
110
|
+
file_info = get_file_info_command(
|
|
111
|
+
divbase_base_url=logged_in_url,
|
|
112
|
+
project_name=project_config.name,
|
|
113
|
+
object_name=file_name,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
if not file_info.versions:
|
|
117
|
+
# API should never return an object with no versions, but just in case
|
|
118
|
+
print("No available versions for this file.")
|
|
119
|
+
return
|
|
120
|
+
|
|
121
|
+
if file_info.is_currently_deleted:
|
|
122
|
+
print(
|
|
123
|
+
"[bold red]Warning: This file is marked as soft deleted.\n[/bold red]"
|
|
124
|
+
+ "[italic] To restore a deleted file, use the 'divbase-cli files restore' command.\n"
|
|
125
|
+
+ " Or upload the file again using the 'divbase-cli files upload' command.\n[/italic]",
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
table = Table(
|
|
129
|
+
title=f"Available Versions for '[bold]{file_info.object_name}[/bold]'",
|
|
130
|
+
caption="Versions shown are ordered with the latest/current version first/at the top",
|
|
131
|
+
)
|
|
132
|
+
table.add_column("File size", justify="left", style="magenta", no_wrap=True)
|
|
133
|
+
table.add_column("Upload date (CET)", justify="left", style="green", no_wrap=True)
|
|
134
|
+
table.add_column("MD5 checksum", justify="left", style="yellow")
|
|
135
|
+
table.add_column("Version ID", justify="left", style="cyan")
|
|
136
|
+
|
|
137
|
+
for version in file_info.versions:
|
|
138
|
+
cet_timestamp = version.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
139
|
+
file_size = format_file_size(size_bytes=version.size)
|
|
140
|
+
table.add_row(
|
|
141
|
+
file_size,
|
|
142
|
+
cet_timestamp,
|
|
143
|
+
version.etag,
|
|
144
|
+
version.version_id,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
if not format_output_as_tsv:
|
|
148
|
+
print(table)
|
|
149
|
+
else:
|
|
150
|
+
print_rich_table_as_tsv(table=table)
|
|
49
151
|
|
|
50
152
|
|
|
51
153
|
@file_app.command("download")
|
|
@@ -70,42 +172,36 @@ def download_files(
|
|
|
70
172
|
"It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
|
|
71
173
|
),
|
|
72
174
|
] = False,
|
|
73
|
-
project_version: str = typer.Option(
|
|
175
|
+
project_version: str | None = typer.Option(
|
|
74
176
|
default=None,
|
|
75
177
|
help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
|
|
76
178
|
),
|
|
77
179
|
project: str | None = PROJECT_NAME_OPTION,
|
|
78
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
79
180
|
):
|
|
80
181
|
"""
|
|
81
|
-
Download files from the project's store on DivBase.
|
|
182
|
+
Download files from the project's store on DivBase.
|
|
183
|
+
|
|
184
|
+
This can be done by either:
|
|
82
185
|
1. providing a list of files paths directly in the command line
|
|
83
|
-
2. providing a
|
|
186
|
+
2. providing a text file with a list of files to download (new file on each line).
|
|
187
|
+
|
|
188
|
+
To download the latest version of a file, just provide its name. "file1" "file2" etc.
|
|
189
|
+
To download a specific/older version of a file, use the format: "file_name:version_id"
|
|
190
|
+
You can get a file's version id using the 'divbase-cli file info [FILE_NAME]' command.
|
|
191
|
+
You can mix and match latest and specific versions in the same command.
|
|
192
|
+
E.g. to download the latest version of file1 and version "3xcdsdsdiw829x"
|
|
193
|
+
of file2: 'divbase-cli files download file1 file2:3xcdsdsdiw829x'
|
|
84
194
|
"""
|
|
85
|
-
project_config = resolve_project(project_name=project
|
|
86
|
-
logged_in_url = ensure_logged_in(
|
|
87
|
-
download_dir_path = resolve_download_dir(download_dir=download_dir
|
|
88
|
-
|
|
89
|
-
if bool(files) + bool(file_list) > 1:
|
|
90
|
-
print("Please specify only one of --files or --file-list.")
|
|
91
|
-
raise typer.Exit(1)
|
|
195
|
+
project_config = resolve_project(project_name=project)
|
|
196
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
197
|
+
download_dir_path = resolve_download_dir(download_dir=download_dir)
|
|
92
198
|
|
|
93
|
-
|
|
94
|
-
if files:
|
|
95
|
-
all_files.update(files)
|
|
96
|
-
if file_list:
|
|
97
|
-
with open(file_list) as f:
|
|
98
|
-
for object_name in f:
|
|
99
|
-
all_files.add(object_name.strip())
|
|
100
|
-
|
|
101
|
-
if not all_files:
|
|
102
|
-
print("No files specified for download.")
|
|
103
|
-
raise typer.Exit(1)
|
|
199
|
+
raw_files_input = _resolve_file_inputs(files=files, file_list=file_list)
|
|
104
200
|
|
|
105
201
|
download_results = download_files_command(
|
|
106
202
|
divbase_base_url=logged_in_url,
|
|
107
203
|
project_name=project_config.name,
|
|
108
|
-
|
|
204
|
+
raw_files_input=raw_files_input,
|
|
109
205
|
download_dir=download_dir_path,
|
|
110
206
|
verify_checksums=not disable_verify_checksums,
|
|
111
207
|
project_version=project_version,
|
|
@@ -123,9 +219,41 @@ def download_files(
|
|
|
123
219
|
raise typer.Exit(1)
|
|
124
220
|
|
|
125
221
|
|
|
222
|
+
@file_app.command("stream")
|
|
223
|
+
def stream_file(
|
|
224
|
+
file_name: str = typer.Argument(..., help="Name of the file you want to stream."),
|
|
225
|
+
version_id: str | None = typer.Option(
|
|
226
|
+
default=None,
|
|
227
|
+
help="Specify this if you want to look at an older/specific version of the file. "
|
|
228
|
+
"If not provided, the latest version of the file is used. "
|
|
229
|
+
"To get a file's version ids, use the 'divbase-cli file info [FILE_NAME]' command.",
|
|
230
|
+
),
|
|
231
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
232
|
+
):
|
|
233
|
+
"""
|
|
234
|
+
Stream a file's content to standard output.
|
|
235
|
+
|
|
236
|
+
This allows your to pipe the output to other tools like 'less', 'head', 'zcat' and 'bcftools'.
|
|
237
|
+
|
|
238
|
+
Examples:
|
|
239
|
+
- View a file: divbase-cli files stream my_file.tsv | less
|
|
240
|
+
- View a gzipped file: divbase-cli files stream my_file.vcf.gz | zcat | less
|
|
241
|
+
- Run a bcftools command: divbase-cli files stream my_file.vcf.gz | bcftools view -h - # The "-" tells bcftools to read from standard input
|
|
242
|
+
"""
|
|
243
|
+
project_config = resolve_project(project_name=project)
|
|
244
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
245
|
+
|
|
246
|
+
stream_file_command(
|
|
247
|
+
divbase_base_url=logged_in_url,
|
|
248
|
+
project_name=project_config.name,
|
|
249
|
+
file_name=file_name,
|
|
250
|
+
version_id=version_id,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
|
|
126
254
|
@file_app.command("upload")
|
|
127
255
|
def upload_files(
|
|
128
|
-
files: list[Path] | None = typer.Argument(None, help="Space
|
|
256
|
+
files: list[Path] | None = typer.Argument(None, help="Space separated list of files to upload."),
|
|
129
257
|
upload_dir: Path | None = typer.Option(None, "--upload-dir", help="Directory to upload all files from."),
|
|
130
258
|
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
|
|
131
259
|
disable_safe_mode: Annotated[
|
|
@@ -139,22 +267,23 @@ def upload_files(
|
|
|
139
267
|
),
|
|
140
268
|
] = False,
|
|
141
269
|
project: str | None = PROJECT_NAME_OPTION,
|
|
142
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
143
270
|
):
|
|
144
271
|
"""
|
|
145
|
-
Upload files to your project's store on DivBase
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
272
|
+
Upload files to your project's store on DivBase:
|
|
273
|
+
|
|
274
|
+
To provide files to upload you can either:
|
|
275
|
+
1. provide a list of files paths directly in the command line
|
|
276
|
+
2. provide a directory to upload
|
|
277
|
+
3. provide a text file with or a file list.
|
|
149
278
|
"""
|
|
150
|
-
project_config = resolve_project(project_name=project
|
|
151
|
-
logged_in_url = ensure_logged_in(
|
|
279
|
+
project_config = resolve_project(project_name=project)
|
|
280
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
152
281
|
|
|
153
282
|
if bool(files) + bool(upload_dir) + bool(file_list) > 1:
|
|
154
283
|
print("Please specify only one of --files, --upload_dir, or --file-list.")
|
|
155
284
|
raise typer.Exit(1)
|
|
156
285
|
|
|
157
|
-
all_files = set()
|
|
286
|
+
all_files: set[Path] = set()
|
|
158
287
|
if files:
|
|
159
288
|
all_files.update(files)
|
|
160
289
|
if upload_dir:
|
|
@@ -167,9 +296,11 @@ def upload_files(
|
|
|
167
296
|
all_files.add(path)
|
|
168
297
|
|
|
169
298
|
if not all_files:
|
|
170
|
-
print(
|
|
299
|
+
print(NO_FILES_SPECIFIED_MSG)
|
|
171
300
|
raise typer.Exit(1)
|
|
172
301
|
|
|
302
|
+
_check_for_unsupported_files(all_files)
|
|
303
|
+
|
|
173
304
|
uploaded_results = upload_files_command(
|
|
174
305
|
project_name=project_config.name,
|
|
175
306
|
divbase_base_url=logged_in_url,
|
|
@@ -190,40 +321,30 @@ def upload_files(
|
|
|
190
321
|
raise typer.Exit(1)
|
|
191
322
|
|
|
192
323
|
|
|
193
|
-
@file_app.command("
|
|
324
|
+
@file_app.command("rm")
|
|
194
325
|
def remove_files(
|
|
195
326
|
files: list[str] | None = typer.Argument(
|
|
196
327
|
None, help="Space seperated list of files/objects in the project's store on DivBase to delete."
|
|
197
328
|
),
|
|
198
|
-
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to
|
|
329
|
+
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to delete."),
|
|
199
330
|
dry_run: bool = typer.Option(
|
|
200
331
|
False, "--dry-run", help="If set, will not actually delete the files, just print what would be deleted."
|
|
201
332
|
),
|
|
202
333
|
project: str | None = PROJECT_NAME_OPTION,
|
|
203
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
204
334
|
):
|
|
205
335
|
"""
|
|
206
|
-
|
|
207
|
-
1. providing a list of files paths directly in the command line
|
|
208
|
-
2. providing a text file with or a file list.
|
|
336
|
+
Soft delete files from the project's store on DivBase
|
|
209
337
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
|
|
338
|
+
To provide files to delete you can either:
|
|
339
|
+
1. provide a list of file names directly in the command line
|
|
340
|
+
2. provide a text file with a list of files to delete.
|
|
214
341
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
342
|
+
Note that deleting a non existent file will be treated as a successful deletion.
|
|
343
|
+
"""
|
|
344
|
+
project_config = resolve_project(project_name=project)
|
|
345
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
218
346
|
|
|
219
|
-
all_files =
|
|
220
|
-
|
|
221
|
-
if files:
|
|
222
|
-
all_files.update(files)
|
|
223
|
-
if file_list:
|
|
224
|
-
with open(file_list) as f:
|
|
225
|
-
for line in f:
|
|
226
|
-
all_files.add(line.strip())
|
|
347
|
+
all_files = _resolve_file_inputs(files=files, file_list=file_list)
|
|
227
348
|
|
|
228
349
|
if dry_run:
|
|
229
350
|
print("Dry run mode enabled. The following files would have been deleted:")
|
|
@@ -234,7 +355,7 @@ def remove_files(
|
|
|
234
355
|
deleted_files = soft_delete_objects_command(
|
|
235
356
|
divbase_base_url=logged_in_url,
|
|
236
357
|
project_name=project_config.name,
|
|
237
|
-
all_files=
|
|
358
|
+
all_files=all_files,
|
|
238
359
|
)
|
|
239
360
|
|
|
240
361
|
if deleted_files:
|
|
@@ -243,3 +364,96 @@ def remove_files(
|
|
|
243
364
|
print(f"- '{file}'")
|
|
244
365
|
else:
|
|
245
366
|
print("No files were deleted.")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
@file_app.command("restore")
|
|
370
|
+
def restore_soft_deleted_files(
|
|
371
|
+
files: list[str] | None = typer.Argument(
|
|
372
|
+
None, help="Space seperated list of files/objects in the project's store on DivBase to restore."
|
|
373
|
+
),
|
|
374
|
+
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to restore."),
|
|
375
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
376
|
+
):
|
|
377
|
+
"""
|
|
378
|
+
Restore soft deleted files from the project's store on DivBase
|
|
379
|
+
|
|
380
|
+
To provide files to restore you can either:
|
|
381
|
+
1. provide a list of files directly in the command line.
|
|
382
|
+
2. provide a text file with a list of files to restore (new file on each line).
|
|
383
|
+
|
|
384
|
+
NOTE: Attempts to restore a file that is not soft deleted will be considered successful and the file will remain live. This means you can repeatedly run this command on the same file and get the same response.
|
|
385
|
+
"""
|
|
386
|
+
project_config = resolve_project(project_name=project)
|
|
387
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
388
|
+
|
|
389
|
+
all_files = _resolve_file_inputs(files=files, file_list=file_list)
|
|
390
|
+
|
|
391
|
+
restored_objects_response = restore_objects_command(
|
|
392
|
+
divbase_base_url=logged_in_url,
|
|
393
|
+
project_name=project_config.name,
|
|
394
|
+
all_files=all_files,
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
if restored_objects_response.restored:
|
|
398
|
+
print("Restored files:")
|
|
399
|
+
for file in restored_objects_response.restored:
|
|
400
|
+
print(f"- '{file}'")
|
|
401
|
+
|
|
402
|
+
if restored_objects_response.not_restored:
|
|
403
|
+
print("[bold red]WARNING: Some files could not be restored:[/bold red]")
|
|
404
|
+
for file in restored_objects_response.not_restored:
|
|
405
|
+
print(f"[red]- '{file}'[/red]")
|
|
406
|
+
|
|
407
|
+
print(
|
|
408
|
+
"Possible reasons for failed restores:\n"
|
|
409
|
+
"1. The object does not exist in the bucket (e.g., a typo in the name).\n"
|
|
410
|
+
"2. The object was hard-deleted and is unrecoverable.\n"
|
|
411
|
+
"3. An unexpected server error occurred during the restore attempt."
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _resolve_file_inputs(files: list[str] | None, file_list: Path | None) -> list[str]:
|
|
416
|
+
"""Helper function to resolve file inputs from command line arguments."""
|
|
417
|
+
if bool(files) + bool(file_list) > 1:
|
|
418
|
+
print("Please specify only one of --files or --file-list.")
|
|
419
|
+
raise typer.Exit(1)
|
|
420
|
+
|
|
421
|
+
all_files = set()
|
|
422
|
+
if files:
|
|
423
|
+
all_files.update(files)
|
|
424
|
+
if file_list:
|
|
425
|
+
with open(file_list) as f:
|
|
426
|
+
for line in f:
|
|
427
|
+
all_files.add(line.strip())
|
|
428
|
+
|
|
429
|
+
if not all_files:
|
|
430
|
+
print(NO_FILES_SPECIFIED_MSG)
|
|
431
|
+
raise typer.Exit(1)
|
|
432
|
+
return list(all_files)
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _check_for_unsupported_files(all_files: set[Path]) -> None:
|
|
436
|
+
"""
|
|
437
|
+
Helper fn to check if any of the files to be uploaded are not supported by DivBase.
|
|
438
|
+
Raises error if so.
|
|
439
|
+
|
|
440
|
+
This can be to prevent users from:
|
|
441
|
+
1. Accidentally uploading unsupported files.
|
|
442
|
+
2. Uploading files that have characters that we reserve for actions like filtering/querying on DivBase.
|
|
443
|
+
(e.g. the syntax "[file_name]:[version_id]" can be used to download specific file versions).
|
|
444
|
+
|
|
445
|
+
This is not a security feature, just for UX purposes.
|
|
446
|
+
"""
|
|
447
|
+
unsupported_file_types, unsupported_chars = [], []
|
|
448
|
+
for file_path in all_files:
|
|
449
|
+
if not any(file_path.name.endswith(supported) for supported in SUPPORTED_DIVBASE_FILE_TYPES):
|
|
450
|
+
unsupported_file_types.append(file_path)
|
|
451
|
+
|
|
452
|
+
if any(char in file_path.name for char in UNSUPPORTED_CHARACTERS_IN_FILENAMES):
|
|
453
|
+
unsupported_chars.append(file_path)
|
|
454
|
+
|
|
455
|
+
if unsupported_file_types:
|
|
456
|
+
raise UnsupportedFileTypeError(unsupported_files=unsupported_file_types)
|
|
457
|
+
|
|
458
|
+
if unsupported_chars:
|
|
459
|
+
raise UnsupportedFileNameError(unsupported_files=unsupported_chars)
|
|
@@ -17,13 +17,11 @@ TODO:
|
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
19
|
import logging
|
|
20
|
-
from pathlib import Path
|
|
21
20
|
|
|
22
21
|
import typer
|
|
23
22
|
from rich import print
|
|
24
23
|
|
|
25
|
-
from divbase_cli.cli_commands.
|
|
26
|
-
from divbase_cli.cli_commands.version_cli import PROJECT_NAME_OPTION
|
|
24
|
+
from divbase_cli.cli_commands.shared_args_options import PROJECT_NAME_OPTION
|
|
27
25
|
from divbase_cli.cli_config import cli_settings
|
|
28
26
|
from divbase_cli.config_resolver import resolve_project
|
|
29
27
|
from divbase_cli.user_auth import make_authenticated_request
|
|
@@ -75,7 +73,6 @@ def sample_metadata_query(
|
|
|
75
73
|
),
|
|
76
74
|
metadata_tsv_name: str = METADATA_TSV_ARGUMENT,
|
|
77
75
|
project: str | None = PROJECT_NAME_OPTION,
|
|
78
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
79
76
|
) -> None:
|
|
80
77
|
"""
|
|
81
78
|
Query the tsv sidecar metadata file for the VCF files in the project's data store on DivBase.
|
|
@@ -86,7 +83,7 @@ def sample_metadata_query(
|
|
|
86
83
|
TODO: handle when the name of the sample column is something other than Sample_ID
|
|
87
84
|
"""
|
|
88
85
|
|
|
89
|
-
project_config = resolve_project(project_name=project
|
|
86
|
+
project_config = resolve_project(project_name=project)
|
|
90
87
|
|
|
91
88
|
request_data = SampleMetadataQueryRequest(tsv_filter=filter, metadata_tsv_name=metadata_tsv_name)
|
|
92
89
|
|
|
@@ -116,7 +113,6 @@ def pipe_query(
|
|
|
116
113
|
command: str = BCFTOOLS_ARGUMENT,
|
|
117
114
|
metadata_tsv_name: str = METADATA_TSV_ARGUMENT,
|
|
118
115
|
project: str | None = PROJECT_NAME_OPTION,
|
|
119
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
120
116
|
) -> None:
|
|
121
117
|
"""
|
|
122
118
|
Submit a query to run on the DivBase API. A single, merged VCF file will be added to the project on success.
|
|
@@ -129,7 +125,7 @@ def pipe_query(
|
|
|
129
125
|
TODO consider handling the bcftools command whitelist checks also on the CLI level since the error messages are nicer looking?
|
|
130
126
|
TODO consider moving downloading of missing files elsewhere, since this is now done before the celery task
|
|
131
127
|
"""
|
|
132
|
-
project_config = resolve_project(project_name=project
|
|
128
|
+
project_config = resolve_project(project_name=project)
|
|
133
129
|
|
|
134
130
|
request_data = BcftoolsQueryRequest(tsv_filter=tsv_filter, command=command, metadata_tsv_name=metadata_tsv_name)
|
|
135
131
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""
|
|
2
|
+
To avoid potential problems with circular imports, we can put shared typer args + options (etc...) here
|
|
3
|
+
|
|
4
|
+
If you have something that is only used in one of the cli subcommands, don't move it here.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
PROJECT_NAME_OPTION = typer.Option(
|
|
10
|
+
None,
|
|
11
|
+
help="Name of the DivBase project, if not provided uses the default in your DivBase config file",
|
|
12
|
+
show_default=False,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
FORMAT_AS_TSV_OPTION = typer.Option(
|
|
17
|
+
False,
|
|
18
|
+
"--tsv",
|
|
19
|
+
help="If set, will print the output in .TSV format for easier programmatic parsing.",
|
|
20
|
+
)
|