divbase-cli 0.1.0.dev3__py3-none-any.whl → 0.1.0.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- divbase_cli/__init__.py +1 -1
- divbase_cli/cli_commands/auth_cli.py +12 -1
- divbase_cli/cli_commands/dimensions_cli.py +319 -19
- divbase_cli/cli_commands/file_cli.py +209 -36
- divbase_cli/cli_commands/query_cli.py +11 -2
- divbase_cli/cli_commands/shared_args_options.py +3 -0
- divbase_cli/cli_commands/version_cli.py +82 -13
- divbase_cli/cli_config.py +14 -7
- divbase_cli/services/announcements.py +46 -0
- divbase_cli/services/pre_signed_urls.py +10 -0
- divbase_cli/services/project_versions.py +26 -0
- divbase_cli/services/s3_files.py +112 -25
- divbase_cli/user_auth.py +88 -43
- divbase_cli/utils.py +0 -19
- {divbase_cli-0.1.0.dev3.dist-info → divbase_cli-0.1.0.dev5.dist-info}/METADATA +3 -5
- divbase_cli-0.1.0.dev5.dist-info/RECORD +28 -0
- {divbase_cli-0.1.0.dev3.dist-info → divbase_cli-0.1.0.dev5.dist-info}/WHEEL +1 -1
- divbase_cli-0.1.0.dev3.dist-info/RECORD +0 -27
- {divbase_cli-0.1.0.dev3.dist-info → divbase_cli-0.1.0.dev5.dist-info}/entry_points.txt +0 -0
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Command line interface for managing files in a DivBase project's store on DivBase.
|
|
3
|
-
|
|
4
|
-
TODO - Download all files option.
|
|
5
|
-
TODO - skip checked option (aka skip files that already exist in same local dir with correct checksum).
|
|
6
3
|
"""
|
|
7
4
|
|
|
8
5
|
from pathlib import Path
|
|
@@ -16,22 +13,57 @@ from typing_extensions import Annotated
|
|
|
16
13
|
from divbase_cli.cli_commands.shared_args_options import FORMAT_AS_TSV_OPTION, PROJECT_NAME_OPTION
|
|
17
14
|
from divbase_cli.cli_exceptions import UnsupportedFileNameError, UnsupportedFileTypeError
|
|
18
15
|
from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
|
|
16
|
+
from divbase_cli.services.project_versions import get_version_details_command
|
|
19
17
|
from divbase_cli.services.s3_files import (
|
|
18
|
+
ToDownload,
|
|
20
19
|
download_files_command,
|
|
20
|
+
filter_out_already_downloaded_files,
|
|
21
21
|
get_file_info_command,
|
|
22
22
|
list_files_command,
|
|
23
|
+
list_soft_deleted_files_command,
|
|
23
24
|
restore_objects_command,
|
|
24
25
|
soft_delete_objects_command,
|
|
25
26
|
stream_file_command,
|
|
26
27
|
upload_files_command,
|
|
27
28
|
)
|
|
28
|
-
from divbase_cli.utils import
|
|
29
|
+
from divbase_cli.utils import print_rich_table_as_tsv
|
|
29
30
|
from divbase_lib.divbase_constants import SUPPORTED_DIVBASE_FILE_TYPES, UNSUPPORTED_CHARACTERS_IN_FILENAMES
|
|
31
|
+
from divbase_lib.utils import format_file_size
|
|
30
32
|
|
|
31
33
|
file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
|
|
32
34
|
|
|
33
35
|
NO_FILES_SPECIFIED_MSG = "No files specified for the command, exiting..."
|
|
34
36
|
|
|
37
|
+
DOWNLOAD_DIR_OPTION = typer.Option(
|
|
38
|
+
None,
|
|
39
|
+
"--download-dir",
|
|
40
|
+
"-d",
|
|
41
|
+
help="""Directory to download the files to.
|
|
42
|
+
If not provided, defaults to what you specified in your user config.
|
|
43
|
+
If also not specified in your user config, downloads to the current directory.
|
|
44
|
+
You can also specify "." to download to the current directory.""",
|
|
45
|
+
)
|
|
46
|
+
DISABLE_VERIFY_CHECKSUMS_OPTION = typer.Option(
|
|
47
|
+
False,
|
|
48
|
+
"--disable-verify-checksums",
|
|
49
|
+
"-nc",
|
|
50
|
+
help="Turn off checksum verification which is on by default. "
|
|
51
|
+
"Checksum verification means all downloaded files are verified against their MD5 checksums."
|
|
52
|
+
"It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
|
|
53
|
+
)
|
|
54
|
+
PROJECT_VERSION_OPTION = typer.Option(
|
|
55
|
+
None,
|
|
56
|
+
"--project-version",
|
|
57
|
+
"-pv",
|
|
58
|
+
help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
|
|
59
|
+
)
|
|
60
|
+
DRY_RUN_OPTION = typer.Option(
|
|
61
|
+
False,
|
|
62
|
+
"--dry-run",
|
|
63
|
+
"-n",
|
|
64
|
+
help="If set, will not actually download the files, just print what would be downloaded.",
|
|
65
|
+
)
|
|
66
|
+
|
|
35
67
|
|
|
36
68
|
@file_app.command("ls")
|
|
37
69
|
def list_files(
|
|
@@ -39,7 +71,7 @@ def list_files(
|
|
|
39
71
|
prefix_filter: str | None = typer.Option(
|
|
40
72
|
None,
|
|
41
73
|
"--prefix",
|
|
42
|
-
"-
|
|
74
|
+
"-pre",
|
|
43
75
|
help="Optional prefix to filter the listed files by name (only list files starting with this prefix).",
|
|
44
76
|
),
|
|
45
77
|
include_results_files: bool = typer.Option(
|
|
@@ -48,6 +80,12 @@ def list_files(
|
|
|
48
80
|
"-r",
|
|
49
81
|
help="If set, will also show DivBase query results files which are hidden by default.",
|
|
50
82
|
),
|
|
83
|
+
show_deleted_files: bool = typer.Option(
|
|
84
|
+
False,
|
|
85
|
+
"--show-deleted-files",
|
|
86
|
+
"-s",
|
|
87
|
+
help="Show the files in the project that are currently soft deleted. These files can be recovered within a certain time frame after deletion",
|
|
88
|
+
),
|
|
51
89
|
project: str | None = PROJECT_NAME_OPTION,
|
|
52
90
|
):
|
|
53
91
|
"""
|
|
@@ -57,9 +95,35 @@ def list_files(
|
|
|
57
95
|
By default, DivBase query results files are hidden from the listing. Use the --include-results-files option to include them.
|
|
58
96
|
To see information about the versions of each file, use the 'divbase-cli files info [FILE_NAME]' command instead
|
|
59
97
|
"""
|
|
98
|
+
if (show_deleted_files and include_results_files) or (show_deleted_files and prefix_filter):
|
|
99
|
+
print(
|
|
100
|
+
"The --show-deleted-files option cannot be used with --include-results-files or --prefix. "
|
|
101
|
+
"Please use these options separately."
|
|
102
|
+
)
|
|
103
|
+
raise typer.Exit(1)
|
|
104
|
+
|
|
60
105
|
project_config = resolve_project(project_name=project)
|
|
61
106
|
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
62
107
|
|
|
108
|
+
if show_deleted_files:
|
|
109
|
+
files = list_soft_deleted_files_command(
|
|
110
|
+
divbase_base_url=logged_in_url,
|
|
111
|
+
project_name=project_config.name,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if not files:
|
|
115
|
+
print(f"No soft deleted files found for the project '{project_config.name}'.")
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
print(f"Soft deleted files for the project '{project_config.name}':")
|
|
119
|
+
for file_details in files:
|
|
120
|
+
cet_timestamp = file_details.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
121
|
+
|
|
122
|
+
print(f"- '{file_details.name}' (deleted at: '{cet_timestamp}')")
|
|
123
|
+
print("\nTo restore a soft deleted file, use the 'divbase-cli files restore' command.")
|
|
124
|
+
print("To get more information about one of the soft deleted files, use the 'divbase-cli files info' command.")
|
|
125
|
+
return
|
|
126
|
+
|
|
63
127
|
files = list_files_command(
|
|
64
128
|
divbase_base_url=logged_in_url,
|
|
65
129
|
project_name=project_config.name,
|
|
@@ -156,26 +220,10 @@ def download_files(
|
|
|
156
220
|
None, help="Space separated list of files/objects to download from the project's store on DivBase."
|
|
157
221
|
),
|
|
158
222
|
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
|
|
159
|
-
download_dir: str =
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
If also not specified in your user config, downloads to the current directory.
|
|
164
|
-
You can also specify "." to download to the current directory.""",
|
|
165
|
-
),
|
|
166
|
-
disable_verify_checksums: Annotated[
|
|
167
|
-
bool,
|
|
168
|
-
typer.Option(
|
|
169
|
-
"--disable-verify-checksums",
|
|
170
|
-
help="Turn off checksum verification which is on by default. "
|
|
171
|
-
"Checksum verification means all downloaded files are verified against their MD5 checksums."
|
|
172
|
-
"It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
|
|
173
|
-
),
|
|
174
|
-
] = False,
|
|
175
|
-
project_version: str | None = typer.Option(
|
|
176
|
-
default=None,
|
|
177
|
-
help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
|
|
178
|
-
),
|
|
223
|
+
download_dir: str = DOWNLOAD_DIR_OPTION,
|
|
224
|
+
dry_run: bool = DRY_RUN_OPTION,
|
|
225
|
+
disable_verify_checksums: bool = DISABLE_VERIFY_CHECKSUMS_OPTION,
|
|
226
|
+
project_version: str | None = PROJECT_VERSION_OPTION,
|
|
179
227
|
project: str | None = PROJECT_NAME_OPTION,
|
|
180
228
|
):
|
|
181
229
|
"""
|
|
@@ -204,19 +252,130 @@ def download_files(
|
|
|
204
252
|
raw_files_input=raw_files_input,
|
|
205
253
|
download_dir=download_dir_path,
|
|
206
254
|
verify_checksums=not disable_verify_checksums,
|
|
255
|
+
dry_run=dry_run,
|
|
207
256
|
project_version=project_version,
|
|
208
257
|
)
|
|
209
258
|
|
|
210
|
-
|
|
211
|
-
print("[green bold]Successfully downloaded the following files:[/green bold]")
|
|
212
|
-
for success in download_results.successful:
|
|
213
|
-
print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
|
|
214
|
-
if download_results.failed:
|
|
215
|
-
print("[red bold]ERROR: Failed to download the following files:[/red bold]")
|
|
216
|
-
for failed in download_results.failed:
|
|
217
|
-
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
259
|
+
_pretty_print_download_results(download_results=download_results)
|
|
218
260
|
|
|
219
|
-
|
|
261
|
+
|
|
262
|
+
@file_app.command("download-all")
|
|
263
|
+
def download_all_files(
|
|
264
|
+
download_dir: str = DOWNLOAD_DIR_OPTION,
|
|
265
|
+
resume: bool = typer.Option(
|
|
266
|
+
False,
|
|
267
|
+
"--resume",
|
|
268
|
+
"-r",
|
|
269
|
+
help="If set, will attempt to resume an interrupted download. Will check which files have already been fully downloaded (by checking if a file with the same name and checksum already exists in the download directory) and skip downloading those files again.",
|
|
270
|
+
),
|
|
271
|
+
dry_run: bool = DRY_RUN_OPTION,
|
|
272
|
+
disable_verify_checksums: bool = DISABLE_VERIFY_CHECKSUMS_OPTION,
|
|
273
|
+
project_version: str | None = PROJECT_VERSION_OPTION,
|
|
274
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
275
|
+
):
|
|
276
|
+
"""
|
|
277
|
+
Download all files in the project's store on DivBase.
|
|
278
|
+
Before the download proceeds you'll be prompted if you want to continue.
|
|
279
|
+
DivBase Query results files will not be included in the download.
|
|
280
|
+
|
|
281
|
+
You can resume ('--resume' / '-r') a 'download-all' command, just make sure you're downloading into the same directory.
|
|
282
|
+
"""
|
|
283
|
+
if resume and disable_verify_checksums:
|
|
284
|
+
print(
|
|
285
|
+
"The --resume and --disable-verify-checksums options cannot be used together, "
|
|
286
|
+
"as checksums are used to determine which files don't need to be downloaded. \n"
|
|
287
|
+
"Exiting... "
|
|
288
|
+
)
|
|
289
|
+
return
|
|
290
|
+
|
|
291
|
+
project_config = resolve_project(project_name=project)
|
|
292
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
293
|
+
download_dir_path = resolve_download_dir(download_dir=download_dir)
|
|
294
|
+
|
|
295
|
+
all_files: list[ToDownload] = []
|
|
296
|
+
if project_version:
|
|
297
|
+
version_details = get_version_details_command(
|
|
298
|
+
project_name=project_config.name, divbase_base_url=logged_in_url, version_name=project_version
|
|
299
|
+
)
|
|
300
|
+
for version_name, file_details in version_details.files.items():
|
|
301
|
+
all_files.append(
|
|
302
|
+
ToDownload(
|
|
303
|
+
name=version_name,
|
|
304
|
+
etag=file_details["etag"],
|
|
305
|
+
size_bytes=file_details["size"],
|
|
306
|
+
version_id=file_details["version_id"],
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
else:
|
|
311
|
+
list_files_response = list_files_command(
|
|
312
|
+
divbase_base_url=logged_in_url,
|
|
313
|
+
project_name=project_config.name,
|
|
314
|
+
prefix_filter=None,
|
|
315
|
+
include_results_files=False,
|
|
316
|
+
)
|
|
317
|
+
for file_details in list_files_response:
|
|
318
|
+
all_files.append(
|
|
319
|
+
ToDownload(
|
|
320
|
+
name=file_details.name,
|
|
321
|
+
etag=file_details.etag,
|
|
322
|
+
size_bytes=file_details.size,
|
|
323
|
+
version_id=None, # latest version
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
if not all_files:
|
|
328
|
+
print("No files to download as there are no files in the project's store.")
|
|
329
|
+
return
|
|
330
|
+
|
|
331
|
+
# filter files to download based on those which already exist.
|
|
332
|
+
if resume:
|
|
333
|
+
files_to_download, files_to_overwrite = filter_out_already_downloaded_files(
|
|
334
|
+
all_files=all_files, download_dir=download_dir_path
|
|
335
|
+
)
|
|
336
|
+
if files_to_overwrite:
|
|
337
|
+
print(
|
|
338
|
+
"[yellow bold]Warning: The following files already exist in the download directory but have a different checksum."
|
|
339
|
+
"If you choose to proceed, these files will be overwritten by the download:[/yellow bold]"
|
|
340
|
+
)
|
|
341
|
+
for file in files_to_overwrite:
|
|
342
|
+
print(f"- '{file.name}'")
|
|
343
|
+
|
|
344
|
+
files_to_download.extend(files_to_overwrite)
|
|
345
|
+
|
|
346
|
+
if not files_to_download:
|
|
347
|
+
print("No files left to download, your folder matches the project's store on DivBase, exiting...")
|
|
348
|
+
return
|
|
349
|
+
else:
|
|
350
|
+
files_to_download = all_files
|
|
351
|
+
|
|
352
|
+
total_size_bytes = sum(file.size_bytes for file in files_to_download)
|
|
353
|
+
formatted_total_size = format_file_size(size_bytes=total_size_bytes)
|
|
354
|
+
|
|
355
|
+
print(f"There are '{len(files_to_download)}' files to download with a total size of: {formatted_total_size}.")
|
|
356
|
+
|
|
357
|
+
if not dry_run:
|
|
358
|
+
# (dry run will auto exit in the download_files_command)
|
|
359
|
+
do_download = typer.confirm("Do you want to proceed with the download?")
|
|
360
|
+
if not do_download:
|
|
361
|
+
print("Download cancelled...")
|
|
362
|
+
return
|
|
363
|
+
|
|
364
|
+
if project_version:
|
|
365
|
+
raw_files_input = [f"{file.name}:{file.version_id}" for file in files_to_download]
|
|
366
|
+
else:
|
|
367
|
+
raw_files_input = [file.name for file in files_to_download]
|
|
368
|
+
download_results = download_files_command(
|
|
369
|
+
divbase_base_url=logged_in_url,
|
|
370
|
+
project_name=project_config.name,
|
|
371
|
+
raw_files_input=raw_files_input,
|
|
372
|
+
download_dir=download_dir_path,
|
|
373
|
+
verify_checksums=not disable_verify_checksums,
|
|
374
|
+
dry_run=dry_run,
|
|
375
|
+
project_version=None, # We already know the version id of each file, so can skip the processing here.
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
_pretty_print_download_results(download_results=download_results)
|
|
220
379
|
|
|
221
380
|
|
|
222
381
|
@file_app.command("stream")
|
|
@@ -309,12 +468,12 @@ def upload_files(
|
|
|
309
468
|
)
|
|
310
469
|
|
|
311
470
|
if uploaded_results.successful:
|
|
312
|
-
print("[green bold]
|
|
471
|
+
print("[green bold]\nThe following files were successfully uploaded: [/green bold]")
|
|
313
472
|
for object in uploaded_results.successful:
|
|
314
473
|
print(f"- '{object.object_name}' created from file at: '{object.file_path.resolve()}'")
|
|
315
474
|
|
|
316
475
|
if uploaded_results.failed:
|
|
317
|
-
print("[red bold]
|
|
476
|
+
print("[red bold]\nERROR: Failed to upload the following files:[/red bold]")
|
|
318
477
|
for failed in uploaded_results.failed:
|
|
319
478
|
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
320
479
|
|
|
@@ -457,3 +616,17 @@ def _check_for_unsupported_files(all_files: set[Path]) -> None:
|
|
|
457
616
|
|
|
458
617
|
if unsupported_chars:
|
|
459
618
|
raise UnsupportedFileNameError(unsupported_files=unsupported_chars)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _pretty_print_download_results(download_results):
|
|
622
|
+
"""Helper fn used by download and download all commands to print the results of the download in a nice format."""
|
|
623
|
+
if download_results.successful:
|
|
624
|
+
print("\n[green bold]Successfully downloaded the following files:[/green bold]")
|
|
625
|
+
for success in download_results.successful:
|
|
626
|
+
print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
|
|
627
|
+
|
|
628
|
+
if download_results.failed:
|
|
629
|
+
print("\n[red bold]ERROR: Failed to download the following files:[/red bold]")
|
|
630
|
+
for failed in download_results.failed:
|
|
631
|
+
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
632
|
+
raise typer.Exit(1)
|
|
@@ -97,14 +97,23 @@ def sample_metadata_query(
|
|
|
97
97
|
|
|
98
98
|
results = SampleMetadataQueryTaskResult(**response.json())
|
|
99
99
|
|
|
100
|
+
if results.warnings:
|
|
101
|
+
print("[yellow]Warnings:[/yellow]")
|
|
102
|
+
for warning in results.warnings:
|
|
103
|
+
print(f" • {warning}")
|
|
104
|
+
print()
|
|
105
|
+
|
|
100
106
|
if show_sample_results:
|
|
101
107
|
print("[bright_blue]Name and file for each sample in query results:[/bright_blue]")
|
|
102
108
|
for sample in results.sample_and_filename_subset:
|
|
103
109
|
print(f"Sample ID: '{sample['Sample_ID']}', Filename: '{sample['Filename']}'")
|
|
104
110
|
|
|
105
111
|
print(f"The results for the query ([bright_blue]{results.query_message}[/bright_blue]):")
|
|
106
|
-
|
|
107
|
-
|
|
112
|
+
|
|
113
|
+
unique_sample_ids = results.unique_sample_ids or []
|
|
114
|
+
unique_filenames = results.unique_filenames or []
|
|
115
|
+
print(f"Unique Sample IDs: {unique_sample_ids}")
|
|
116
|
+
print(f"Unique filenames: {unique_filenames}\n")
|
|
108
117
|
|
|
109
118
|
|
|
110
119
|
@query_app.command("bcftools-pipe")
|
|
@@ -8,6 +8,8 @@ import typer
|
|
|
8
8
|
|
|
9
9
|
PROJECT_NAME_OPTION = typer.Option(
|
|
10
10
|
None,
|
|
11
|
+
"--project",
|
|
12
|
+
"-p",
|
|
11
13
|
help="Name of the DivBase project, if not provided uses the default in your DivBase config file",
|
|
12
14
|
show_default=False,
|
|
13
15
|
)
|
|
@@ -16,5 +18,6 @@ PROJECT_NAME_OPTION = typer.Option(
|
|
|
16
18
|
FORMAT_AS_TSV_OPTION = typer.Option(
|
|
17
19
|
False,
|
|
18
20
|
"--tsv",
|
|
21
|
+
"-t",
|
|
19
22
|
help="If set, will print the output in .TSV format for easier programmatic parsing.",
|
|
20
23
|
)
|
|
@@ -14,8 +14,10 @@ from divbase_cli.services.project_versions import (
|
|
|
14
14
|
delete_version_command,
|
|
15
15
|
get_version_details_command,
|
|
16
16
|
list_versions_command,
|
|
17
|
+
update_version_command,
|
|
17
18
|
)
|
|
18
19
|
from divbase_cli.utils import print_rich_table_as_tsv
|
|
20
|
+
from divbase_lib.utils import format_file_size
|
|
19
21
|
|
|
20
22
|
version_app = typer.Typer(
|
|
21
23
|
no_args_is_help=True,
|
|
@@ -49,7 +51,54 @@ def add_version(
|
|
|
49
51
|
print(f"New version: '{add_version_response.name}' added to the project: '{project_config.name}'")
|
|
50
52
|
|
|
51
53
|
|
|
52
|
-
@version_app.command("
|
|
54
|
+
@version_app.command("update")
|
|
55
|
+
def update_version(
|
|
56
|
+
version_name: str = typer.Argument(help="Name of the existing version you want to modify", show_default=False),
|
|
57
|
+
new_name: str | None = typer.Option(
|
|
58
|
+
None, "--new-name", "-n", help="New name for the version. If not specified, the name will remain unchanged."
|
|
59
|
+
),
|
|
60
|
+
new_description: str | None = typer.Option(
|
|
61
|
+
None,
|
|
62
|
+
"--new-description",
|
|
63
|
+
"-d",
|
|
64
|
+
help="Optional new description of the version. If not specified, the description will remain unchanged.",
|
|
65
|
+
),
|
|
66
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
67
|
+
):
|
|
68
|
+
"""
|
|
69
|
+
Update an existing project version entry's name and/or description.
|
|
70
|
+
|
|
71
|
+
The files and timestamp associated with a version entry are immutable and cannot be changed.
|
|
72
|
+
This is by design to ensure version entries are representations of the project's state at the timepoint they were created at.
|
|
73
|
+
You can create a new version entry to capture the current state of the project instead.
|
|
74
|
+
|
|
75
|
+
To see your current version entries run: `divbase-cli version ls`
|
|
76
|
+
"""
|
|
77
|
+
if not new_name and not new_description:
|
|
78
|
+
print(
|
|
79
|
+
"No updates specified. Please provide a new name (--new-name) and/or description (--new-description) to update for this entry."
|
|
80
|
+
)
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
project_config = resolve_project(project_name=project)
|
|
84
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
85
|
+
|
|
86
|
+
updated_version = update_version_command(
|
|
87
|
+
project_name=project_config.name,
|
|
88
|
+
divbase_base_url=logged_in_url,
|
|
89
|
+
version_name=version_name,
|
|
90
|
+
new_name=new_name,
|
|
91
|
+
new_description=new_description,
|
|
92
|
+
)
|
|
93
|
+
if new_name and new_name != version_name:
|
|
94
|
+
print(f"Version '{version_name}' renamed to '{updated_version.name}' in project: '{project_config.name}'")
|
|
95
|
+
else:
|
|
96
|
+
print(f"Version '{updated_version.name}' updated in project: '{project_config.name}'")
|
|
97
|
+
if new_description:
|
|
98
|
+
print(f"New description: '{updated_version.description}'")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@version_app.command("ls")
|
|
53
102
|
def list_versions(
|
|
54
103
|
project: str | None = PROJECT_NAME_OPTION,
|
|
55
104
|
include_deleted: bool = typer.Option(False, help="Include soft-deleted versions in the listing."),
|
|
@@ -100,8 +149,11 @@ def list_versions(
|
|
|
100
149
|
def get_version_info(
|
|
101
150
|
version: str = typer.Argument(help="Specific version to retrieve information for"),
|
|
102
151
|
project: str | None = PROJECT_NAME_OPTION,
|
|
152
|
+
format_output_as_tsv: bool = FORMAT_AS_TSV_OPTION,
|
|
103
153
|
):
|
|
104
|
-
"""
|
|
154
|
+
"""
|
|
155
|
+
Provide detailed information about a user specified project version, including all files present and their unique hashes.
|
|
156
|
+
"""
|
|
105
157
|
project_config = resolve_project(project_name=project)
|
|
106
158
|
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
107
159
|
|
|
@@ -109,26 +161,43 @@ def get_version_info(
|
|
|
109
161
|
project_name=project_config.name, divbase_base_url=logged_in_url, version_name=version
|
|
110
162
|
)
|
|
111
163
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
164
|
+
table = Table(title=f"Project version files for {project_config.name}")
|
|
165
|
+
table.add_column("Name", style="cyan", no_wrap=True)
|
|
166
|
+
table.add_column("Version ID", style="magenta")
|
|
167
|
+
table.add_column("MD5 Checksum", style="green")
|
|
168
|
+
table.add_column("Size", style="yellow")
|
|
169
|
+
for object_name, file_details in version_details.files.items():
|
|
170
|
+
file_size = format_file_size(file_details["size"])
|
|
171
|
+
|
|
172
|
+
table.add_row(
|
|
173
|
+
object_name,
|
|
174
|
+
file_details["version_id"],
|
|
175
|
+
file_details["etag"],
|
|
176
|
+
file_size,
|
|
119
177
|
)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
print(f"
|
|
178
|
+
|
|
179
|
+
if not format_output_as_tsv:
|
|
180
|
+
print(f"Project version entry for project: '{project_config.name}' with name: '{version_details.name}'")
|
|
181
|
+
print(f"Entry created at: {format_timestamp(version_details.created_at)}")
|
|
182
|
+
if version_details.description:
|
|
183
|
+
print(f"Description: {version_details.description}")
|
|
184
|
+
if version_details.is_deleted:
|
|
185
|
+
print(
|
|
186
|
+
"[red]WARNING: This version has been soft-deleted and will soon be permanently deleted unless restored - Contact a DivBase admin to prevent this.[/red]"
|
|
187
|
+
)
|
|
188
|
+
print(table)
|
|
189
|
+
else:
|
|
190
|
+
print_rich_table_as_tsv(table=table)
|
|
123
191
|
|
|
124
192
|
|
|
125
|
-
@version_app.command("
|
|
193
|
+
@version_app.command("rm")
|
|
126
194
|
def delete_version(
|
|
127
195
|
name: str = typer.Argument(help="Name of the version (e.g., semantic version).", show_default=False),
|
|
128
196
|
project: str | None = PROJECT_NAME_OPTION,
|
|
129
197
|
):
|
|
130
198
|
"""
|
|
131
199
|
Delete a version entry in the project versioning table. This does not delete the files themselves.
|
|
200
|
+
|
|
132
201
|
Deleted version entries older than 30 days will be permanently deleted.
|
|
133
202
|
You can ask a DivBase admin to restore a deleted version within that time period.
|
|
134
203
|
"""
|
divbase_cli/cli_config.py
CHANGED
|
@@ -3,7 +3,7 @@ Settings for DivBase CLI.
|
|
|
3
3
|
|
|
4
4
|
This class creates a single 'settings' object at module load time that can be imported and used throughout the entire package.
|
|
5
5
|
|
|
6
|
-
The user config and tokens are stored in the
|
|
6
|
+
The user config and tokens are stored in the users local app dir:
|
|
7
7
|
https://typer.tiangolo.com/tutorial/app-dir/
|
|
8
8
|
"""
|
|
9
9
|
|
|
@@ -15,11 +15,18 @@ import typer
|
|
|
15
15
|
|
|
16
16
|
APP_NAME = "divbase-cli"
|
|
17
17
|
APP_DIR = Path(typer.get_app_dir(APP_NAME))
|
|
18
|
+
DEFAULT_METADATA_TSV_NAME = "sample_metadata.tsv"
|
|
19
|
+
DEFAULT_LOG_LEVEL = "INFO"
|
|
20
|
+
DEV_MODE = os.getenv("DIVBASE_DEV", "0") == "1"
|
|
18
21
|
CONFIG_PATH = APP_DIR / "config.yaml"
|
|
19
22
|
TOKENS_PATH = APP_DIR / ".secrets"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
DEFAULT_DIVBASE_API_URL = "
|
|
23
|
+
|
|
24
|
+
if DEV_MODE:
|
|
25
|
+
DEFAULT_DIVBASE_API_URL = "http://localhost:8000/api"
|
|
26
|
+
DEFAULT_LOGGING_ON = "1"
|
|
27
|
+
else:
|
|
28
|
+
DEFAULT_DIVBASE_API_URL = "https://divbase-dev.scilifelab-2-dev.sys.kth.se/api" # TODO - prod url when time
|
|
29
|
+
DEFAULT_LOGGING_ON = "0"
|
|
23
30
|
|
|
24
31
|
|
|
25
32
|
@dataclass
|
|
@@ -27,7 +34,7 @@ class DivBaseCLISettings:
|
|
|
27
34
|
"""
|
|
28
35
|
Settings for DivBase CLI.
|
|
29
36
|
|
|
30
|
-
|
|
37
|
+
You do not need to create an instance of this class yourself,
|
|
31
38
|
instead, import the 'cli_settings' instance created at this module's load time.
|
|
32
39
|
"""
|
|
33
40
|
|
|
@@ -35,8 +42,8 @@ class DivBaseCLISettings:
|
|
|
35
42
|
TOKENS_PATH: Path = Path(os.getenv("DIVBASE_CLI_TOKENS_PATH", TOKENS_PATH))
|
|
36
43
|
DIVBASE_API_URL: str = os.getenv("DIVBASE_API_URL", DEFAULT_DIVBASE_API_URL)
|
|
37
44
|
METADATA_TSV_NAME: str = os.getenv("DIVBASE_METADATA_TSV_NAME", DEFAULT_METADATA_TSV_NAME)
|
|
38
|
-
LOGGING_ON: bool =
|
|
39
|
-
LOG_LEVEL: str = os.getenv("DIVBASE_LOG_LEVEL",
|
|
45
|
+
LOGGING_ON: bool = os.getenv("DIVBASE_LOGGING_ON", DEFAULT_LOGGING_ON) == "1"
|
|
46
|
+
LOG_LEVEL: str = os.getenv("DIVBASE_LOG_LEVEL", DEFAULT_LOG_LEVEL).upper()
|
|
40
47
|
|
|
41
48
|
def __post_init__(self):
|
|
42
49
|
valid_levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Get announcements from divbase API and display them to the user.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from rich import print
|
|
6
|
+
|
|
7
|
+
from divbase_cli.user_auth import make_unauthenticated_request
|
|
8
|
+
from divbase_lib.api_schemas.announcements import AnnouncementResponse
|
|
9
|
+
|
|
10
|
+
COLOR_MAPPING = {
|
|
11
|
+
"info": "blue",
|
|
12
|
+
"success": "green",
|
|
13
|
+
"warning": "yellow",
|
|
14
|
+
"danger": "red",
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_and_display_announcements(divbase_base_url: str) -> None:
|
|
19
|
+
"""
|
|
20
|
+
Get announcements from the API and display them to the user.
|
|
21
|
+
Used when logging in to display any important messages to the user.
|
|
22
|
+
"""
|
|
23
|
+
response = make_unauthenticated_request(
|
|
24
|
+
method="GET",
|
|
25
|
+
divbase_base_url=divbase_base_url,
|
|
26
|
+
api_route="/v1/core/announcements",
|
|
27
|
+
)
|
|
28
|
+
announcements_data = response.json()
|
|
29
|
+
if not announcements_data:
|
|
30
|
+
return
|
|
31
|
+
announcements = [AnnouncementResponse.model_validate(ann) for ann in announcements_data]
|
|
32
|
+
|
|
33
|
+
if len(announcements) == 1:
|
|
34
|
+
print("[bold]An Announcement from DivBase Server:[/bold]")
|
|
35
|
+
color = COLOR_MAPPING.get(announcements[0].level)
|
|
36
|
+
print(f"[bold {color}]{announcements[0].heading}[/bold {color}]")
|
|
37
|
+
if announcements[0].message:
|
|
38
|
+
print(f"{announcements[0].message}\n")
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
print("[bold]Announcements from DivBase Server:[/bold]")
|
|
42
|
+
for idx, ann in enumerate(announcements, start=1):
|
|
43
|
+
color = COLOR_MAPPING.get(ann.level)
|
|
44
|
+
print(f"[bold {color}]{idx}) {ann.heading}[/bold {color}]")
|
|
45
|
+
if ann.message:
|
|
46
|
+
print(f"{ann.message}\n")
|
|
@@ -21,6 +21,7 @@ from pathlib import Path
|
|
|
21
21
|
|
|
22
22
|
import httpx
|
|
23
23
|
import stamina
|
|
24
|
+
from rich import print
|
|
24
25
|
|
|
25
26
|
from divbase_cli.cli_exceptions import DivBaseAPIConnectionError, DivBaseAPIError
|
|
26
27
|
from divbase_cli.retries import retry_only_on_retryable_http_errors
|
|
@@ -95,6 +96,7 @@ def download_multiple_pre_signed_urls(
|
|
|
95
96
|
for obj in pre_signed_urls:
|
|
96
97
|
output_file_path = download_dir / obj.name
|
|
97
98
|
object_name = obj.name
|
|
99
|
+
print(f"Downloading '{object_name}'...", end=" ")
|
|
98
100
|
try:
|
|
99
101
|
result = _download_single_pre_signed_url(
|
|
100
102
|
httpx_client=client,
|
|
@@ -103,9 +105,11 @@ def download_multiple_pre_signed_urls(
|
|
|
103
105
|
output_file_path=output_file_path,
|
|
104
106
|
object_name=object_name,
|
|
105
107
|
)
|
|
108
|
+
print("[bold green]Success[/bold green]")
|
|
106
109
|
except httpx.HTTPError as err:
|
|
107
110
|
output_file_path.unlink(missing_ok=True) # Clean up possible partial file
|
|
108
111
|
result = FailedDownload(object_name=object_name, file_path=output_file_path, exception=err)
|
|
112
|
+
print("[bold red]Failed[/bold red]")
|
|
109
113
|
|
|
110
114
|
if isinstance(result, SuccessfulDownload):
|
|
111
115
|
successful_downloads.append(result)
|
|
@@ -258,6 +262,7 @@ def upload_multiple_singlepart_pre_signed_urls(
|
|
|
258
262
|
successful_uploads, failed_uploads = [], []
|
|
259
263
|
with httpx.Client() as client:
|
|
260
264
|
for obj in pre_signed_urls:
|
|
265
|
+
print(f"Uploading '{obj.name}'...", end=" ")
|
|
261
266
|
result = _upload_one_singlepart_pre_signed_url(
|
|
262
267
|
httpx_client=client,
|
|
263
268
|
pre_signed_url=obj.pre_signed_url,
|
|
@@ -267,8 +272,10 @@ def upload_multiple_singlepart_pre_signed_urls(
|
|
|
267
272
|
)
|
|
268
273
|
|
|
269
274
|
if isinstance(result, SuccessfulUpload):
|
|
275
|
+
print("[bold green]Success[/bold green]")
|
|
270
276
|
successful_uploads.append(result)
|
|
271
277
|
else:
|
|
278
|
+
print("[bold red]Failed[/bold red]")
|
|
272
279
|
failed_uploads.append(result)
|
|
273
280
|
|
|
274
281
|
return UploadOutcome(successful=successful_uploads, failed=failed_uploads)
|
|
@@ -311,6 +318,7 @@ def perform_multipart_upload(
|
|
|
311
318
|
"""
|
|
312
319
|
object_name = file_path.name
|
|
313
320
|
file_size = file_path.stat().st_size
|
|
321
|
+
print(f"Uploading '{object_name}'...", end=" ")
|
|
314
322
|
|
|
315
323
|
# 1. Create multipart upload
|
|
316
324
|
create_request = CreateMultipartUploadRequest(name=object_name, content_length=file_size)
|
|
@@ -353,6 +361,7 @@ def perform_multipart_upload(
|
|
|
353
361
|
json=complete_request_body.model_dump(),
|
|
354
362
|
)
|
|
355
363
|
completed_upload = CompleteMultipartUploadResponse(**response.json())
|
|
364
|
+
print("[bold green]Success[/bold green]")
|
|
356
365
|
return SuccessfulUpload(file_path=file_path, object_name=completed_upload.name)
|
|
357
366
|
|
|
358
367
|
# 4. If any unexpected error occurs, abort the multipart upload
|
|
@@ -369,6 +378,7 @@ def perform_multipart_upload(
|
|
|
369
378
|
except (DivBaseAPIConnectionError, DivBaseAPIError):
|
|
370
379
|
logger.error(f"Failed to abort multipart upload for object '{object_name}' after an upload error.")
|
|
371
380
|
|
|
381
|
+
print("[bold red]Failed[/bold red]")
|
|
372
382
|
return FailedUpload(object_name=object_name, file_path=file_path, exception=e)
|
|
373
383
|
|
|
374
384
|
|