divbase-cli 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- divbase_cli/__init__.py +1 -1
- divbase_cli/cli_commands/auth_cli.py +16 -10
- divbase_cli/cli_commands/dimensions_cli.py +322 -26
- divbase_cli/cli_commands/file_cli.py +488 -101
- divbase_cli/cli_commands/query_cli.py +14 -9
- divbase_cli/cli_commands/shared_args_options.py +23 -0
- divbase_cli/cli_commands/task_history_cli.py +3 -8
- divbase_cli/cli_commands/user_config_cli.py +14 -44
- divbase_cli/cli_commands/version_cli.py +98 -37
- divbase_cli/cli_config.py +27 -9
- divbase_cli/cli_exceptions.py +37 -22
- divbase_cli/config_resolver.py +10 -10
- divbase_cli/divbase_cli.py +1 -1
- divbase_cli/retries.py +34 -0
- divbase_cli/services/__init__.py +0 -0
- divbase_cli/services/announcements.py +46 -0
- divbase_cli/services/pre_signed_urls.py +456 -0
- divbase_cli/services/project_versions.py +103 -0
- divbase_cli/services/s3_files.py +442 -0
- divbase_cli/user_auth.py +104 -46
- divbase_cli/user_config.py +20 -9
- divbase_cli/utils.py +28 -0
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev4.dist-info}/METADATA +3 -4
- divbase_cli-0.1.0.dev4.dist-info/RECORD +28 -0
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev4.dist-info}/WHEEL +1 -1
- divbase_cli/pre_signed_urls.py +0 -169
- divbase_cli/services.py +0 -219
- divbase_cli-0.1.0.dev2.dist-info/RECORD +0 -22
- {divbase_cli-0.1.0.dev2.dist-info → divbase_cli-0.1.0.dev4.dist-info}/entry_points.txt +0 -0
|
@@ -1,51 +1,217 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Command line interface for managing files in a DivBase project's store on DivBase.
|
|
3
|
-
|
|
4
|
-
TODO - support for specifying versions of files when downloading files?
|
|
5
|
-
TODO - Download all files option.
|
|
6
|
-
TODO - skip checked option (aka skip files that already exist in same local dir with correct checksum).
|
|
7
3
|
"""
|
|
8
4
|
|
|
9
5
|
from pathlib import Path
|
|
6
|
+
from zoneinfo import ZoneInfo
|
|
10
7
|
|
|
11
8
|
import typer
|
|
12
9
|
from rich import print
|
|
10
|
+
from rich.table import Table
|
|
13
11
|
from typing_extensions import Annotated
|
|
14
12
|
|
|
15
|
-
from divbase_cli.cli_commands.
|
|
16
|
-
from divbase_cli.
|
|
13
|
+
from divbase_cli.cli_commands.shared_args_options import FORMAT_AS_TSV_OPTION, PROJECT_NAME_OPTION
|
|
14
|
+
from divbase_cli.cli_exceptions import UnsupportedFileNameError, UnsupportedFileTypeError
|
|
17
15
|
from divbase_cli.config_resolver import ensure_logged_in, resolve_download_dir, resolve_project
|
|
18
|
-
from divbase_cli.services import
|
|
16
|
+
from divbase_cli.services.project_versions import get_version_details_command
|
|
17
|
+
from divbase_cli.services.s3_files import (
|
|
18
|
+
ToDownload,
|
|
19
19
|
download_files_command,
|
|
20
|
+
filter_out_already_downloaded_files,
|
|
21
|
+
get_file_info_command,
|
|
20
22
|
list_files_command,
|
|
23
|
+
list_soft_deleted_files_command,
|
|
24
|
+
restore_objects_command,
|
|
21
25
|
soft_delete_objects_command,
|
|
26
|
+
stream_file_command,
|
|
22
27
|
upload_files_command,
|
|
23
28
|
)
|
|
29
|
+
from divbase_cli.utils import print_rich_table_as_tsv
|
|
30
|
+
from divbase_lib.divbase_constants import SUPPORTED_DIVBASE_FILE_TYPES, UNSUPPORTED_CHARACTERS_IN_FILENAMES
|
|
31
|
+
from divbase_lib.utils import format_file_size
|
|
24
32
|
|
|
25
33
|
file_app = typer.Typer(no_args_is_help=True, help="Download/upload/list files to/from the project's store on DivBase.")
|
|
26
34
|
|
|
35
|
+
NO_FILES_SPECIFIED_MSG = "No files specified for the command, exiting..."
|
|
36
|
+
|
|
37
|
+
DOWNLOAD_DIR_OPTION = typer.Option(
|
|
38
|
+
None,
|
|
39
|
+
"--download-dir",
|
|
40
|
+
"-d",
|
|
41
|
+
help="""Directory to download the files to.
|
|
42
|
+
If not provided, defaults to what you specified in your user config.
|
|
43
|
+
If also not specified in your user config, downloads to the current directory.
|
|
44
|
+
You can also specify "." to download to the current directory.""",
|
|
45
|
+
)
|
|
46
|
+
DISABLE_VERIFY_CHECKSUMS_OPTION = typer.Option(
|
|
47
|
+
False,
|
|
48
|
+
"--disable-verify-checksums",
|
|
49
|
+
"-nc",
|
|
50
|
+
help="Turn off checksum verification which is on by default. "
|
|
51
|
+
"Checksum verification means all downloaded files are verified against their MD5 checksums."
|
|
52
|
+
"It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
|
|
53
|
+
)
|
|
54
|
+
PROJECT_VERSION_OPTION = typer.Option(
|
|
55
|
+
None,
|
|
56
|
+
"--project-version",
|
|
57
|
+
"-pv",
|
|
58
|
+
help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
|
|
59
|
+
)
|
|
60
|
+
DRY_RUN_OPTION = typer.Option(
|
|
61
|
+
False,
|
|
62
|
+
"--dry-run",
|
|
63
|
+
"-n",
|
|
64
|
+
help="If set, will not actually download the files, just print what would be downloaded.",
|
|
65
|
+
)
|
|
66
|
+
|
|
27
67
|
|
|
28
|
-
@file_app.command("
|
|
68
|
+
@file_app.command("ls")
|
|
29
69
|
def list_files(
|
|
70
|
+
format_output_as_tsv: bool = FORMAT_AS_TSV_OPTION,
|
|
71
|
+
prefix_filter: str | None = typer.Option(
|
|
72
|
+
None,
|
|
73
|
+
"--prefix",
|
|
74
|
+
"-pre",
|
|
75
|
+
help="Optional prefix to filter the listed files by name (only list files starting with this prefix).",
|
|
76
|
+
),
|
|
77
|
+
include_results_files: bool = typer.Option(
|
|
78
|
+
False,
|
|
79
|
+
"--include-results-files",
|
|
80
|
+
"-r",
|
|
81
|
+
help="If set, will also show DivBase query results files which are hidden by default.",
|
|
82
|
+
),
|
|
83
|
+
show_deleted_files: bool = typer.Option(
|
|
84
|
+
False,
|
|
85
|
+
"--show-deleted-files",
|
|
86
|
+
"-s",
|
|
87
|
+
help="Show the files in the project that are currently soft deleted. These files can be recovered within a certain time frame after deletion",
|
|
88
|
+
),
|
|
30
89
|
project: str | None = PROJECT_NAME_OPTION,
|
|
31
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
32
90
|
):
|
|
33
91
|
"""
|
|
34
|
-
list all files in the project's DivBase store.
|
|
92
|
+
list all currently available files in the project's DivBase store.
|
|
35
93
|
|
|
36
|
-
|
|
37
|
-
|
|
94
|
+
You can optionally filter the listed files by providing a prefix.
|
|
95
|
+
By default, DivBase query results files are hidden from the listing. Use the --include-results-files option to include them.
|
|
96
|
+
To see information about the versions of each file, use the 'divbase-cli files info [FILE_NAME]' command instead
|
|
38
97
|
"""
|
|
39
|
-
|
|
40
|
-
|
|
98
|
+
if (show_deleted_files and include_results_files) or (show_deleted_files and prefix_filter):
|
|
99
|
+
print(
|
|
100
|
+
"The --show-deleted-files option cannot be used with --include-results-files or --prefix. "
|
|
101
|
+
"Please use these options separately."
|
|
102
|
+
)
|
|
103
|
+
raise typer.Exit(1)
|
|
104
|
+
|
|
105
|
+
project_config = resolve_project(project_name=project)
|
|
106
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
107
|
+
|
|
108
|
+
if show_deleted_files:
|
|
109
|
+
files = list_soft_deleted_files_command(
|
|
110
|
+
divbase_base_url=logged_in_url,
|
|
111
|
+
project_name=project_config.name,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
if not files:
|
|
115
|
+
print(f"No soft deleted files found for the project '{project_config.name}'.")
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
print(f"Soft deleted files for the project '{project_config.name}':")
|
|
119
|
+
for file_details in files:
|
|
120
|
+
cet_timestamp = file_details.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
121
|
+
|
|
122
|
+
print(f"- '{file_details.name}' (deleted at: '{cet_timestamp}')")
|
|
123
|
+
print("\nTo restore a soft deleted file, use the 'divbase-cli files restore' command.")
|
|
124
|
+
print("To get more information about one of the soft deleted files, use the 'divbase-cli files info' command.")
|
|
125
|
+
return
|
|
126
|
+
|
|
127
|
+
files = list_files_command(
|
|
128
|
+
divbase_base_url=logged_in_url,
|
|
129
|
+
project_name=project_config.name,
|
|
130
|
+
prefix_filter=prefix_filter,
|
|
131
|
+
include_results_files=include_results_files,
|
|
132
|
+
)
|
|
41
133
|
|
|
42
|
-
files = list_files_command(divbase_base_url=logged_in_url, project_name=project_config.name)
|
|
43
134
|
if not files:
|
|
44
135
|
print("No files found in the project's store on DivBase.")
|
|
136
|
+
return
|
|
137
|
+
|
|
138
|
+
table = Table(title=f"Files in [bold]{project_config.name}'s [/bold] DivBase Store:")
|
|
139
|
+
table.add_column("Name", justify="left", style="cyan")
|
|
140
|
+
table.add_column("File size", justify="left", style="magenta", no_wrap=True)
|
|
141
|
+
table.add_column("Upload date (CET)", justify="left", style="green", no_wrap=True)
|
|
142
|
+
table.add_column("MD5 checksum", justify="left", style="yellow")
|
|
143
|
+
|
|
144
|
+
for file_details in files:
|
|
145
|
+
cet_timestamp = file_details.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
146
|
+
file_size = format_file_size(size_bytes=file_details.size)
|
|
147
|
+
table.add_row(
|
|
148
|
+
file_details.name,
|
|
149
|
+
file_size,
|
|
150
|
+
cet_timestamp,
|
|
151
|
+
file_details.etag,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
if not format_output_as_tsv:
|
|
155
|
+
print(table)
|
|
45
156
|
else:
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
157
|
+
print_rich_table_as_tsv(table=table)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@file_app.command("info")
|
|
161
|
+
def file_info(
|
|
162
|
+
file_name: str = typer.Argument(..., help="Name of the file to get information about."),
|
|
163
|
+
format_output_as_tsv: bool = FORMAT_AS_TSV_OPTION,
|
|
164
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
165
|
+
):
|
|
166
|
+
"""
|
|
167
|
+
Get detailed information about a specific file in the project's DivBase store.
|
|
168
|
+
|
|
169
|
+
This includes all versions of the file and whether the file is currently marked as soft deleted.
|
|
170
|
+
"""
|
|
171
|
+
project_config = resolve_project(project_name=project)
|
|
172
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
173
|
+
|
|
174
|
+
file_info = get_file_info_command(
|
|
175
|
+
divbase_base_url=logged_in_url,
|
|
176
|
+
project_name=project_config.name,
|
|
177
|
+
object_name=file_name,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
if not file_info.versions:
|
|
181
|
+
# API should never return an object with no versions, but just in case
|
|
182
|
+
print("No available versions for this file.")
|
|
183
|
+
return
|
|
184
|
+
|
|
185
|
+
if file_info.is_currently_deleted:
|
|
186
|
+
print(
|
|
187
|
+
"[bold red]Warning: This file is marked as soft deleted.\n[/bold red]"
|
|
188
|
+
+ "[italic] To restore a deleted file, use the 'divbase-cli files restore' command.\n"
|
|
189
|
+
+ " Or upload the file again using the 'divbase-cli files upload' command.\n[/italic]",
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
table = Table(
|
|
193
|
+
title=f"Available Versions for '[bold]{file_info.object_name}[/bold]'",
|
|
194
|
+
caption="Versions shown are ordered with the latest/current version first/at the top",
|
|
195
|
+
)
|
|
196
|
+
table.add_column("File size", justify="left", style="magenta", no_wrap=True)
|
|
197
|
+
table.add_column("Upload date (CET)", justify="left", style="green", no_wrap=True)
|
|
198
|
+
table.add_column("MD5 checksum", justify="left", style="yellow")
|
|
199
|
+
table.add_column("Version ID", justify="left", style="cyan")
|
|
200
|
+
|
|
201
|
+
for version in file_info.versions:
|
|
202
|
+
cet_timestamp = version.last_modified.astimezone(ZoneInfo("CET")).strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
203
|
+
file_size = format_file_size(size_bytes=version.size)
|
|
204
|
+
table.add_row(
|
|
205
|
+
file_size,
|
|
206
|
+
cet_timestamp,
|
|
207
|
+
version.etag,
|
|
208
|
+
version.version_id,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
if not format_output_as_tsv:
|
|
212
|
+
print(table)
|
|
213
|
+
else:
|
|
214
|
+
print_rich_table_as_tsv(table=table)
|
|
49
215
|
|
|
50
216
|
|
|
51
217
|
@file_app.command("download")
|
|
@@ -54,78 +220,199 @@ def download_files(
|
|
|
54
220
|
None, help="Space separated list of files/objects to download from the project's store on DivBase."
|
|
55
221
|
),
|
|
56
222
|
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
|
|
57
|
-
download_dir: str =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
If also not specified in your user config, downloads to the current directory.
|
|
62
|
-
You can also specify "." to download to the current directory.""",
|
|
63
|
-
),
|
|
64
|
-
disable_verify_checksums: Annotated[
|
|
65
|
-
bool,
|
|
66
|
-
typer.Option(
|
|
67
|
-
"--disable-verify-checksums",
|
|
68
|
-
help="Turn off checksum verification which is on by default. "
|
|
69
|
-
"Checksum verification means all downloaded files are verified against their MD5 checksums."
|
|
70
|
-
"It is recommended to leave checksum verification enabled unless you have a specific reason to disable it.",
|
|
71
|
-
),
|
|
72
|
-
] = False,
|
|
73
|
-
project_version: str = typer.Option(
|
|
74
|
-
default=None,
|
|
75
|
-
help="User defined version of the project's at which to download the files. If not provided, downloads the latest version of all selected files.",
|
|
76
|
-
),
|
|
223
|
+
download_dir: str = DOWNLOAD_DIR_OPTION,
|
|
224
|
+
dry_run: bool = DRY_RUN_OPTION,
|
|
225
|
+
disable_verify_checksums: bool = DISABLE_VERIFY_CHECKSUMS_OPTION,
|
|
226
|
+
project_version: str | None = PROJECT_VERSION_OPTION,
|
|
77
227
|
project: str | None = PROJECT_NAME_OPTION,
|
|
78
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
79
228
|
):
|
|
80
229
|
"""
|
|
81
|
-
Download files from the project's store on DivBase.
|
|
230
|
+
Download files from the project's store on DivBase.
|
|
231
|
+
|
|
232
|
+
This can be done by either:
|
|
82
233
|
1. providing a list of files paths directly in the command line
|
|
83
|
-
2. providing a
|
|
234
|
+
2. providing a text file with a list of files to download (new file on each line).
|
|
235
|
+
|
|
236
|
+
To download the latest version of a file, just provide its name. "file1" "file2" etc.
|
|
237
|
+
To download a specific/older version of a file, use the format: "file_name:version_id"
|
|
238
|
+
You can get a file's version id using the 'divbase-cli file info [FILE_NAME]' command.
|
|
239
|
+
You can mix and match latest and specific versions in the same command.
|
|
240
|
+
E.g. to download the latest version of file1 and version "3xcdsdsdiw829x"
|
|
241
|
+
of file2: 'divbase-cli files download file1 file2:3xcdsdsdiw829x'
|
|
84
242
|
"""
|
|
85
|
-
project_config = resolve_project(project_name=project
|
|
86
|
-
logged_in_url = ensure_logged_in(
|
|
87
|
-
download_dir_path = resolve_download_dir(download_dir=download_dir
|
|
243
|
+
project_config = resolve_project(project_name=project)
|
|
244
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
245
|
+
download_dir_path = resolve_download_dir(download_dir=download_dir)
|
|
88
246
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
247
|
+
raw_files_input = _resolve_file_inputs(files=files, file_list=file_list)
|
|
248
|
+
|
|
249
|
+
download_results = download_files_command(
|
|
250
|
+
divbase_base_url=logged_in_url,
|
|
251
|
+
project_name=project_config.name,
|
|
252
|
+
raw_files_input=raw_files_input,
|
|
253
|
+
download_dir=download_dir_path,
|
|
254
|
+
verify_checksums=not disable_verify_checksums,
|
|
255
|
+
dry_run=dry_run,
|
|
256
|
+
project_version=project_version,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
_pretty_print_download_results(download_results=download_results)
|
|
92
260
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
261
|
+
|
|
262
|
+
@file_app.command("download-all")
|
|
263
|
+
def download_all_files(
|
|
264
|
+
download_dir: str = DOWNLOAD_DIR_OPTION,
|
|
265
|
+
resume: bool = typer.Option(
|
|
266
|
+
False,
|
|
267
|
+
"--resume",
|
|
268
|
+
"-r",
|
|
269
|
+
help="If set, will attempt to resume an interrupted download. Will check which files have already been fully downloaded (by checking if a file with the same name and checksum already exists in the download directory) and skip downloading those files again.",
|
|
270
|
+
),
|
|
271
|
+
dry_run: bool = DRY_RUN_OPTION,
|
|
272
|
+
disable_verify_checksums: bool = DISABLE_VERIFY_CHECKSUMS_OPTION,
|
|
273
|
+
project_version: str | None = PROJECT_VERSION_OPTION,
|
|
274
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
275
|
+
):
|
|
276
|
+
"""
|
|
277
|
+
Download all files in the project's store on DivBase.
|
|
278
|
+
Before the download proceeds you'll be prompted if you want to continue.
|
|
279
|
+
DivBase Query results files will not be included in the download.
|
|
280
|
+
|
|
281
|
+
You can resume ('--resume' / '-r') a 'download-all' command, just make sure you're downloading into the same directory.
|
|
282
|
+
"""
|
|
283
|
+
if resume and disable_verify_checksums:
|
|
284
|
+
print(
|
|
285
|
+
"The --resume and --disable-verify-checksums options cannot be used together, "
|
|
286
|
+
"as checksums are used to determine which files don't need to be downloaded. \n"
|
|
287
|
+
"Exiting... "
|
|
288
|
+
)
|
|
289
|
+
return
|
|
290
|
+
|
|
291
|
+
project_config = resolve_project(project_name=project)
|
|
292
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
293
|
+
download_dir_path = resolve_download_dir(download_dir=download_dir)
|
|
294
|
+
|
|
295
|
+
all_files: list[ToDownload] = []
|
|
296
|
+
if project_version:
|
|
297
|
+
version_details = get_version_details_command(
|
|
298
|
+
project_name=project_config.name, divbase_base_url=logged_in_url, version_name=project_version
|
|
299
|
+
)
|
|
300
|
+
for version_name, file_details in version_details.files.items():
|
|
301
|
+
all_files.append(
|
|
302
|
+
ToDownload(
|
|
303
|
+
name=version_name,
|
|
304
|
+
etag=file_details["etag"],
|
|
305
|
+
size_bytes=file_details["size"],
|
|
306
|
+
version_id=file_details["version_id"],
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
else:
|
|
311
|
+
list_files_response = list_files_command(
|
|
312
|
+
divbase_base_url=logged_in_url,
|
|
313
|
+
project_name=project_config.name,
|
|
314
|
+
prefix_filter=None,
|
|
315
|
+
include_results_files=False,
|
|
316
|
+
)
|
|
317
|
+
for file_details in list_files_response:
|
|
318
|
+
all_files.append(
|
|
319
|
+
ToDownload(
|
|
320
|
+
name=file_details.name,
|
|
321
|
+
etag=file_details.etag,
|
|
322
|
+
size_bytes=file_details.size,
|
|
323
|
+
version_id=None, # latest version
|
|
324
|
+
)
|
|
325
|
+
)
|
|
100
326
|
|
|
101
327
|
if not all_files:
|
|
102
|
-
print("No files
|
|
103
|
-
|
|
328
|
+
print("No files to download as there are no files in the project's store.")
|
|
329
|
+
return
|
|
104
330
|
|
|
331
|
+
# filter files to download based on those which already exist.
|
|
332
|
+
if resume:
|
|
333
|
+
files_to_download, files_to_overwrite = filter_out_already_downloaded_files(
|
|
334
|
+
all_files=all_files, download_dir=download_dir_path
|
|
335
|
+
)
|
|
336
|
+
if files_to_overwrite:
|
|
337
|
+
print(
|
|
338
|
+
"[yellow bold]Warning: The following files already exist in the download directory but have a different checksum."
|
|
339
|
+
"If you choose to proceed, these files will be overwritten by the download:[/yellow bold]"
|
|
340
|
+
)
|
|
341
|
+
for file in files_to_overwrite:
|
|
342
|
+
print(f"- '{file.name}'")
|
|
343
|
+
|
|
344
|
+
files_to_download.extend(files_to_overwrite)
|
|
345
|
+
|
|
346
|
+
if not files_to_download:
|
|
347
|
+
print("No files left to download, your folder matches the project's store on DivBase, exiting...")
|
|
348
|
+
return
|
|
349
|
+
else:
|
|
350
|
+
files_to_download = all_files
|
|
351
|
+
|
|
352
|
+
total_size_bytes = sum(file.size_bytes for file in files_to_download)
|
|
353
|
+
formatted_total_size = format_file_size(size_bytes=total_size_bytes)
|
|
354
|
+
|
|
355
|
+
print(f"There are '{len(files_to_download)}' files to download with a total size of: {formatted_total_size}.")
|
|
356
|
+
|
|
357
|
+
if not dry_run:
|
|
358
|
+
# (dry run will auto exit in the download_files_command)
|
|
359
|
+
do_download = typer.confirm("Do you want to proceed with the download?")
|
|
360
|
+
if not do_download:
|
|
361
|
+
print("Download cancelled...")
|
|
362
|
+
return
|
|
363
|
+
|
|
364
|
+
if project_version:
|
|
365
|
+
raw_files_input = [f"{file.name}:{file.version_id}" for file in files_to_download]
|
|
366
|
+
else:
|
|
367
|
+
raw_files_input = [file.name for file in files_to_download]
|
|
105
368
|
download_results = download_files_command(
|
|
106
369
|
divbase_base_url=logged_in_url,
|
|
107
370
|
project_name=project_config.name,
|
|
108
|
-
|
|
371
|
+
raw_files_input=raw_files_input,
|
|
109
372
|
download_dir=download_dir_path,
|
|
110
373
|
verify_checksums=not disable_verify_checksums,
|
|
111
|
-
|
|
374
|
+
dry_run=dry_run,
|
|
375
|
+
project_version=None, # We already know the version id of each file, so can skip the processing here.
|
|
112
376
|
)
|
|
113
377
|
|
|
114
|
-
|
|
115
|
-
print("[green bold]Successfully downloaded the following files:[/green bold]")
|
|
116
|
-
for success in download_results.successful:
|
|
117
|
-
print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
|
|
118
|
-
if download_results.failed:
|
|
119
|
-
print("[red bold]ERROR: Failed to download the following files:[/red bold]")
|
|
120
|
-
for failed in download_results.failed:
|
|
121
|
-
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
378
|
+
_pretty_print_download_results(download_results=download_results)
|
|
122
379
|
|
|
123
|
-
|
|
380
|
+
|
|
381
|
+
@file_app.command("stream")
|
|
382
|
+
def stream_file(
|
|
383
|
+
file_name: str = typer.Argument(..., help="Name of the file you want to stream."),
|
|
384
|
+
version_id: str | None = typer.Option(
|
|
385
|
+
default=None,
|
|
386
|
+
help="Specify this if you want to look at an older/specific version of the file. "
|
|
387
|
+
"If not provided, the latest version of the file is used. "
|
|
388
|
+
"To get a file's version ids, use the 'divbase-cli file info [FILE_NAME]' command.",
|
|
389
|
+
),
|
|
390
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
391
|
+
):
|
|
392
|
+
"""
|
|
393
|
+
Stream a file's content to standard output.
|
|
394
|
+
|
|
395
|
+
This allows your to pipe the output to other tools like 'less', 'head', 'zcat' and 'bcftools'.
|
|
396
|
+
|
|
397
|
+
Examples:
|
|
398
|
+
- View a file: divbase-cli files stream my_file.tsv | less
|
|
399
|
+
- View a gzipped file: divbase-cli files stream my_file.vcf.gz | zcat | less
|
|
400
|
+
- Run a bcftools command: divbase-cli files stream my_file.vcf.gz | bcftools view -h - # The "-" tells bcftools to read from standard input
|
|
401
|
+
"""
|
|
402
|
+
project_config = resolve_project(project_name=project)
|
|
403
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
404
|
+
|
|
405
|
+
stream_file_command(
|
|
406
|
+
divbase_base_url=logged_in_url,
|
|
407
|
+
project_name=project_config.name,
|
|
408
|
+
file_name=file_name,
|
|
409
|
+
version_id=version_id,
|
|
410
|
+
)
|
|
124
411
|
|
|
125
412
|
|
|
126
413
|
@file_app.command("upload")
|
|
127
414
|
def upload_files(
|
|
128
|
-
files: list[Path] | None = typer.Argument(None, help="Space
|
|
415
|
+
files: list[Path] | None = typer.Argument(None, help="Space separated list of files to upload."),
|
|
129
416
|
upload_dir: Path | None = typer.Option(None, "--upload-dir", help="Directory to upload all files from."),
|
|
130
417
|
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to upload."),
|
|
131
418
|
disable_safe_mode: Annotated[
|
|
@@ -139,22 +426,23 @@ def upload_files(
|
|
|
139
426
|
),
|
|
140
427
|
] = False,
|
|
141
428
|
project: str | None = PROJECT_NAME_OPTION,
|
|
142
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
143
429
|
):
|
|
144
430
|
"""
|
|
145
|
-
Upload files to your project's store on DivBase
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
431
|
+
Upload files to your project's store on DivBase:
|
|
432
|
+
|
|
433
|
+
To provide files to upload you can either:
|
|
434
|
+
1. provide a list of files paths directly in the command line
|
|
435
|
+
2. provide a directory to upload
|
|
436
|
+
3. provide a text file with or a file list.
|
|
149
437
|
"""
|
|
150
|
-
project_config = resolve_project(project_name=project
|
|
151
|
-
logged_in_url = ensure_logged_in(
|
|
438
|
+
project_config = resolve_project(project_name=project)
|
|
439
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
152
440
|
|
|
153
441
|
if bool(files) + bool(upload_dir) + bool(file_list) > 1:
|
|
154
442
|
print("Please specify only one of --files, --upload_dir, or --file-list.")
|
|
155
443
|
raise typer.Exit(1)
|
|
156
444
|
|
|
157
|
-
all_files = set()
|
|
445
|
+
all_files: set[Path] = set()
|
|
158
446
|
if files:
|
|
159
447
|
all_files.update(files)
|
|
160
448
|
if upload_dir:
|
|
@@ -167,9 +455,11 @@ def upload_files(
|
|
|
167
455
|
all_files.add(path)
|
|
168
456
|
|
|
169
457
|
if not all_files:
|
|
170
|
-
print(
|
|
458
|
+
print(NO_FILES_SPECIFIED_MSG)
|
|
171
459
|
raise typer.Exit(1)
|
|
172
460
|
|
|
461
|
+
_check_for_unsupported_files(all_files)
|
|
462
|
+
|
|
173
463
|
uploaded_results = upload_files_command(
|
|
174
464
|
project_name=project_config.name,
|
|
175
465
|
divbase_base_url=logged_in_url,
|
|
@@ -178,52 +468,42 @@ def upload_files(
|
|
|
178
468
|
)
|
|
179
469
|
|
|
180
470
|
if uploaded_results.successful:
|
|
181
|
-
print("[green bold]
|
|
471
|
+
print("[green bold]\nThe following files were successfully uploaded: [/green bold]")
|
|
182
472
|
for object in uploaded_results.successful:
|
|
183
473
|
print(f"- '{object.object_name}' created from file at: '{object.file_path.resolve()}'")
|
|
184
474
|
|
|
185
475
|
if uploaded_results.failed:
|
|
186
|
-
print("[red bold]
|
|
476
|
+
print("[red bold]\nERROR: Failed to upload the following files:[/red bold]")
|
|
187
477
|
for failed in uploaded_results.failed:
|
|
188
478
|
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
189
479
|
|
|
190
480
|
raise typer.Exit(1)
|
|
191
481
|
|
|
192
482
|
|
|
193
|
-
@file_app.command("
|
|
483
|
+
@file_app.command("rm")
|
|
194
484
|
def remove_files(
|
|
195
485
|
files: list[str] | None = typer.Argument(
|
|
196
486
|
None, help="Space seperated list of files/objects in the project's store on DivBase to delete."
|
|
197
487
|
),
|
|
198
|
-
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to
|
|
488
|
+
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to delete."),
|
|
199
489
|
dry_run: bool = typer.Option(
|
|
200
490
|
False, "--dry-run", help="If set, will not actually delete the files, just print what would be deleted."
|
|
201
491
|
),
|
|
202
492
|
project: str | None = PROJECT_NAME_OPTION,
|
|
203
|
-
config_file: Path = CONFIG_FILE_OPTION,
|
|
204
493
|
):
|
|
205
494
|
"""
|
|
206
|
-
|
|
207
|
-
1. providing a list of files paths directly in the command line
|
|
208
|
-
2. providing a text file with or a file list.
|
|
209
|
-
|
|
210
|
-
'dry_run' mode will not actually delete the files, just print what would be deleted.
|
|
211
|
-
"""
|
|
212
|
-
project_config = resolve_project(project_name=project, config_path=config_file)
|
|
213
|
-
logged_in_url = ensure_logged_in(config_path=config_file, desired_url=project_config.divbase_url)
|
|
495
|
+
Soft delete files from the project's store on DivBase
|
|
214
496
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
497
|
+
To provide files to delete you can either:
|
|
498
|
+
1. provide a list of file names directly in the command line
|
|
499
|
+
2. provide a text file with a list of files to delete.
|
|
218
500
|
|
|
219
|
-
|
|
501
|
+
Note that deleting a non existent file will be treated as a successful deletion.
|
|
502
|
+
"""
|
|
503
|
+
project_config = resolve_project(project_name=project)
|
|
504
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
220
505
|
|
|
221
|
-
|
|
222
|
-
all_files.update(files)
|
|
223
|
-
if file_list:
|
|
224
|
-
with open(file_list) as f:
|
|
225
|
-
for line in f:
|
|
226
|
-
all_files.add(line.strip())
|
|
506
|
+
all_files = _resolve_file_inputs(files=files, file_list=file_list)
|
|
227
507
|
|
|
228
508
|
if dry_run:
|
|
229
509
|
print("Dry run mode enabled. The following files would have been deleted:")
|
|
@@ -234,7 +514,7 @@ def remove_files(
|
|
|
234
514
|
deleted_files = soft_delete_objects_command(
|
|
235
515
|
divbase_base_url=logged_in_url,
|
|
236
516
|
project_name=project_config.name,
|
|
237
|
-
all_files=
|
|
517
|
+
all_files=all_files,
|
|
238
518
|
)
|
|
239
519
|
|
|
240
520
|
if deleted_files:
|
|
@@ -243,3 +523,110 @@ def remove_files(
|
|
|
243
523
|
print(f"- '{file}'")
|
|
244
524
|
else:
|
|
245
525
|
print("No files were deleted.")
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
@file_app.command("restore")
|
|
529
|
+
def restore_soft_deleted_files(
|
|
530
|
+
files: list[str] | None = typer.Argument(
|
|
531
|
+
None, help="Space seperated list of files/objects in the project's store on DivBase to restore."
|
|
532
|
+
),
|
|
533
|
+
file_list: Path | None = typer.Option(None, "--file-list", help="Text file with list of files to restore."),
|
|
534
|
+
project: str | None = PROJECT_NAME_OPTION,
|
|
535
|
+
):
|
|
536
|
+
"""
|
|
537
|
+
Restore soft deleted files from the project's store on DivBase
|
|
538
|
+
|
|
539
|
+
To provide files to restore you can either:
|
|
540
|
+
1. provide a list of files directly in the command line.
|
|
541
|
+
2. provide a text file with a list of files to restore (new file on each line).
|
|
542
|
+
|
|
543
|
+
NOTE: Attempts to restore a file that is not soft deleted will be considered successful and the file will remain live. This means you can repeatedly run this command on the same file and get the same response.
|
|
544
|
+
"""
|
|
545
|
+
project_config = resolve_project(project_name=project)
|
|
546
|
+
logged_in_url = ensure_logged_in(desired_url=project_config.divbase_url)
|
|
547
|
+
|
|
548
|
+
all_files = _resolve_file_inputs(files=files, file_list=file_list)
|
|
549
|
+
|
|
550
|
+
restored_objects_response = restore_objects_command(
|
|
551
|
+
divbase_base_url=logged_in_url,
|
|
552
|
+
project_name=project_config.name,
|
|
553
|
+
all_files=all_files,
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
if restored_objects_response.restored:
|
|
557
|
+
print("Restored files:")
|
|
558
|
+
for file in restored_objects_response.restored:
|
|
559
|
+
print(f"- '{file}'")
|
|
560
|
+
|
|
561
|
+
if restored_objects_response.not_restored:
|
|
562
|
+
print("[bold red]WARNING: Some files could not be restored:[/bold red]")
|
|
563
|
+
for file in restored_objects_response.not_restored:
|
|
564
|
+
print(f"[red]- '{file}'[/red]")
|
|
565
|
+
|
|
566
|
+
print(
|
|
567
|
+
"Possible reasons for failed restores:\n"
|
|
568
|
+
"1. The object does not exist in the bucket (e.g., a typo in the name).\n"
|
|
569
|
+
"2. The object was hard-deleted and is unrecoverable.\n"
|
|
570
|
+
"3. An unexpected server error occurred during the restore attempt."
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def _resolve_file_inputs(files: list[str] | None, file_list: Path | None) -> list[str]:
|
|
575
|
+
"""Helper function to resolve file inputs from command line arguments."""
|
|
576
|
+
if bool(files) + bool(file_list) > 1:
|
|
577
|
+
print("Please specify only one of --files or --file-list.")
|
|
578
|
+
raise typer.Exit(1)
|
|
579
|
+
|
|
580
|
+
all_files = set()
|
|
581
|
+
if files:
|
|
582
|
+
all_files.update(files)
|
|
583
|
+
if file_list:
|
|
584
|
+
with open(file_list) as f:
|
|
585
|
+
for line in f:
|
|
586
|
+
all_files.add(line.strip())
|
|
587
|
+
|
|
588
|
+
if not all_files:
|
|
589
|
+
print(NO_FILES_SPECIFIED_MSG)
|
|
590
|
+
raise typer.Exit(1)
|
|
591
|
+
return list(all_files)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _check_for_unsupported_files(all_files: set[Path]) -> None:
|
|
595
|
+
"""
|
|
596
|
+
Helper fn to check if any of the files to be uploaded are not supported by DivBase.
|
|
597
|
+
Raises error if so.
|
|
598
|
+
|
|
599
|
+
This can be to prevent users from:
|
|
600
|
+
1. Accidentally uploading unsupported files.
|
|
601
|
+
2. Uploading files that have characters that we reserve for actions like filtering/querying on DivBase.
|
|
602
|
+
(e.g. the syntax "[file_name]:[version_id]" can be used to download specific file versions).
|
|
603
|
+
|
|
604
|
+
This is not a security feature, just for UX purposes.
|
|
605
|
+
"""
|
|
606
|
+
unsupported_file_types, unsupported_chars = [], []
|
|
607
|
+
for file_path in all_files:
|
|
608
|
+
if not any(file_path.name.endswith(supported) for supported in SUPPORTED_DIVBASE_FILE_TYPES):
|
|
609
|
+
unsupported_file_types.append(file_path)
|
|
610
|
+
|
|
611
|
+
if any(char in file_path.name for char in UNSUPPORTED_CHARACTERS_IN_FILENAMES):
|
|
612
|
+
unsupported_chars.append(file_path)
|
|
613
|
+
|
|
614
|
+
if unsupported_file_types:
|
|
615
|
+
raise UnsupportedFileTypeError(unsupported_files=unsupported_file_types)
|
|
616
|
+
|
|
617
|
+
if unsupported_chars:
|
|
618
|
+
raise UnsupportedFileNameError(unsupported_files=unsupported_chars)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _pretty_print_download_results(download_results):
|
|
622
|
+
"""Helper fn used by download and download all commands to print the results of the download in a nice format."""
|
|
623
|
+
if download_results.successful:
|
|
624
|
+
print("\n[green bold]Successfully downloaded the following files:[/green bold]")
|
|
625
|
+
for success in download_results.successful:
|
|
626
|
+
print(f"- '{success.object_name}' downloaded to: '{success.file_path.resolve()}'")
|
|
627
|
+
|
|
628
|
+
if download_results.failed:
|
|
629
|
+
print("\n[red bold]ERROR: Failed to download the following files:[/red bold]")
|
|
630
|
+
for failed in download_results.failed:
|
|
631
|
+
print(f"[red]- '{failed.object_name}': Exception: '{failed.exception}'[/red]")
|
|
632
|
+
raise typer.Exit(1)
|