dds-cli 2.2.64__tar.gz → 2.2.65__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dds_cli-2.2.64 → dds_cli-2.2.65}/PKG-INFO +1 -1
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/__init__.py +2 -1
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/__main__.py +31 -26
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/account_manager.py +15 -12
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/auth.py +8 -3
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/base.py +21 -23
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/custom_decorators.py +13 -11
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/data_getter.py +6 -7
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/data_lister.py +64 -64
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/data_putter.py +19 -14
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/data_remover.py +14 -12
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/directory.py +5 -4
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/file_compressor.py +5 -5
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/file_encryptor.py +7 -7
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/file_handler.py +4 -5
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/file_handler_local.py +20 -18
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/file_handler_remote.py +3 -4
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/motd_manager.py +2 -7
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/options.py +9 -2
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/project_creator.py +3 -4
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/project_info.py +19 -11
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/project_status.py +19 -20
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/s3_connector.py +5 -6
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/status.py +1 -1
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/timestamp.py +7 -5
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/unit_manager.py +0 -5
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/user.py +34 -25
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/utils.py +18 -13
- dds_cli-2.2.65/dds_cli/version.py +3 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/PKG-INFO +1 -1
- {dds_cli-2.2.64 → dds_cli-2.2.65}/setup.py +4 -2
- dds_cli-2.2.64/dds_cli/version.py +0 -1
- {dds_cli-2.2.64 → dds_cli-2.2.65}/LICENSE +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/README.md +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/exceptions.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/maintenance_manager.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli/text_handler.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/SOURCES.txt +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/dependency_links.txt +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/entry_points.txt +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/not-zip-safe +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/requires.txt +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/dds_cli.egg-info/top_level.txt +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/pyproject.toml +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/setup.cfg +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/__init__.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_account_manager.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_data_remover.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_file_compressor.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_file_encryptor.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_file_handler_local.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_maintenance_manager.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_motd_manager.py +0 -0
- {dds_cli-2.2.64 → dds_cli-2.2.65}/tests/test_utils.py +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# pylint: skip-file
|
|
1
2
|
"""DDS CLI."""
|
|
2
3
|
|
|
3
4
|
import datetime
|
|
@@ -59,7 +60,7 @@ class DDSEndpoint:
|
|
|
59
60
|
BASE_ENDPOINT_LOCAL = "http://127.0.0.1:5000/api/v1"
|
|
60
61
|
BASE_ENDPOINT_DOCKER = "http://dds_backend:5000/api/v1"
|
|
61
62
|
BASE_ENDPOINT_REMOTE = "https://delivery.scilifelab.se/api/v1"
|
|
62
|
-
BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.
|
|
63
|
+
BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.dckube3.scilifelab.se/api/v1"
|
|
63
64
|
if os.getenv("DDS_CLI_ENV") == "development":
|
|
64
65
|
BASE_ENDPOINT = BASE_ENDPOINT_LOCAL
|
|
65
66
|
elif os.getenv("DDS_CLI_ENV") == "docker-dev":
|
|
@@ -8,7 +8,6 @@
|
|
|
8
8
|
import concurrent.futures
|
|
9
9
|
import itertools
|
|
10
10
|
import logging
|
|
11
|
-
import os
|
|
12
11
|
import sys
|
|
13
12
|
import typing
|
|
14
13
|
|
|
@@ -71,7 +70,6 @@ LOG = logging.getLogger("dds_cli")
|
|
|
71
70
|
# Configuration for rich-click output
|
|
72
71
|
click.rich_click.MAX_WIDTH = 100
|
|
73
72
|
|
|
74
|
-
|
|
75
73
|
## # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
|
76
74
|
# #
|
|
77
75
|
# MMMM MMMM AAAA II NNNN NN #
|
|
@@ -83,15 +81,15 @@ click.rich_click.MAX_WIDTH = 100
|
|
|
83
81
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ##
|
|
84
82
|
|
|
85
83
|
|
|
86
|
-
|
|
84
|
+
DDS_URL = dds_cli.DDSEndpoint.BASE_ENDPOINT
|
|
85
|
+
DDS_URL_BASE = DDS_URL[: DDS_URL.index("/", 8)]
|
|
86
|
+
|
|
87
87
|
# Print header to STDERR
|
|
88
88
|
dds_cli.utils.stderr_console.print(
|
|
89
89
|
"[green] ︵",
|
|
90
90
|
"\n[green] ︵ ( ) ︵",
|
|
91
91
|
"\n[green]( ) ) ( ( )[/] [bold]SciLifeLab Data Delivery System",
|
|
92
|
-
"\n[green] ︶ ( ) ) ([/] [blue][link={
|
|
93
|
-
dds_url[: dds_url.index("/", 8)]
|
|
94
|
-
),
|
|
92
|
+
f"\n[green] ︶ ( ) ) ([/] [blue][link={DDS_URL_BASE}]{DDS_URL_BASE}/[/link]",
|
|
95
93
|
f"\n[green] ︶ ( )[/] [dim]CLI Version {dds_cli.__version__}",
|
|
96
94
|
"\n[green] ︶",
|
|
97
95
|
highlight=False,
|
|
@@ -100,7 +98,7 @@ dds_cli.utils.stderr_console.print(
|
|
|
100
98
|
if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] != "motd"):
|
|
101
99
|
motds = dds_cli.motd_manager.MotdManager.list_all_active_motds(table=False)
|
|
102
100
|
if motds:
|
|
103
|
-
dds_cli.utils.stderr_console.print(
|
|
101
|
+
dds_cli.utils.stderr_console.print("[bold]Important information:[/bold]")
|
|
104
102
|
for motd in motds:
|
|
105
103
|
dds_cli.utils.stderr_console.print(f"{motd['Created']} - {motd['Message']} \n")
|
|
106
104
|
|
|
@@ -380,7 +378,10 @@ def auth_group_command(_):
|
|
|
380
378
|
is_flag=True,
|
|
381
379
|
required=False,
|
|
382
380
|
default=False,
|
|
383
|
-
help=
|
|
381
|
+
help=(
|
|
382
|
+
"[Not recommended, use with care] Allow read permissions to group. Sets 640 permission instead of 600, "
|
|
383
|
+
"allowing others to access your authenticated session token."
|
|
384
|
+
),
|
|
384
385
|
)
|
|
385
386
|
@click.pass_obj
|
|
386
387
|
def login(click_ctx, totp, allow_group):
|
|
@@ -404,7 +405,7 @@ def login(click_ctx, totp, allow_group):
|
|
|
404
405
|
token_path=click_ctx.get("TOKEN_PATH"), totp=totp, allow_group=allow_group
|
|
405
406
|
):
|
|
406
407
|
# Authentication token renewed in the init method.
|
|
407
|
-
LOG.info(
|
|
408
|
+
LOG.info("[green] :white_check_mark: Authentication successful![/green]")
|
|
408
409
|
except (
|
|
409
410
|
dds_cli.exceptions.APIError,
|
|
410
411
|
dds_cli.exceptions.AuthenticationError,
|
|
@@ -562,7 +563,7 @@ def list_users(click_ctx, unit, invites):
|
|
|
562
563
|
token_path=click_ctx.get("TOKEN_PATH"),
|
|
563
564
|
) as lister:
|
|
564
565
|
if invites:
|
|
565
|
-
lister.list_invites(
|
|
566
|
+
lister.list_invites(invites=invites)
|
|
566
567
|
else:
|
|
567
568
|
lister.list_users(unit=unit)
|
|
568
569
|
|
|
@@ -583,7 +584,7 @@ def list_users(click_ctx, unit, invites):
|
|
|
583
584
|
required=True, help_message="[Super Admins only] The username of the account you want to check."
|
|
584
585
|
)
|
|
585
586
|
@click.pass_obj
|
|
586
|
-
def
|
|
587
|
+
def find_users(click_ctx, username):
|
|
587
588
|
"""Check if a username is registered to an account in the DDS."""
|
|
588
589
|
try:
|
|
589
590
|
with dds_cli.account_manager.AccountManager(
|
|
@@ -621,7 +622,7 @@ def list_users(click_ctx, username):
|
|
|
621
622
|
),
|
|
622
623
|
help=(
|
|
623
624
|
"Type of account. To include a space in the chosen role, use quotes "
|
|
624
|
-
'(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
|
|
625
|
+
r'(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
|
|
625
626
|
),
|
|
626
627
|
)
|
|
627
628
|
@click.option(
|
|
@@ -999,8 +1000,9 @@ def create(
|
|
|
999
1000
|
email_overlap = set(owner) & set(researcher)
|
|
1000
1001
|
if email_overlap:
|
|
1001
1002
|
LOG.info(
|
|
1002
|
-
|
|
1003
|
-
"Please specify a unique role for each email."
|
|
1003
|
+
"The email(s) %s specified as both owner and researcher! "
|
|
1004
|
+
"Please specify a unique role for each email.",
|
|
1005
|
+
email_overlap,
|
|
1004
1006
|
)
|
|
1005
1007
|
sys.exit(1)
|
|
1006
1008
|
if owner:
|
|
@@ -1687,7 +1689,8 @@ def get_data(
|
|
|
1687
1689
|
sys.exit(1)
|
|
1688
1690
|
elif not get_all and not (source or source_path_file):
|
|
1689
1691
|
LOG.error(
|
|
1690
|
-
"Specify either '--source' or '--source-path-file' to download specific directories/files,
|
|
1692
|
+
"Specify either '--source' or '--source-path-file' to download specific directories/files, "
|
|
1693
|
+
"or '--get-all' to download all project contents."
|
|
1691
1694
|
)
|
|
1692
1695
|
sys.exit(1)
|
|
1693
1696
|
|
|
@@ -1726,7 +1729,7 @@ def get_data(
|
|
|
1726
1729
|
|
|
1727
1730
|
# Schedule the first num_threads futures for upload
|
|
1728
1731
|
for file in itertools.islice(iterator, num_threads):
|
|
1729
|
-
LOG.debug(
|
|
1732
|
+
LOG.debug("Starting: %s", rich.markup.escape(str(file)))
|
|
1730
1733
|
# Execute download
|
|
1731
1734
|
download_threads[
|
|
1732
1735
|
texec.submit(getter.download_and_verify, file=file, progress=progress)
|
|
@@ -1742,19 +1745,21 @@ def get_data(
|
|
|
1742
1745
|
|
|
1743
1746
|
for dfut in ddone:
|
|
1744
1747
|
downloaded_file = download_threads.pop(dfut)
|
|
1745
|
-
LOG.debug(
|
|
1746
|
-
f"Future done: {rich.markup.escape(str(downloaded_file))}",
|
|
1747
|
-
)
|
|
1748
|
+
LOG.debug("Future done: %s", rich.markup.escape(str(downloaded_file)))
|
|
1748
1749
|
|
|
1749
1750
|
# Get result
|
|
1750
1751
|
try:
|
|
1751
1752
|
file_downloaded = dfut.result()
|
|
1752
1753
|
LOG.debug(
|
|
1753
|
-
|
|
1754
|
+
"Download of %s successful: %s",
|
|
1755
|
+
rich.markup.escape(str(downloaded_file)),
|
|
1756
|
+
file_downloaded,
|
|
1754
1757
|
)
|
|
1755
1758
|
except concurrent.futures.BrokenExecutor as err:
|
|
1756
1759
|
LOG.critical(
|
|
1757
|
-
|
|
1760
|
+
"Download of file %s failed! Error: %s",
|
|
1761
|
+
rich.markup.escape(str(downloaded_file)),
|
|
1762
|
+
err,
|
|
1758
1763
|
)
|
|
1759
1764
|
continue
|
|
1760
1765
|
|
|
@@ -1763,7 +1768,7 @@ def get_data(
|
|
|
1763
1768
|
|
|
1764
1769
|
# Schedule the next set of futures for download
|
|
1765
1770
|
for next_file in itertools.islice(iterator, new_tasks):
|
|
1766
|
-
LOG.debug(
|
|
1771
|
+
LOG.debug("Starting: %s", rich.markup.escape(str(next_file)))
|
|
1767
1772
|
# Execute download
|
|
1768
1773
|
download_threads[
|
|
1769
1774
|
texec.submit(
|
|
@@ -1868,7 +1873,7 @@ def rm_data(click_ctx, project, file, folder, rm_all):
|
|
|
1868
1873
|
# Warn if trying to remove all contents
|
|
1869
1874
|
if rm_all:
|
|
1870
1875
|
if no_prompt:
|
|
1871
|
-
LOG.warning(
|
|
1876
|
+
LOG.warning("Deleting all files within project '%s'", project)
|
|
1872
1877
|
else:
|
|
1873
1878
|
if not rich.prompt.Confirm.ask(
|
|
1874
1879
|
f"Are you sure you want to delete all files within project '{project}'?"
|
|
@@ -2103,20 +2108,20 @@ def get_stats(click_ctx, stat_type):
|
|
|
2103
2108
|
token_path=click_ctx.get("TOKEN_PATH"),
|
|
2104
2109
|
) as lister:
|
|
2105
2110
|
# Get projects, only active by default
|
|
2106
|
-
projects: typing.List = lister.list_projects(show_all=
|
|
2111
|
+
projects: typing.List = lister.list_projects(show_all=stat_type == "all")
|
|
2107
2112
|
|
|
2108
2113
|
if stat_type == "size":
|
|
2109
2114
|
# Calculate total amount of saved data in active projects
|
|
2110
2115
|
title_bold_part: str = "Bytes"
|
|
2111
2116
|
title_rest: str = "currently stored in DDS"
|
|
2112
|
-
value: int = sum(
|
|
2117
|
+
value: int = sum(x["Size"] for x in projects)
|
|
2113
2118
|
else:
|
|
2114
2119
|
# Get number of projects
|
|
2115
2120
|
title_bold_part: str = "Active" if stat_type == "active" else "Total"
|
|
2116
2121
|
title_rest: str = "projects"
|
|
2117
2122
|
value: int = len(projects)
|
|
2118
2123
|
|
|
2119
|
-
LOG.info(
|
|
2124
|
+
LOG.info("[bold]%s[/bold] %s: %s", title_bold_part, title_rest, value)
|
|
2120
2125
|
except (
|
|
2121
2126
|
dds_cli.exceptions.APIError,
|
|
2122
2127
|
dds_cli.exceptions.AuthenticationError,
|
|
@@ -5,12 +5,10 @@
|
|
|
5
5
|
###################################################################################################
|
|
6
6
|
|
|
7
7
|
# Standard library
|
|
8
|
-
from email import header
|
|
9
8
|
import logging
|
|
10
9
|
|
|
11
10
|
# Installed
|
|
12
11
|
import rich.markup
|
|
13
|
-
from rich.table import Table
|
|
14
12
|
|
|
15
13
|
# Own modules
|
|
16
14
|
import dds_cli
|
|
@@ -139,11 +137,16 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
139
137
|
info = response.get("info")
|
|
140
138
|
if info:
|
|
141
139
|
LOG.info(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
"Username: %s \n"
|
|
141
|
+
"Role: %s \n"
|
|
142
|
+
"Name: %s \n"
|
|
143
|
+
"Primary Email: %s \n"
|
|
144
|
+
"Associated Emails: %s \n",
|
|
145
|
+
info["username"],
|
|
146
|
+
info["role"],
|
|
147
|
+
info["name"],
|
|
148
|
+
info["email_primary"],
|
|
149
|
+
", ".join(str(x) for x in info["emails_all"]),
|
|
147
150
|
)
|
|
148
151
|
|
|
149
152
|
def user_activation(self, email, action):
|
|
@@ -173,7 +176,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
173
176
|
)
|
|
174
177
|
|
|
175
178
|
if project_errors:
|
|
176
|
-
LOG.warning(
|
|
179
|
+
LOG.warning("Could not fix user '%s' access to the following projects:", email)
|
|
177
180
|
msg = project_errors
|
|
178
181
|
else:
|
|
179
182
|
msg = response_json.get(
|
|
@@ -197,7 +200,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
197
200
|
)
|
|
198
201
|
|
|
199
202
|
if response.get("empty"):
|
|
200
|
-
LOG.info(
|
|
203
|
+
LOG.info("There are no Unit Admins or Unit Personnel connected to unit '%s'", unit)
|
|
201
204
|
return
|
|
202
205
|
|
|
203
206
|
users, keys = dds_cli.utils.get_required_in_response(
|
|
@@ -226,7 +229,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
226
229
|
# Print out table
|
|
227
230
|
dds_cli.utils.print_or_page(item=table)
|
|
228
231
|
|
|
229
|
-
def list_invites(self,
|
|
232
|
+
def list_invites(self, invites: bool = None) -> None:
|
|
230
233
|
"""List all unit users within a specific unit."""
|
|
231
234
|
response, _ = dds_cli.utils.perform_request(
|
|
232
235
|
endpoint=dds_cli.DDSEndpoint.LIST_INVITED_USERS,
|
|
@@ -239,7 +242,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
239
242
|
invites = response.get("invites")
|
|
240
243
|
|
|
241
244
|
if not invites:
|
|
242
|
-
LOG.info(
|
|
245
|
+
LOG.info("There are no current invites")
|
|
243
246
|
return
|
|
244
247
|
|
|
245
248
|
table = dds_cli.utils.create_table(
|
|
@@ -269,5 +272,5 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
269
272
|
)
|
|
270
273
|
|
|
271
274
|
LOG.info(
|
|
272
|
-
|
|
275
|
+
"Account exists: [bold]%s[/bold]", "[blue]Yes[/blue]" if exists else "[red]No[/red]"
|
|
273
276
|
)
|
|
@@ -4,7 +4,7 @@ import logging
|
|
|
4
4
|
import getpass
|
|
5
5
|
|
|
6
6
|
# Installed
|
|
7
|
-
import
|
|
7
|
+
from rich.prompt import Prompt
|
|
8
8
|
|
|
9
9
|
# Own modules
|
|
10
10
|
import dds_cli
|
|
@@ -48,6 +48,7 @@ class Auth(base.DDSBaseClass):
|
|
|
48
48
|
)
|
|
49
49
|
|
|
50
50
|
def check(self):
|
|
51
|
+
"""Check if token exists and return info."""
|
|
51
52
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
52
53
|
if token_file.file_exists():
|
|
53
54
|
token = token_file.read_token()
|
|
@@ -55,10 +56,12 @@ class Auth(base.DDSBaseClass):
|
|
|
55
56
|
token_file.token_report(token=token)
|
|
56
57
|
else:
|
|
57
58
|
LOG.info(
|
|
58
|
-
"[red]No saved token found, or token has expired.
|
|
59
|
+
"[red]No saved token found, or token has expired. "
|
|
60
|
+
"Authenticate yourself with `dds auth login` to use this functionality![/red]"
|
|
59
61
|
)
|
|
60
62
|
|
|
61
63
|
def logout(self):
|
|
64
|
+
"""Logout user by removing authenticated token."""
|
|
62
65
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
63
66
|
if token_file.file_exists():
|
|
64
67
|
token_file.delete_token()
|
|
@@ -67,6 +70,7 @@ class Auth(base.DDSBaseClass):
|
|
|
67
70
|
LOG.info("[green]Already logged out![/green]")
|
|
68
71
|
|
|
69
72
|
def twofactor(self, auth_method: str = None):
|
|
73
|
+
"""Perform 2FA for user."""
|
|
70
74
|
if auth_method == "totp":
|
|
71
75
|
response_json, _ = dds_cli.utils.perform_request(
|
|
72
76
|
endpoint=dds_cli.DDSEndpoint.USER_ACTIVATE_TOTP,
|
|
@@ -78,7 +82,7 @@ class Auth(base.DDSBaseClass):
|
|
|
78
82
|
LOG.info(
|
|
79
83
|
"Activating authentication via email, please (re-)enter your username and password:"
|
|
80
84
|
)
|
|
81
|
-
username: str =
|
|
85
|
+
username: str = Prompt.ask("DDS username")
|
|
82
86
|
password: str = getpass.getpass(prompt="DDS password: ")
|
|
83
87
|
|
|
84
88
|
if password == "":
|
|
@@ -95,6 +99,7 @@ class Auth(base.DDSBaseClass):
|
|
|
95
99
|
LOG.info(response_json.get("message"))
|
|
96
100
|
|
|
97
101
|
def deactivate(self, username: str = None):
|
|
102
|
+
"""Deactivate TOTP for user."""
|
|
98
103
|
response_json, _ = dds_cli.utils.perform_request(
|
|
99
104
|
endpoint=dds_cli.DDSEndpoint.TOTP_DEACTIVATE,
|
|
100
105
|
headers=self.token,
|
|
@@ -6,14 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
# Standard library
|
|
8
8
|
import logging
|
|
9
|
-
import os
|
|
10
9
|
import pathlib
|
|
11
10
|
import typing
|
|
12
11
|
|
|
13
12
|
# Installed
|
|
14
|
-
import http
|
|
15
|
-
import time
|
|
16
|
-
import simplejson
|
|
17
13
|
from rich.progress import Progress, SpinnerColumn
|
|
18
14
|
|
|
19
15
|
# Own modules
|
|
@@ -71,7 +67,7 @@ class DDSBaseClass:
|
|
|
71
67
|
# Get attempted operation e.g. put/ls/rm/get
|
|
72
68
|
if self.method not in DDS_METHODS:
|
|
73
69
|
raise exceptions.InvalidMethodError(attempted_method=self.method)
|
|
74
|
-
LOG.debug(
|
|
70
|
+
LOG.debug("Attempted operation: %s", self.method)
|
|
75
71
|
|
|
76
72
|
# Use user defined destination if any specified
|
|
77
73
|
if self.method in DDS_DIR_REQUIRED_METHODS:
|
|
@@ -114,14 +110,14 @@ class DDSBaseClass:
|
|
|
114
110
|
|
|
115
111
|
self.keys = self.__get_project_keys()
|
|
116
112
|
|
|
117
|
-
self.status =
|
|
113
|
+
self.status: typing.Dict = {}
|
|
118
114
|
self.filehandler = None
|
|
119
115
|
|
|
120
116
|
def __enter__(self):
|
|
121
117
|
"""Return self when using context manager."""
|
|
122
118
|
return self
|
|
123
119
|
|
|
124
|
-
def __exit__(self,
|
|
120
|
+
def __exit__(self, exception_type, exception_value, traceback, max_fileerrs: int = 40):
|
|
125
121
|
"""Finish and print out delivery summary.
|
|
126
122
|
|
|
127
123
|
This is not entered if there's an error during __init__.
|
|
@@ -131,8 +127,8 @@ class DDSBaseClass:
|
|
|
131
127
|
self.__printout_delivery_summary()
|
|
132
128
|
|
|
133
129
|
# Exception is not handled
|
|
134
|
-
if
|
|
135
|
-
LOG.debug(
|
|
130
|
+
if exception_type is not None:
|
|
131
|
+
LOG.debug("Exception: %s with value %s", exception_type, exception_value)
|
|
136
132
|
return False
|
|
137
133
|
|
|
138
134
|
return True
|
|
@@ -188,7 +184,7 @@ class DDSBaseClass:
|
|
|
188
184
|
def __printout_delivery_summary(self):
|
|
189
185
|
"""Print out the delivery summary if any files were cancelled."""
|
|
190
186
|
if self.stop_doing:
|
|
191
|
-
LOG.info(
|
|
187
|
+
LOG.info("%s cancelled.\n", "Upload" if self.method == "put" else "Download")
|
|
192
188
|
return
|
|
193
189
|
|
|
194
190
|
# TODO: Look into a better summary print out - old deleted for now
|
|
@@ -210,21 +206,23 @@ class DDSBaseClass:
|
|
|
210
206
|
f"Please verify that the following error log has been generated: {self.failed_delivery_log}\n"
|
|
211
207
|
"[red][bold]Do not[/bold][/red] delete this file; The Data Centre may need it during DDS support."
|
|
212
208
|
)
|
|
213
|
-
else:
|
|
214
|
-
# TODO: --destination should be able to >at least< overwrite the files in the
|
|
215
|
-
# previously created download location.
|
|
216
|
-
raise exceptions.DownloadError(
|
|
217
|
-
"Errors occurred during download.\n"
|
|
218
|
-
"If you wish to retry the download, re-run the `dds data get` command again, "
|
|
219
|
-
"specifying the same options as you did now. A new directory will "
|
|
220
|
-
"automatically be created and all files will be downloaded again.\n\n"
|
|
221
|
-
f"See {self.failed_delivery_log} for more information."
|
|
222
|
-
)
|
|
223
209
|
|
|
224
|
-
|
|
210
|
+
# TODO: --destination should be able to >at least< overwrite the files in the
|
|
211
|
+
# previously created download location.
|
|
212
|
+
raise exceptions.DownloadError(
|
|
213
|
+
"Errors occurred during download.\n"
|
|
214
|
+
"If you wish to retry the download, re-run the `dds data get` command again, "
|
|
215
|
+
"specifying the same options as you did now. A new directory will "
|
|
216
|
+
"automatically be created and all files will be downloaded again.\n\n"
|
|
217
|
+
f"See {self.failed_delivery_log} for more information."
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if nr_uploaded:
|
|
225
221
|
# Raise exception in order to give exit code 1
|
|
226
222
|
LOG.warning(
|
|
227
|
-
|
|
223
|
+
"%s files have already been uploaded to this project.\n"
|
|
224
|
+
"Upload [bold]partially[/bold] completed!\n",
|
|
225
|
+
nr_uploaded,
|
|
228
226
|
)
|
|
229
227
|
|
|
230
228
|
else:
|
|
@@ -234,7 +232,7 @@ class DDSBaseClass:
|
|
|
234
232
|
)
|
|
235
233
|
|
|
236
234
|
if self.method == "get" and len(self.filehandler.data) > len(any_failed):
|
|
237
|
-
LOG.info(
|
|
235
|
+
LOG.info("Any downloaded files are located at: %s.", self.filehandler.local_destination)
|
|
238
236
|
|
|
239
237
|
def __collect_all_failed(self, sort: bool = True) -> list:
|
|
240
238
|
"""Put cancelled files from status in to failed dict and sort the output."""
|
|
@@ -52,13 +52,13 @@ def verify_proceed(func):
|
|
|
52
52
|
|
|
53
53
|
# Mark as started
|
|
54
54
|
self.status[file]["started"] = True
|
|
55
|
-
LOG.debug(
|
|
55
|
+
LOG.debug("File '%s' started: %s", escape(str(file)), func.__name__)
|
|
56
56
|
|
|
57
57
|
# Run function
|
|
58
58
|
ok_to_proceed, message = func(self, file=file, *args, **kwargs)
|
|
59
59
|
# Cancel file(s) if something failed
|
|
60
60
|
if not ok_to_proceed:
|
|
61
|
-
LOG.warning(
|
|
61
|
+
LOG.warning("%s failed: %s", func.__name__, message)
|
|
62
62
|
self.status[file].update({"cancel": True, "message": message})
|
|
63
63
|
if self.status[file].get("failed_op") is None:
|
|
64
64
|
self.status[file]["failed_op"] = "crypto"
|
|
@@ -91,13 +91,17 @@ def update_status(func):
|
|
|
91
91
|
def wrapped(self, file, *args, **kwargs):
|
|
92
92
|
# TODO (ina): add processing?
|
|
93
93
|
if func.__name__ not in ["put", "add_file_db", "get", "update_db"]:
|
|
94
|
-
raise
|
|
94
|
+
raise dds_cli.exceptions.DDSCLIException(
|
|
95
|
+
f"The function {func.__name__} cannot be used with this decorator."
|
|
96
|
+
)
|
|
95
97
|
if func.__name__ not in self.status[file]:
|
|
96
|
-
raise
|
|
98
|
+
raise dds_cli.exceptions.DDSCLIException(
|
|
99
|
+
f"No status found for function {func.__name__}."
|
|
100
|
+
)
|
|
97
101
|
|
|
98
102
|
# Update status to started
|
|
99
103
|
self.status[file][func.__name__].update({"started": True})
|
|
100
|
-
LOG.debug(
|
|
104
|
+
LOG.debug("File '%s' status updated to %s: started", escape(str(file)), func.__name__)
|
|
101
105
|
|
|
102
106
|
# Run function
|
|
103
107
|
ok_to_continue, message, *_ = func(self, file=file, *args, **kwargs)
|
|
@@ -107,12 +111,12 @@ def update_status(func):
|
|
|
107
111
|
# Save info about which operation failed
|
|
108
112
|
|
|
109
113
|
self.status[file]["failed_op"] = func.__name__
|
|
110
|
-
LOG.warning(
|
|
114
|
+
LOG.warning("%s failed: %s", func.__name__, message)
|
|
111
115
|
|
|
112
116
|
else:
|
|
113
117
|
# Update status to done
|
|
114
118
|
self.status[file][func.__name__].update({"done": True})
|
|
115
|
-
LOG.debug(
|
|
119
|
+
LOG.debug("File %s status updated to %s: done", escape(str(file)), func.__name__)
|
|
116
120
|
|
|
117
121
|
return ok_to_continue, message
|
|
118
122
|
|
|
@@ -137,7 +141,7 @@ def subpath_required(func):
|
|
|
137
141
|
except OSError as err:
|
|
138
142
|
return False, str(err)
|
|
139
143
|
|
|
140
|
-
LOG.debug(
|
|
144
|
+
LOG.debug("New directory created: '%s'", full_subpath)
|
|
141
145
|
|
|
142
146
|
return func(self, file=file, *args, **kwargs)
|
|
143
147
|
|
|
@@ -149,8 +153,6 @@ def removal_spinner(func):
|
|
|
149
153
|
|
|
150
154
|
@functools.wraps(func)
|
|
151
155
|
def create_and_remove_task(self, *args, **kwargs):
|
|
152
|
-
message = ""
|
|
153
|
-
|
|
154
156
|
with Progress(
|
|
155
157
|
"[bold]{task.description}",
|
|
156
158
|
SpinnerColumn(spinner_name="dots12", style="white"),
|
|
@@ -186,7 +188,7 @@ def removal_spinner(func):
|
|
|
186
188
|
dds_cli.utils.console.print(self.failed_table)
|
|
187
189
|
else:
|
|
188
190
|
dds_cli.utils.console.print(self.failed_table)
|
|
189
|
-
LOG.warning(
|
|
191
|
+
LOG.warning("Finished %s with errors, see table above", description_lc)
|
|
190
192
|
elif self.failed_files is not None:
|
|
191
193
|
self.failed_files["result"] = f"Finished {description_lc} with errors"
|
|
192
194
|
dds_cli.utils.console.print(self.failed_files)
|
|
@@ -10,7 +10,6 @@ import pathlib
|
|
|
10
10
|
|
|
11
11
|
# Installed
|
|
12
12
|
import requests
|
|
13
|
-
import simplejson
|
|
14
13
|
from rich.markup import escape
|
|
15
14
|
from rich.progress import Progress, SpinnerColumn
|
|
16
15
|
|
|
@@ -101,7 +100,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
101
100
|
|
|
102
101
|
if not self.filehandler.data:
|
|
103
102
|
if self.temporary_directory and self.temporary_directory.is_dir():
|
|
104
|
-
LOG.debug(
|
|
103
|
+
LOG.debug("Deleting temporary folder '%s'.", self.temporary_directory)
|
|
105
104
|
dds_cli.utils.delete_folder(self.temporary_directory)
|
|
106
105
|
raise dds_cli.exceptions.DownloadError("No files to download.")
|
|
107
106
|
|
|
@@ -134,13 +133,13 @@ class DataGetter(base.DDSBaseClass):
|
|
|
134
133
|
total=file_info["size_original"],
|
|
135
134
|
)
|
|
136
135
|
|
|
137
|
-
LOG.debug(
|
|
136
|
+
LOG.debug("File '%s' downloaded: %s", escape(str(file)), file_downloaded)
|
|
138
137
|
|
|
139
138
|
if file_downloaded:
|
|
140
139
|
db_updated, message = self.update_db(file=file)
|
|
141
|
-
LOG.debug(
|
|
140
|
+
LOG.debug("Database updated: %s", db_updated)
|
|
142
141
|
|
|
143
|
-
LOG.debug(
|
|
142
|
+
LOG.debug("Beginning decryption of file '%s'...", escape(str(file)))
|
|
144
143
|
file_saved = False
|
|
145
144
|
with fe.Decryptor(
|
|
146
145
|
project_keys=self.keys,
|
|
@@ -160,7 +159,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
160
159
|
outfile=file,
|
|
161
160
|
)
|
|
162
161
|
|
|
163
|
-
LOG.debug(
|
|
162
|
+
LOG.debug("File saved? %s", file_saved)
|
|
164
163
|
if file_saved:
|
|
165
164
|
# TODO (ina): decide on checksum verification method --
|
|
166
165
|
# this checks original, the other is generated from compressed
|
|
@@ -184,6 +183,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
184
183
|
file_remote = self.filehandler.data[file]["url"]
|
|
185
184
|
|
|
186
185
|
try:
|
|
186
|
+
# TODO: Set timeout? (pylint)
|
|
187
187
|
with requests.get(file_remote, stream=True) as req:
|
|
188
188
|
req.raise_for_status()
|
|
189
189
|
with file_local.open(mode="wb") as new_file:
|
|
@@ -214,7 +214,6 @@ class DataGetter(base.DDSBaseClass):
|
|
|
214
214
|
def update_db(self, file):
|
|
215
215
|
"""Update file info in db."""
|
|
216
216
|
updated_in_db = False
|
|
217
|
-
error = ""
|
|
218
217
|
|
|
219
218
|
# Get file info
|
|
220
219
|
fileinfo = self.filehandler.data[file]
|