dds-cli 2.2.63__tar.gz → 2.2.65__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dds_cli-2.2.63 → dds_cli-2.2.65}/PKG-INFO +1 -1
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/__init__.py +2 -1
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/__main__.py +72 -23
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/account_manager.py +15 -12
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/auth.py +8 -3
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/base.py +21 -23
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/custom_decorators.py +13 -11
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_getter.py +6 -7
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_lister.py +64 -64
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_putter.py +19 -14
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_remover.py +14 -12
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/directory.py +5 -4
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_compressor.py +5 -5
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_encryptor.py +7 -7
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler.py +4 -5
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler_local.py +20 -18
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler_remote.py +3 -9
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/motd_manager.py +2 -7
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/options.py +9 -2
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_creator.py +3 -4
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_info.py +19 -11
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_status.py +19 -20
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/s3_connector.py +5 -6
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/status.py +1 -1
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/timestamp.py +7 -5
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/unit_manager.py +0 -5
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/user.py +34 -25
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/utils.py +18 -13
- dds_cli-2.2.65/dds_cli/version.py +3 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/PKG-INFO +1 -1
- {dds_cli-2.2.63 → dds_cli-2.2.65}/setup.py +4 -2
- dds_cli-2.2.63/dds_cli/version.py +0 -1
- {dds_cli-2.2.63 → dds_cli-2.2.65}/LICENSE +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/README.md +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/exceptions.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/maintenance_manager.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/text_handler.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/SOURCES.txt +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/dependency_links.txt +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/entry_points.txt +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/not-zip-safe +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/requires.txt +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/top_level.txt +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/pyproject.toml +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/setup.cfg +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/__init__.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_account_manager.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_data_remover.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_compressor.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_encryptor.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_handler_local.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_maintenance_manager.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_motd_manager.py +0 -0
- {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_utils.py +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# pylint: skip-file
|
|
1
2
|
"""DDS CLI."""
|
|
2
3
|
|
|
3
4
|
import datetime
|
|
@@ -59,7 +60,7 @@ class DDSEndpoint:
|
|
|
59
60
|
BASE_ENDPOINT_LOCAL = "http://127.0.0.1:5000/api/v1"
|
|
60
61
|
BASE_ENDPOINT_DOCKER = "http://dds_backend:5000/api/v1"
|
|
61
62
|
BASE_ENDPOINT_REMOTE = "https://delivery.scilifelab.se/api/v1"
|
|
62
|
-
BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.
|
|
63
|
+
BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.dckube3.scilifelab.se/api/v1"
|
|
63
64
|
if os.getenv("DDS_CLI_ENV") == "development":
|
|
64
65
|
BASE_ENDPOINT = BASE_ENDPOINT_LOCAL
|
|
65
66
|
elif os.getenv("DDS_CLI_ENV") == "docker-dev":
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
import concurrent.futures
|
|
9
9
|
import itertools
|
|
10
10
|
import logging
|
|
11
|
-
import os
|
|
12
11
|
import sys
|
|
12
|
+
import typing
|
|
13
13
|
|
|
14
14
|
# Installed
|
|
15
15
|
import pathlib
|
|
@@ -70,7 +70,6 @@ LOG = logging.getLogger("dds_cli")
|
|
|
70
70
|
# Configuration for rich-click output
|
|
71
71
|
click.rich_click.MAX_WIDTH = 100
|
|
72
72
|
|
|
73
|
-
|
|
74
73
|
## # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
|
75
74
|
# #
|
|
76
75
|
# MMMM MMMM AAAA II NNNN NN #
|
|
@@ -82,15 +81,15 @@ click.rich_click.MAX_WIDTH = 100
|
|
|
82
81
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ##
|
|
83
82
|
|
|
84
83
|
|
|
85
|
-
|
|
84
|
+
DDS_URL = dds_cli.DDSEndpoint.BASE_ENDPOINT
|
|
85
|
+
DDS_URL_BASE = DDS_URL[: DDS_URL.index("/", 8)]
|
|
86
|
+
|
|
86
87
|
# Print header to STDERR
|
|
87
88
|
dds_cli.utils.stderr_console.print(
|
|
88
89
|
"[green] ︵",
|
|
89
90
|
"\n[green] ︵ ( ) ︵",
|
|
90
91
|
"\n[green]( ) ) ( ( )[/] [bold]SciLifeLab Data Delivery System",
|
|
91
|
-
"\n[green] ︶ ( ) ) ([/] [blue][link={
|
|
92
|
-
dds_url[: dds_url.index("/", 8)]
|
|
93
|
-
),
|
|
92
|
+
f"\n[green] ︶ ( ) ) ([/] [blue][link={DDS_URL_BASE}]{DDS_URL_BASE}/[/link]",
|
|
94
93
|
f"\n[green] ︶ ( )[/] [dim]CLI Version {dds_cli.__version__}",
|
|
95
94
|
"\n[green] ︶",
|
|
96
95
|
highlight=False,
|
|
@@ -99,7 +98,7 @@ dds_cli.utils.stderr_console.print(
|
|
|
99
98
|
if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] != "motd"):
|
|
100
99
|
motds = dds_cli.motd_manager.MotdManager.list_all_active_motds(table=False)
|
|
101
100
|
if motds:
|
|
102
|
-
dds_cli.utils.stderr_console.print(
|
|
101
|
+
dds_cli.utils.stderr_console.print("[bold]Important information:[/bold]")
|
|
103
102
|
for motd in motds:
|
|
104
103
|
dds_cli.utils.stderr_console.print(f"{motd['Created']} - {motd['Message']} \n")
|
|
105
104
|
|
|
@@ -379,7 +378,10 @@ def auth_group_command(_):
|
|
|
379
378
|
is_flag=True,
|
|
380
379
|
required=False,
|
|
381
380
|
default=False,
|
|
382
|
-
help=
|
|
381
|
+
help=(
|
|
382
|
+
"[Not recommended, use with care] Allow read permissions to group. Sets 640 permission instead of 600, "
|
|
383
|
+
"allowing others to access your authenticated session token."
|
|
384
|
+
),
|
|
383
385
|
)
|
|
384
386
|
@click.pass_obj
|
|
385
387
|
def login(click_ctx, totp, allow_group):
|
|
@@ -403,7 +405,7 @@ def login(click_ctx, totp, allow_group):
|
|
|
403
405
|
token_path=click_ctx.get("TOKEN_PATH"), totp=totp, allow_group=allow_group
|
|
404
406
|
):
|
|
405
407
|
# Authentication token renewed in the init method.
|
|
406
|
-
LOG.info(
|
|
408
|
+
LOG.info("[green] :white_check_mark: Authentication successful![/green]")
|
|
407
409
|
except (
|
|
408
410
|
dds_cli.exceptions.APIError,
|
|
409
411
|
dds_cli.exceptions.AuthenticationError,
|
|
@@ -561,7 +563,7 @@ def list_users(click_ctx, unit, invites):
|
|
|
561
563
|
token_path=click_ctx.get("TOKEN_PATH"),
|
|
562
564
|
) as lister:
|
|
563
565
|
if invites:
|
|
564
|
-
lister.list_invites(
|
|
566
|
+
lister.list_invites(invites=invites)
|
|
565
567
|
else:
|
|
566
568
|
lister.list_users(unit=unit)
|
|
567
569
|
|
|
@@ -582,7 +584,7 @@ def list_users(click_ctx, unit, invites):
|
|
|
582
584
|
required=True, help_message="[Super Admins only] The username of the account you want to check."
|
|
583
585
|
)
|
|
584
586
|
@click.pass_obj
|
|
585
|
-
def
|
|
587
|
+
def find_users(click_ctx, username):
|
|
586
588
|
"""Check if a username is registered to an account in the DDS."""
|
|
587
589
|
try:
|
|
588
590
|
with dds_cli.account_manager.AccountManager(
|
|
@@ -620,7 +622,7 @@ def list_users(click_ctx, username):
|
|
|
620
622
|
),
|
|
621
623
|
help=(
|
|
622
624
|
"Type of account. To include a space in the chosen role, use quotes "
|
|
623
|
-
'(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
|
|
625
|
+
r'(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
|
|
624
626
|
),
|
|
625
627
|
)
|
|
626
628
|
@click.option(
|
|
@@ -998,8 +1000,9 @@ def create(
|
|
|
998
1000
|
email_overlap = set(owner) & set(researcher)
|
|
999
1001
|
if email_overlap:
|
|
1000
1002
|
LOG.info(
|
|
1001
|
-
|
|
1002
|
-
"Please specify a unique role for each email."
|
|
1003
|
+
"The email(s) %s specified as both owner and researcher! "
|
|
1004
|
+
"Please specify a unique role for each email.",
|
|
1005
|
+
email_overlap,
|
|
1003
1006
|
)
|
|
1004
1007
|
sys.exit(1)
|
|
1005
1008
|
if owner:
|
|
@@ -1686,7 +1689,8 @@ def get_data(
|
|
|
1686
1689
|
sys.exit(1)
|
|
1687
1690
|
elif not get_all and not (source or source_path_file):
|
|
1688
1691
|
LOG.error(
|
|
1689
|
-
"Specify either '--source' or '--source-path-file' to download specific directories/files,
|
|
1692
|
+
"Specify either '--source' or '--source-path-file' to download specific directories/files, "
|
|
1693
|
+
"or '--get-all' to download all project contents."
|
|
1690
1694
|
)
|
|
1691
1695
|
sys.exit(1)
|
|
1692
1696
|
|
|
@@ -1725,7 +1729,7 @@ def get_data(
|
|
|
1725
1729
|
|
|
1726
1730
|
# Schedule the first num_threads futures for upload
|
|
1727
1731
|
for file in itertools.islice(iterator, num_threads):
|
|
1728
|
-
LOG.debug(
|
|
1732
|
+
LOG.debug("Starting: %s", rich.markup.escape(str(file)))
|
|
1729
1733
|
# Execute download
|
|
1730
1734
|
download_threads[
|
|
1731
1735
|
texec.submit(getter.download_and_verify, file=file, progress=progress)
|
|
@@ -1741,19 +1745,21 @@ def get_data(
|
|
|
1741
1745
|
|
|
1742
1746
|
for dfut in ddone:
|
|
1743
1747
|
downloaded_file = download_threads.pop(dfut)
|
|
1744
|
-
LOG.debug(
|
|
1745
|
-
f"Future done: {rich.markup.escape(str(downloaded_file))}",
|
|
1746
|
-
)
|
|
1748
|
+
LOG.debug("Future done: %s", rich.markup.escape(str(downloaded_file)))
|
|
1747
1749
|
|
|
1748
1750
|
# Get result
|
|
1749
1751
|
try:
|
|
1750
1752
|
file_downloaded = dfut.result()
|
|
1751
1753
|
LOG.debug(
|
|
1752
|
-
|
|
1754
|
+
"Download of %s successful: %s",
|
|
1755
|
+
rich.markup.escape(str(downloaded_file)),
|
|
1756
|
+
file_downloaded,
|
|
1753
1757
|
)
|
|
1754
1758
|
except concurrent.futures.BrokenExecutor as err:
|
|
1755
1759
|
LOG.critical(
|
|
1756
|
-
|
|
1760
|
+
"Download of file %s failed! Error: %s",
|
|
1761
|
+
rich.markup.escape(str(downloaded_file)),
|
|
1762
|
+
err,
|
|
1757
1763
|
)
|
|
1758
1764
|
continue
|
|
1759
1765
|
|
|
@@ -1762,7 +1768,7 @@ def get_data(
|
|
|
1762
1768
|
|
|
1763
1769
|
# Schedule the next set of futures for download
|
|
1764
1770
|
for next_file in itertools.islice(iterator, new_tasks):
|
|
1765
|
-
LOG.debug(
|
|
1771
|
+
LOG.debug("Starting: %s", rich.markup.escape(str(next_file)))
|
|
1766
1772
|
# Execute download
|
|
1767
1773
|
download_threads[
|
|
1768
1774
|
texec.submit(
|
|
@@ -1867,7 +1873,7 @@ def rm_data(click_ctx, project, file, folder, rm_all):
|
|
|
1867
1873
|
# Warn if trying to remove all contents
|
|
1868
1874
|
if rm_all:
|
|
1869
1875
|
if no_prompt:
|
|
1870
|
-
LOG.warning(
|
|
1876
|
+
LOG.warning("Deleting all files within project '%s'", project)
|
|
1871
1877
|
else:
|
|
1872
1878
|
if not rich.prompt.Confirm.ask(
|
|
1873
1879
|
f"Are you sure you want to delete all files within project '{project}'?"
|
|
@@ -2081,3 +2087,46 @@ def set_maintenance_mode(click_ctx, setting):
|
|
|
2081
2087
|
) as err:
|
|
2082
2088
|
LOG.error(err)
|
|
2083
2089
|
sys.exit(1)
|
|
2090
|
+
|
|
2091
|
+
|
|
2092
|
+
# Stats
|
|
2093
|
+
|
|
2094
|
+
|
|
2095
|
+
@dds_main.command(name="stats", no_args_is_help=False)
|
|
2096
|
+
@click.argument(
|
|
2097
|
+
"stat_type", nargs=1, type=click.Choice(["active", "all", "size"], case_sensitive=True)
|
|
2098
|
+
)
|
|
2099
|
+
@click.pass_obj
|
|
2100
|
+
def get_stats(click_ctx, stat_type):
|
|
2101
|
+
"""Get statistics in the DDS."""
|
|
2102
|
+
try:
|
|
2103
|
+
# Num projects
|
|
2104
|
+
with dds_cli.data_lister.DataLister(
|
|
2105
|
+
show_usage=True,
|
|
2106
|
+
no_prompt=click_ctx.get("NO_PROMPT", False),
|
|
2107
|
+
json=True,
|
|
2108
|
+
token_path=click_ctx.get("TOKEN_PATH"),
|
|
2109
|
+
) as lister:
|
|
2110
|
+
# Get projects, only active by default
|
|
2111
|
+
projects: typing.List = lister.list_projects(show_all=stat_type == "all")
|
|
2112
|
+
|
|
2113
|
+
if stat_type == "size":
|
|
2114
|
+
# Calculate total amount of saved data in active projects
|
|
2115
|
+
title_bold_part: str = "Bytes"
|
|
2116
|
+
title_rest: str = "currently stored in DDS"
|
|
2117
|
+
value: int = sum(x["Size"] for x in projects)
|
|
2118
|
+
else:
|
|
2119
|
+
# Get number of projects
|
|
2120
|
+
title_bold_part: str = "Active" if stat_type == "active" else "Total"
|
|
2121
|
+
title_rest: str = "projects"
|
|
2122
|
+
value: int = len(projects)
|
|
2123
|
+
|
|
2124
|
+
LOG.info("[bold]%s[/bold] %s: %s", title_bold_part, title_rest, value)
|
|
2125
|
+
except (
|
|
2126
|
+
dds_cli.exceptions.APIError,
|
|
2127
|
+
dds_cli.exceptions.AuthenticationError,
|
|
2128
|
+
dds_cli.exceptions.ApiResponseError,
|
|
2129
|
+
dds_cli.exceptions.ApiRequestError,
|
|
2130
|
+
) as err:
|
|
2131
|
+
LOG.error(err)
|
|
2132
|
+
sys.exit(1)
|
|
@@ -5,12 +5,10 @@
|
|
|
5
5
|
###################################################################################################
|
|
6
6
|
|
|
7
7
|
# Standard library
|
|
8
|
-
from email import header
|
|
9
8
|
import logging
|
|
10
9
|
|
|
11
10
|
# Installed
|
|
12
11
|
import rich.markup
|
|
13
|
-
from rich.table import Table
|
|
14
12
|
|
|
15
13
|
# Own modules
|
|
16
14
|
import dds_cli
|
|
@@ -139,11 +137,16 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
139
137
|
info = response.get("info")
|
|
140
138
|
if info:
|
|
141
139
|
LOG.info(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
140
|
+
"Username: %s \n"
|
|
141
|
+
"Role: %s \n"
|
|
142
|
+
"Name: %s \n"
|
|
143
|
+
"Primary Email: %s \n"
|
|
144
|
+
"Associated Emails: %s \n",
|
|
145
|
+
info["username"],
|
|
146
|
+
info["role"],
|
|
147
|
+
info["name"],
|
|
148
|
+
info["email_primary"],
|
|
149
|
+
", ".join(str(x) for x in info["emails_all"]),
|
|
147
150
|
)
|
|
148
151
|
|
|
149
152
|
def user_activation(self, email, action):
|
|
@@ -173,7 +176,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
173
176
|
)
|
|
174
177
|
|
|
175
178
|
if project_errors:
|
|
176
|
-
LOG.warning(
|
|
179
|
+
LOG.warning("Could not fix user '%s' access to the following projects:", email)
|
|
177
180
|
msg = project_errors
|
|
178
181
|
else:
|
|
179
182
|
msg = response_json.get(
|
|
@@ -197,7 +200,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
197
200
|
)
|
|
198
201
|
|
|
199
202
|
if response.get("empty"):
|
|
200
|
-
LOG.info(
|
|
203
|
+
LOG.info("There are no Unit Admins or Unit Personnel connected to unit '%s'", unit)
|
|
201
204
|
return
|
|
202
205
|
|
|
203
206
|
users, keys = dds_cli.utils.get_required_in_response(
|
|
@@ -226,7 +229,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
226
229
|
# Print out table
|
|
227
230
|
dds_cli.utils.print_or_page(item=table)
|
|
228
231
|
|
|
229
|
-
def list_invites(self,
|
|
232
|
+
def list_invites(self, invites: bool = None) -> None:
|
|
230
233
|
"""List all unit users within a specific unit."""
|
|
231
234
|
response, _ = dds_cli.utils.perform_request(
|
|
232
235
|
endpoint=dds_cli.DDSEndpoint.LIST_INVITED_USERS,
|
|
@@ -239,7 +242,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
239
242
|
invites = response.get("invites")
|
|
240
243
|
|
|
241
244
|
if not invites:
|
|
242
|
-
LOG.info(
|
|
245
|
+
LOG.info("There are no current invites")
|
|
243
246
|
return
|
|
244
247
|
|
|
245
248
|
table = dds_cli.utils.create_table(
|
|
@@ -269,5 +272,5 @@ class AccountManager(dds_cli.base.DDSBaseClass):
|
|
|
269
272
|
)
|
|
270
273
|
|
|
271
274
|
LOG.info(
|
|
272
|
-
|
|
275
|
+
"Account exists: [bold]%s[/bold]", "[blue]Yes[/blue]" if exists else "[red]No[/red]"
|
|
273
276
|
)
|
|
@@ -4,7 +4,7 @@ import logging
|
|
|
4
4
|
import getpass
|
|
5
5
|
|
|
6
6
|
# Installed
|
|
7
|
-
import
|
|
7
|
+
from rich.prompt import Prompt
|
|
8
8
|
|
|
9
9
|
# Own modules
|
|
10
10
|
import dds_cli
|
|
@@ -48,6 +48,7 @@ class Auth(base.DDSBaseClass):
|
|
|
48
48
|
)
|
|
49
49
|
|
|
50
50
|
def check(self):
|
|
51
|
+
"""Check if token exists and return info."""
|
|
51
52
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
52
53
|
if token_file.file_exists():
|
|
53
54
|
token = token_file.read_token()
|
|
@@ -55,10 +56,12 @@ class Auth(base.DDSBaseClass):
|
|
|
55
56
|
token_file.token_report(token=token)
|
|
56
57
|
else:
|
|
57
58
|
LOG.info(
|
|
58
|
-
"[red]No saved token found, or token has expired.
|
|
59
|
+
"[red]No saved token found, or token has expired. "
|
|
60
|
+
"Authenticate yourself with `dds auth login` to use this functionality![/red]"
|
|
59
61
|
)
|
|
60
62
|
|
|
61
63
|
def logout(self):
|
|
64
|
+
"""Logout user by removing authenticated token."""
|
|
62
65
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
63
66
|
if token_file.file_exists():
|
|
64
67
|
token_file.delete_token()
|
|
@@ -67,6 +70,7 @@ class Auth(base.DDSBaseClass):
|
|
|
67
70
|
LOG.info("[green]Already logged out![/green]")
|
|
68
71
|
|
|
69
72
|
def twofactor(self, auth_method: str = None):
|
|
73
|
+
"""Perform 2FA for user."""
|
|
70
74
|
if auth_method == "totp":
|
|
71
75
|
response_json, _ = dds_cli.utils.perform_request(
|
|
72
76
|
endpoint=dds_cli.DDSEndpoint.USER_ACTIVATE_TOTP,
|
|
@@ -78,7 +82,7 @@ class Auth(base.DDSBaseClass):
|
|
|
78
82
|
LOG.info(
|
|
79
83
|
"Activating authentication via email, please (re-)enter your username and password:"
|
|
80
84
|
)
|
|
81
|
-
username: str =
|
|
85
|
+
username: str = Prompt.ask("DDS username")
|
|
82
86
|
password: str = getpass.getpass(prompt="DDS password: ")
|
|
83
87
|
|
|
84
88
|
if password == "":
|
|
@@ -95,6 +99,7 @@ class Auth(base.DDSBaseClass):
|
|
|
95
99
|
LOG.info(response_json.get("message"))
|
|
96
100
|
|
|
97
101
|
def deactivate(self, username: str = None):
|
|
102
|
+
"""Deactivate TOTP for user."""
|
|
98
103
|
response_json, _ = dds_cli.utils.perform_request(
|
|
99
104
|
endpoint=dds_cli.DDSEndpoint.TOTP_DEACTIVATE,
|
|
100
105
|
headers=self.token,
|
|
@@ -6,14 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
# Standard library
|
|
8
8
|
import logging
|
|
9
|
-
import os
|
|
10
9
|
import pathlib
|
|
11
10
|
import typing
|
|
12
11
|
|
|
13
12
|
# Installed
|
|
14
|
-
import http
|
|
15
|
-
import time
|
|
16
|
-
import simplejson
|
|
17
13
|
from rich.progress import Progress, SpinnerColumn
|
|
18
14
|
|
|
19
15
|
# Own modules
|
|
@@ -71,7 +67,7 @@ class DDSBaseClass:
|
|
|
71
67
|
# Get attempted operation e.g. put/ls/rm/get
|
|
72
68
|
if self.method not in DDS_METHODS:
|
|
73
69
|
raise exceptions.InvalidMethodError(attempted_method=self.method)
|
|
74
|
-
LOG.debug(
|
|
70
|
+
LOG.debug("Attempted operation: %s", self.method)
|
|
75
71
|
|
|
76
72
|
# Use user defined destination if any specified
|
|
77
73
|
if self.method in DDS_DIR_REQUIRED_METHODS:
|
|
@@ -114,14 +110,14 @@ class DDSBaseClass:
|
|
|
114
110
|
|
|
115
111
|
self.keys = self.__get_project_keys()
|
|
116
112
|
|
|
117
|
-
self.status =
|
|
113
|
+
self.status: typing.Dict = {}
|
|
118
114
|
self.filehandler = None
|
|
119
115
|
|
|
120
116
|
def __enter__(self):
|
|
121
117
|
"""Return self when using context manager."""
|
|
122
118
|
return self
|
|
123
119
|
|
|
124
|
-
def __exit__(self,
|
|
120
|
+
def __exit__(self, exception_type, exception_value, traceback, max_fileerrs: int = 40):
|
|
125
121
|
"""Finish and print out delivery summary.
|
|
126
122
|
|
|
127
123
|
This is not entered if there's an error during __init__.
|
|
@@ -131,8 +127,8 @@ class DDSBaseClass:
|
|
|
131
127
|
self.__printout_delivery_summary()
|
|
132
128
|
|
|
133
129
|
# Exception is not handled
|
|
134
|
-
if
|
|
135
|
-
LOG.debug(
|
|
130
|
+
if exception_type is not None:
|
|
131
|
+
LOG.debug("Exception: %s with value %s", exception_type, exception_value)
|
|
136
132
|
return False
|
|
137
133
|
|
|
138
134
|
return True
|
|
@@ -188,7 +184,7 @@ class DDSBaseClass:
|
|
|
188
184
|
def __printout_delivery_summary(self):
|
|
189
185
|
"""Print out the delivery summary if any files were cancelled."""
|
|
190
186
|
if self.stop_doing:
|
|
191
|
-
LOG.info(
|
|
187
|
+
LOG.info("%s cancelled.\n", "Upload" if self.method == "put" else "Download")
|
|
192
188
|
return
|
|
193
189
|
|
|
194
190
|
# TODO: Look into a better summary print out - old deleted for now
|
|
@@ -210,21 +206,23 @@ class DDSBaseClass:
|
|
|
210
206
|
f"Please verify that the following error log has been generated: {self.failed_delivery_log}\n"
|
|
211
207
|
"[red][bold]Do not[/bold][/red] delete this file; The Data Centre may need it during DDS support."
|
|
212
208
|
)
|
|
213
|
-
else:
|
|
214
|
-
# TODO: --destination should be able to >at least< overwrite the files in the
|
|
215
|
-
# previously created download location.
|
|
216
|
-
raise exceptions.DownloadError(
|
|
217
|
-
"Errors occurred during download.\n"
|
|
218
|
-
"If you wish to retry the download, re-run the `dds data get` command again, "
|
|
219
|
-
"specifying the same options as you did now. A new directory will "
|
|
220
|
-
"automatically be created and all files will be downloaded again.\n\n"
|
|
221
|
-
f"See {self.failed_delivery_log} for more information."
|
|
222
|
-
)
|
|
223
209
|
|
|
224
|
-
|
|
210
|
+
# TODO: --destination should be able to >at least< overwrite the files in the
|
|
211
|
+
# previously created download location.
|
|
212
|
+
raise exceptions.DownloadError(
|
|
213
|
+
"Errors occurred during download.\n"
|
|
214
|
+
"If you wish to retry the download, re-run the `dds data get` command again, "
|
|
215
|
+
"specifying the same options as you did now. A new directory will "
|
|
216
|
+
"automatically be created and all files will be downloaded again.\n\n"
|
|
217
|
+
f"See {self.failed_delivery_log} for more information."
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if nr_uploaded:
|
|
225
221
|
# Raise exception in order to give exit code 1
|
|
226
222
|
LOG.warning(
|
|
227
|
-
|
|
223
|
+
"%s files have already been uploaded to this project.\n"
|
|
224
|
+
"Upload [bold]partially[/bold] completed!\n",
|
|
225
|
+
nr_uploaded,
|
|
228
226
|
)
|
|
229
227
|
|
|
230
228
|
else:
|
|
@@ -234,7 +232,7 @@ class DDSBaseClass:
|
|
|
234
232
|
)
|
|
235
233
|
|
|
236
234
|
if self.method == "get" and len(self.filehandler.data) > len(any_failed):
|
|
237
|
-
LOG.info(
|
|
235
|
+
LOG.info("Any downloaded files are located at: %s.", self.filehandler.local_destination)
|
|
238
236
|
|
|
239
237
|
def __collect_all_failed(self, sort: bool = True) -> list:
|
|
240
238
|
"""Put cancelled files from status in to failed dict and sort the output."""
|
|
@@ -52,13 +52,13 @@ def verify_proceed(func):
|
|
|
52
52
|
|
|
53
53
|
# Mark as started
|
|
54
54
|
self.status[file]["started"] = True
|
|
55
|
-
LOG.debug(
|
|
55
|
+
LOG.debug("File '%s' started: %s", escape(str(file)), func.__name__)
|
|
56
56
|
|
|
57
57
|
# Run function
|
|
58
58
|
ok_to_proceed, message = func(self, file=file, *args, **kwargs)
|
|
59
59
|
# Cancel file(s) if something failed
|
|
60
60
|
if not ok_to_proceed:
|
|
61
|
-
LOG.warning(
|
|
61
|
+
LOG.warning("%s failed: %s", func.__name__, message)
|
|
62
62
|
self.status[file].update({"cancel": True, "message": message})
|
|
63
63
|
if self.status[file].get("failed_op") is None:
|
|
64
64
|
self.status[file]["failed_op"] = "crypto"
|
|
@@ -91,13 +91,17 @@ def update_status(func):
|
|
|
91
91
|
def wrapped(self, file, *args, **kwargs):
|
|
92
92
|
# TODO (ina): add processing?
|
|
93
93
|
if func.__name__ not in ["put", "add_file_db", "get", "update_db"]:
|
|
94
|
-
raise
|
|
94
|
+
raise dds_cli.exceptions.DDSCLIException(
|
|
95
|
+
f"The function {func.__name__} cannot be used with this decorator."
|
|
96
|
+
)
|
|
95
97
|
if func.__name__ not in self.status[file]:
|
|
96
|
-
raise
|
|
98
|
+
raise dds_cli.exceptions.DDSCLIException(
|
|
99
|
+
f"No status found for function {func.__name__}."
|
|
100
|
+
)
|
|
97
101
|
|
|
98
102
|
# Update status to started
|
|
99
103
|
self.status[file][func.__name__].update({"started": True})
|
|
100
|
-
LOG.debug(
|
|
104
|
+
LOG.debug("File '%s' status updated to %s: started", escape(str(file)), func.__name__)
|
|
101
105
|
|
|
102
106
|
# Run function
|
|
103
107
|
ok_to_continue, message, *_ = func(self, file=file, *args, **kwargs)
|
|
@@ -107,12 +111,12 @@ def update_status(func):
|
|
|
107
111
|
# Save info about which operation failed
|
|
108
112
|
|
|
109
113
|
self.status[file]["failed_op"] = func.__name__
|
|
110
|
-
LOG.warning(
|
|
114
|
+
LOG.warning("%s failed: %s", func.__name__, message)
|
|
111
115
|
|
|
112
116
|
else:
|
|
113
117
|
# Update status to done
|
|
114
118
|
self.status[file][func.__name__].update({"done": True})
|
|
115
|
-
LOG.debug(
|
|
119
|
+
LOG.debug("File %s status updated to %s: done", escape(str(file)), func.__name__)
|
|
116
120
|
|
|
117
121
|
return ok_to_continue, message
|
|
118
122
|
|
|
@@ -137,7 +141,7 @@ def subpath_required(func):
|
|
|
137
141
|
except OSError as err:
|
|
138
142
|
return False, str(err)
|
|
139
143
|
|
|
140
|
-
LOG.debug(
|
|
144
|
+
LOG.debug("New directory created: '%s'", full_subpath)
|
|
141
145
|
|
|
142
146
|
return func(self, file=file, *args, **kwargs)
|
|
143
147
|
|
|
@@ -149,8 +153,6 @@ def removal_spinner(func):
|
|
|
149
153
|
|
|
150
154
|
@functools.wraps(func)
|
|
151
155
|
def create_and_remove_task(self, *args, **kwargs):
|
|
152
|
-
message = ""
|
|
153
|
-
|
|
154
156
|
with Progress(
|
|
155
157
|
"[bold]{task.description}",
|
|
156
158
|
SpinnerColumn(spinner_name="dots12", style="white"),
|
|
@@ -186,7 +188,7 @@ def removal_spinner(func):
|
|
|
186
188
|
dds_cli.utils.console.print(self.failed_table)
|
|
187
189
|
else:
|
|
188
190
|
dds_cli.utils.console.print(self.failed_table)
|
|
189
|
-
LOG.warning(
|
|
191
|
+
LOG.warning("Finished %s with errors, see table above", description_lc)
|
|
190
192
|
elif self.failed_files is not None:
|
|
191
193
|
self.failed_files["result"] = f"Finished {description_lc} with errors"
|
|
192
194
|
dds_cli.utils.console.print(self.failed_files)
|
|
@@ -10,7 +10,6 @@ import pathlib
|
|
|
10
10
|
|
|
11
11
|
# Installed
|
|
12
12
|
import requests
|
|
13
|
-
import simplejson
|
|
14
13
|
from rich.markup import escape
|
|
15
14
|
from rich.progress import Progress, SpinnerColumn
|
|
16
15
|
|
|
@@ -101,7 +100,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
101
100
|
|
|
102
101
|
if not self.filehandler.data:
|
|
103
102
|
if self.temporary_directory and self.temporary_directory.is_dir():
|
|
104
|
-
LOG.debug(
|
|
103
|
+
LOG.debug("Deleting temporary folder '%s'.", self.temporary_directory)
|
|
105
104
|
dds_cli.utils.delete_folder(self.temporary_directory)
|
|
106
105
|
raise dds_cli.exceptions.DownloadError("No files to download.")
|
|
107
106
|
|
|
@@ -134,13 +133,13 @@ class DataGetter(base.DDSBaseClass):
|
|
|
134
133
|
total=file_info["size_original"],
|
|
135
134
|
)
|
|
136
135
|
|
|
137
|
-
LOG.debug(
|
|
136
|
+
LOG.debug("File '%s' downloaded: %s", escape(str(file)), file_downloaded)
|
|
138
137
|
|
|
139
138
|
if file_downloaded:
|
|
140
139
|
db_updated, message = self.update_db(file=file)
|
|
141
|
-
LOG.debug(
|
|
140
|
+
LOG.debug("Database updated: %s", db_updated)
|
|
142
141
|
|
|
143
|
-
LOG.debug(
|
|
142
|
+
LOG.debug("Beginning decryption of file '%s'...", escape(str(file)))
|
|
144
143
|
file_saved = False
|
|
145
144
|
with fe.Decryptor(
|
|
146
145
|
project_keys=self.keys,
|
|
@@ -160,7 +159,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
160
159
|
outfile=file,
|
|
161
160
|
)
|
|
162
161
|
|
|
163
|
-
LOG.debug(
|
|
162
|
+
LOG.debug("File saved? %s", file_saved)
|
|
164
163
|
if file_saved:
|
|
165
164
|
# TODO (ina): decide on checksum verification method --
|
|
166
165
|
# this checks original, the other is generated from compressed
|
|
@@ -184,6 +183,7 @@ class DataGetter(base.DDSBaseClass):
|
|
|
184
183
|
file_remote = self.filehandler.data[file]["url"]
|
|
185
184
|
|
|
186
185
|
try:
|
|
186
|
+
# TODO: Set timeout? (pylint)
|
|
187
187
|
with requests.get(file_remote, stream=True) as req:
|
|
188
188
|
req.raise_for_status()
|
|
189
189
|
with file_local.open(mode="wb") as new_file:
|
|
@@ -214,7 +214,6 @@ class DataGetter(base.DDSBaseClass):
|
|
|
214
214
|
def update_db(self, file):
|
|
215
215
|
"""Update file info in db."""
|
|
216
216
|
updated_in_db = False
|
|
217
|
-
error = ""
|
|
218
217
|
|
|
219
218
|
# Get file info
|
|
220
219
|
fileinfo = self.filehandler.data[file]
|