dds-cli 2.11.0__tar.gz → 2.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dds_cli-2.11.0 → dds_cli-2.13.0}/PKG-INFO +5 -4
- {dds_cli-2.11.0 → dds_cli-2.13.0}/README.md +1 -1
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/__init__.py +1 -1
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/__main__.py +52 -12
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/auth.py +75 -13
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/base.py +11 -2
- dds_cli-2.13.0/dds_cli/constants.py +14 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/custom_decorators.py +1 -5
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/data_getter.py +64 -8
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/data_putter.py +17 -8
- dds_cli-2.13.0/dds_cli/dds_gui/__init__.py +6 -0
- dds_cli-2.13.0/dds_cli/dds_gui/app.py +71 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_button.py +65 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_container.py +80 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_footer.py +17 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_form.py +27 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_input.py +19 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_modal.py +138 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_select.py +23 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_status_chip.py +61 -0
- dds_cli-2.13.0/dds_cli/dds_gui/components/dds_text_item.py +17 -0
- dds_cli-2.13.0/dds_cli/dds_gui/dds_state_manager.py +165 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/authentication.py +56 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/authentication_form.py +138 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/modals/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/modals/login_modal.py +35 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/modals/logout_modal.py +28 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/authentication/modals/reauthenticate_modal.py +35 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view.py +69 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view_mode/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view_mode/project_actions.py +32 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/download_data.py +61 -0
- dds_cli-2.13.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/user_access.py +41 -0
- dds_cli-2.13.0/dds_cli/dds_gui/types/__init__.py +1 -0
- dds_cli-2.13.0/dds_cli/dds_gui/types/dds_severity_types.py +12 -0
- dds_cli-2.13.0/dds_cli/dds_gui/types/dds_status_types.py +14 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/exceptions.py +8 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/file_compressor.py +7 -6
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/file_encryptor.py +22 -10
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/file_handler_local.py +8 -3
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/file_handler_remote.py +1 -1
- dds_cli-2.13.0/dds_cli/message_helper.py +72 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/motd_manager.py +41 -38
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/project_status.py +1 -1
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/s3_connector.py +10 -1
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/user.py +132 -93
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/version.py +1 -1
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/PKG-INFO +5 -4
- dds_cli-2.13.0/dds_cli.egg-info/SOURCES.txt +97 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/requires.txt +3 -2
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/top_level.txt +1 -0
- dds_cli-2.13.0/gui_build/gui_standalone.py +11 -0
- dds_cli-2.13.0/tests/__init__.py +0 -0
- dds_cli-2.13.0/tests/gui_tests/__init__.py +0 -0
- dds_cli-2.13.0/tests/gui_tests/test_authentication.py +1567 -0
- dds_cli-2.13.0/tests/gui_tests/test_important_information.py +458 -0
- dds_cli-2.13.0/tests/gui_tests/test_project_content.py +1000 -0
- dds_cli-2.13.0/tests/gui_tests/test_project_information.py +650 -0
- dds_cli-2.13.0/tests/gui_tests/test_project_list.py +558 -0
- dds_cli-2.13.0/tests/test_auth.py +630 -0
- dds_cli-2.13.0/tests/test_base.py +230 -0
- dds_cli-2.13.0/tests/test_data_getter.py +133 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_data_remover.py +17 -20
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_file_compressor.py +29 -33
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_file_encryptor.py +5 -3
- dds_cli-2.13.0/tests/test_file_handler_local.py +104 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_motd_manager.py +90 -37
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_project_status.py +19 -28
- dds_cli-2.13.0/tests/test_s3_connector.py +112 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_superadmin_helper.py +3 -3
- dds_cli-2.13.0/tests/test_user.py +544 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_utils.py +26 -16
- dds_cli-2.11.0/dds_cli.egg-info/SOURCES.txt +0 -52
- dds_cli-2.11.0/tests/test_file_handler_local.py +0 -90
- {dds_cli-2.11.0 → dds_cli-2.13.0}/LICENSE +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/account_manager.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/data_lister.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/data_remover.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/directory.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/file_handler.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/options.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/project_creator.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/project_info.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/status.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/superadmin_helper.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/text_handler.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/timestamp.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/unit_manager.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli/utils.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/dependency_links.txt +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/entry_points.txt +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/dds_cli.egg-info/not-zip-safe +0 -0
- {dds_cli-2.11.0/tests → dds_cli-2.13.0/gui_build}/__init__.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/pyproject.toml +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/setup.cfg +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/setup.py +0 -0
- {dds_cli-2.11.0 → dds_cli-2.13.0}/tests/test_account_manager.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dds_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.13.0
|
|
4
4
|
Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
|
|
5
5
|
Home-page: https://github.com/ScilifelabDataCentre/dds_cli
|
|
6
6
|
Author: SciLifeLab Data Centre
|
|
@@ -18,7 +18,7 @@ Requires-Dist: boto3==1.24.73
|
|
|
18
18
|
Requires-Dist: botocore==1.27.73
|
|
19
19
|
Requires-Dist: click==8.1.3
|
|
20
20
|
Requires-Dist: click-pathlib==2020.3.13.0
|
|
21
|
-
Requires-Dist: cryptography==
|
|
21
|
+
Requires-Dist: cryptography==44.0.1
|
|
22
22
|
Requires-Dist: immutabledict==2.2.1
|
|
23
23
|
Requires-Dist: jwcrypto==1.5.6
|
|
24
24
|
Requires-Dist: prettytable==3.7.0
|
|
@@ -27,12 +27,13 @@ Requires-Dist: PyNaCl==1.5.0
|
|
|
27
27
|
Requires-Dist: pytz==2022.2.1
|
|
28
28
|
Requires-Dist: PyYAML==6.0.2
|
|
29
29
|
Requires-Dist: questionary==1.10.0
|
|
30
|
-
Requires-Dist: requests==2.32.
|
|
30
|
+
Requires-Dist: requests==2.32.4
|
|
31
31
|
Requires-Dist: rich==13.6.0
|
|
32
32
|
Requires-Dist: rich-click==1.5.2
|
|
33
33
|
Requires-Dist: simplejson==3.17.6
|
|
34
34
|
Requires-Dist: tzlocal==4.2
|
|
35
35
|
Requires-Dist: zstandard==0.23.0
|
|
36
|
+
Requires-Dist: textual==2.1.2
|
|
36
37
|
Dynamic: author
|
|
37
38
|
Dynamic: classifier
|
|
38
39
|
Dynamic: description
|
|
@@ -60,7 +61,7 @@ Dynamic: summary
|
|
|
60
61
|
<a href="https://opensource.org/licenses/MIT">
|
|
61
62
|
<img alt="Licence: MIT" src="https://img.shields.io/badge/License-MIT-yellow.svg">
|
|
62
63
|
</a>
|
|
63
|
-
<a href="
|
|
64
|
+
<a href="https://github.com/psf/black">
|
|
64
65
|
<img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg">
|
|
65
66
|
</a>
|
|
66
67
|
<a href="https://prettier.io/">
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<a href="https://opensource.org/licenses/MIT">
|
|
16
16
|
<img alt="Licence: MIT" src="https://img.shields.io/badge/License-MIT-yellow.svg">
|
|
17
17
|
</a>
|
|
18
|
-
<a href="
|
|
18
|
+
<a href="https://github.com/psf/black">
|
|
19
19
|
<img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg">
|
|
20
20
|
</a>
|
|
21
21
|
<a href="https://prettier.io/">
|
|
@@ -44,7 +44,7 @@ DDS_DIR_REQUIRED_METHODS = ["put", "get"]
|
|
|
44
44
|
DDS_KEYS_REQUIRED_METHODS = ["put", "get"]
|
|
45
45
|
|
|
46
46
|
# Token related variables
|
|
47
|
-
TOKEN_FILE = pathlib.Path
|
|
47
|
+
TOKEN_FILE = pathlib.Path.home() / ".dds_cli_token"
|
|
48
48
|
TOKEN_EXPIRATION_WARNING_THRESHOLD = datetime.timedelta(hours=6)
|
|
49
49
|
|
|
50
50
|
|
|
@@ -38,6 +38,7 @@ import dds_cli.project_status
|
|
|
38
38
|
import dds_cli.project_info
|
|
39
39
|
import dds_cli.user
|
|
40
40
|
import dds_cli.utils
|
|
41
|
+
import dds_cli.message_helper
|
|
41
42
|
from dds_cli.options import (
|
|
42
43
|
destination_option,
|
|
43
44
|
email_arg,
|
|
@@ -60,6 +61,8 @@ from dds_cli.options import (
|
|
|
60
61
|
users_flag,
|
|
61
62
|
)
|
|
62
63
|
|
|
64
|
+
# import dds_cli.dds_gui.app
|
|
65
|
+
|
|
63
66
|
####################################################################################################
|
|
64
67
|
# START LOGGING CONFIG ###################################################### START LOGGING CONFIG #
|
|
65
68
|
####################################################################################################
|
|
@@ -95,11 +98,24 @@ dds_cli.utils.stderr_console.print(
|
|
|
95
98
|
)
|
|
96
99
|
|
|
97
100
|
if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] != "motd"):
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
try:
|
|
102
|
+
motds = dds_cli.motd_manager.MotdManager.list_all_active_motds(table=False)
|
|
103
|
+
if motds:
|
|
104
|
+
dds_cli.utils.stderr_console.print("[bold]Important information:[/bold]")
|
|
105
|
+
for motd in motds:
|
|
106
|
+
dds_cli.utils.stderr_console.print(f"{motd['Created']} - {motd['Message']} \n")
|
|
107
|
+
except dds_cli.exceptions.NoMOTDsError as no_motds_err:
|
|
108
|
+
# Print message about no MOTD
|
|
109
|
+
LOG.info(no_motds_err)
|
|
110
|
+
except (
|
|
111
|
+
dds_cli.exceptions.ApiResponseError,
|
|
112
|
+
dds_cli.exceptions.ApiRequestError,
|
|
113
|
+
) as api_err:
|
|
114
|
+
# Avoid breaking CLI startup on MOTD fetch issues
|
|
115
|
+
LOG.debug("Skipping MOTD display due to API error: %s", api_err)
|
|
116
|
+
except dds_cli.exceptions.DDSCLIException as dds_cli_err:
|
|
117
|
+
# Covers 400/403 and other handled DDS CLI errors from perform_request
|
|
118
|
+
LOG.debug("Skipping MOTD display due to DDS error: %s", dds_cli_err)
|
|
103
119
|
|
|
104
120
|
|
|
105
121
|
# -- dds -- #
|
|
@@ -193,6 +209,21 @@ def dds_main(click_ctx, verbose, force_no_log, log_file, no_prompt, token_path):
|
|
|
193
209
|
click_ctx.obj.update({"DEFAULT_LOG": False})
|
|
194
210
|
|
|
195
211
|
|
|
212
|
+
### GUI COMMAND ###
|
|
213
|
+
|
|
214
|
+
# TODO: Should totp be passed to the gui?
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# @dds_main.command(name="gui")
|
|
218
|
+
# @click.pass_obj
|
|
219
|
+
# def gui(click_ctx):
|
|
220
|
+
# """Start the DDS GUI."""
|
|
221
|
+
# gui_app = dds_cli.dds_gui.app.DDSApp(token_path=click_ctx.get("TOKEN_PATH"))
|
|
222
|
+
# gui_app.title = "SciLifeLab Data Delivery System"
|
|
223
|
+
# gui_app.sub_title = "CLI Version: " + dds_cli.__version__
|
|
224
|
+
# gui_app.run()
|
|
225
|
+
|
|
226
|
+
|
|
196
227
|
# ************************************************************************************************ #
|
|
197
228
|
# MAIN DDS COMMANDS ************************************************************ MAIN DDS COMMANDS #
|
|
198
229
|
# ************************************************************************************************ #
|
|
@@ -425,7 +456,9 @@ def login(click_ctx, totp, allow_group):
|
|
|
425
456
|
LOG.warning("The --no-prompt flag is ignored for `dds auth login`")
|
|
426
457
|
try:
|
|
427
458
|
with dds_cli.auth.Auth(
|
|
428
|
-
token_path=click_ctx.get("TOKEN_PATH"),
|
|
459
|
+
token_path=click_ctx.get("TOKEN_PATH"),
|
|
460
|
+
totp=totp,
|
|
461
|
+
allow_group=allow_group,
|
|
429
462
|
):
|
|
430
463
|
# Authentication token renewed in the init method.
|
|
431
464
|
LOG.info("[green] :white_check_mark: Authentication successful![/green]")
|
|
@@ -452,7 +485,8 @@ def logout(click_ctx):
|
|
|
452
485
|
with dds_cli.auth.Auth(
|
|
453
486
|
authenticate=False, token_path=click_ctx.get("TOKEN_PATH")
|
|
454
487
|
) as authenticator:
|
|
455
|
-
authenticator.logout()
|
|
488
|
+
logout_ok = authenticator.logout()
|
|
489
|
+
dds_cli.message_helper.CLIMessageHelper().logout_message(logout_ok=logout_ok)
|
|
456
490
|
|
|
457
491
|
except (dds_cli.exceptions.DDSCLIException, dds_cli.exceptions.ApiRequestError) as err:
|
|
458
492
|
LOG.error(err)
|
|
@@ -474,7 +508,13 @@ def info(click_ctx):
|
|
|
474
508
|
with dds_cli.auth.Auth(
|
|
475
509
|
authenticate=False, token_path=click_ctx.get("TOKEN_PATH")
|
|
476
510
|
) as authenticator:
|
|
477
|
-
authenticator.check()
|
|
511
|
+
expiration_time = authenticator.check()
|
|
512
|
+
if expiration_time:
|
|
513
|
+
dds_cli.message_helper.CLIMessageHelper().token_report_message(
|
|
514
|
+
expiration_time=expiration_time
|
|
515
|
+
)
|
|
516
|
+
else:
|
|
517
|
+
dds_cli.message_helper.CLIMessageHelper().token_expired_message()
|
|
478
518
|
except (dds_cli.exceptions.DDSCLIException, dds_cli.exceptions.ApiRequestError) as err:
|
|
479
519
|
LOG.error(err)
|
|
480
520
|
sys.exit(1)
|
|
@@ -744,12 +784,12 @@ def delete_user(click_ctx, email, self, is_invite):
|
|
|
744
784
|
proceed_deletion = True
|
|
745
785
|
else:
|
|
746
786
|
if is_invite and self:
|
|
747
|
-
LOG.error("You cannot specify both `--self` and `--is-invite
|
|
787
|
+
LOG.error("You cannot specify both `--self` and `--is-invite`. Choose one.")
|
|
748
788
|
sys.exit(0)
|
|
749
789
|
|
|
750
790
|
if not self and not email:
|
|
751
791
|
LOG.error(
|
|
752
|
-
"You must specify an email
|
|
792
|
+
"You must specify an email address associated to the user you're requesting to delete."
|
|
753
793
|
)
|
|
754
794
|
sys.exit(0)
|
|
755
795
|
|
|
@@ -1837,7 +1877,6 @@ def get_data(
|
|
|
1837
1877
|
|
|
1838
1878
|
# Schedule the first num_threads futures for upload
|
|
1839
1879
|
for file in itertools.islice(iterator, num_threads):
|
|
1840
|
-
LOG.debug("Starting: %s", rich.markup.escape(str(file)))
|
|
1841
1880
|
# Execute download
|
|
1842
1881
|
download_threads[
|
|
1843
1882
|
texec.submit(getter.download_and_verify, file=file, progress=progress)
|
|
@@ -1876,7 +1915,6 @@ def get_data(
|
|
|
1876
1915
|
|
|
1877
1916
|
# Schedule the next set of futures for download
|
|
1878
1917
|
for next_file in itertools.islice(iterator, new_tasks):
|
|
1879
|
-
LOG.debug("Starting: %s", rich.markup.escape(str(next_file)))
|
|
1880
1918
|
# Execute download
|
|
1881
1919
|
download_threads[
|
|
1882
1920
|
texec.submit(
|
|
@@ -2120,6 +2158,8 @@ def list_active_motds(click_ctx):
|
|
|
2120
2158
|
) as err:
|
|
2121
2159
|
LOG.error(err)
|
|
2122
2160
|
sys.exit(1)
|
|
2161
|
+
except dds_cli.exceptions.NoMOTDsError as err:
|
|
2162
|
+
LOG.info(err)
|
|
2123
2163
|
|
|
2124
2164
|
|
|
2125
2165
|
# -- dds motd deactivate -- #
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
# Standard library
|
|
4
4
|
import logging
|
|
5
5
|
import getpass
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Optional
|
|
6
8
|
|
|
7
9
|
# Installed
|
|
8
10
|
from rich.prompt import Prompt
|
|
@@ -39,6 +41,10 @@ class Auth(base.DDSBaseClass):
|
|
|
39
41
|
):
|
|
40
42
|
"""Handle actions regarding session management in DDS."""
|
|
41
43
|
# Initiate DDSBaseClass to authenticate user
|
|
44
|
+
# Will authenticate user automatically if authenticate is True,
|
|
45
|
+
# else need to call login and confirm_twofactor methods to authenticate user
|
|
46
|
+
# This is to be able to use the auth class in the GUI code,
|
|
47
|
+
# where the user is not prompted for username and password
|
|
42
48
|
super().__init__(
|
|
43
49
|
authenticate=authenticate,
|
|
44
50
|
force_renew_token=force_renew_token,
|
|
@@ -47,27 +53,83 @@ class Auth(base.DDSBaseClass):
|
|
|
47
53
|
allow_group=allow_group,
|
|
48
54
|
)
|
|
49
55
|
|
|
50
|
-
|
|
51
|
-
|
|
56
|
+
self.allow_group = allow_group
|
|
57
|
+
|
|
58
|
+
def login(self, username: Optional[str] = None, password: Optional[str] = None) -> tuple:
|
|
59
|
+
"""Login user to DDS. Used to manually authenticate users with username and password.
|
|
60
|
+
If not provided, will prompt for them. Currently only used in the GUI.
|
|
61
|
+
|
|
62
|
+
:param username: The username to login with.
|
|
63
|
+
:param password: The password to login with.
|
|
64
|
+
|
|
65
|
+
:return: Partial auth token and second factor method
|
|
66
|
+
"""
|
|
67
|
+
# Create a User instance to call the login method
|
|
68
|
+
user_instance = user.User(
|
|
69
|
+
force_renew_token=False,
|
|
70
|
+
no_prompt=False,
|
|
71
|
+
token_path=self.token_path,
|
|
72
|
+
allow_group=self.allow_group,
|
|
73
|
+
retrieve_token=False,
|
|
74
|
+
)
|
|
75
|
+
return user_instance.login(username, password)
|
|
76
|
+
|
|
77
|
+
def confirm_twofactor(
|
|
78
|
+
self,
|
|
79
|
+
partial_auth_token: str,
|
|
80
|
+
secondfactor_method: str,
|
|
81
|
+
totp: str = None,
|
|
82
|
+
twofactor_code: Optional[str] = None,
|
|
83
|
+
):
|
|
84
|
+
"""Confirm 2FA for user. Used to manually confirm the 2FA code.
|
|
85
|
+
If not provided, will prompt for it. Currently only used in the GUI.
|
|
86
|
+
|
|
87
|
+
Sets the token for the base class after confirming 2FA.
|
|
88
|
+
|
|
89
|
+
:param partial_auth_token: The partial auth token.
|
|
90
|
+
:param twofactor_code: The 2FA code to confirm.
|
|
91
|
+
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
user_instance = user.User(
|
|
95
|
+
force_renew_token=False,
|
|
96
|
+
token_path=self.token_path,
|
|
97
|
+
allow_group=self.allow_group,
|
|
98
|
+
totp=totp,
|
|
99
|
+
retrieve_token=False,
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
user_instance.confirm_twofactor(
|
|
103
|
+
partial_auth_token=partial_auth_token,
|
|
104
|
+
secondfactor_method=secondfactor_method,
|
|
105
|
+
totp=totp,
|
|
106
|
+
twofactor_code=twofactor_code,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
self.set_token(user_instance.token_dict)
|
|
110
|
+
|
|
111
|
+
def check(self) -> Optional[datetime]:
|
|
112
|
+
"""Check if token exists and returns the token expiration time.
|
|
113
|
+
|
|
114
|
+
:return: Token info if token exists, None otherwise.
|
|
115
|
+
"""
|
|
52
116
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
53
117
|
if token_file.file_exists():
|
|
54
118
|
token = token_file.read_token()
|
|
55
119
|
if token:
|
|
56
|
-
token_file.token_report(token=token)
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
)
|
|
120
|
+
return token_file.token_report(token=token)
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
def logout(self) -> bool:
|
|
124
|
+
"""Logout user by removing authenticated token.
|
|
62
125
|
|
|
63
|
-
|
|
64
|
-
"""
|
|
126
|
+
:return: True if logout was successful, False if already logged out.
|
|
127
|
+
"""
|
|
65
128
|
token_file = user.TokenFile(token_path=self.token_path)
|
|
66
129
|
if token_file.file_exists():
|
|
67
130
|
token_file.delete_token()
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
LOG.info("[green]Already logged out![/green]")
|
|
131
|
+
return True
|
|
132
|
+
return False
|
|
71
133
|
|
|
72
134
|
def twofactor(self, auth_method: str = None):
|
|
73
135
|
"""Perform 2FA for user."""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Base class for the DDS CLI. Verifies the
|
|
1
|
+
"""Base class for the DDS CLI. Verifies the user's access to the DDS."""
|
|
2
2
|
|
|
3
3
|
###############################################################################
|
|
4
4
|
# IMPORTS ########################################################### IMPORTS #
|
|
@@ -58,6 +58,8 @@ class DDSBaseClass:
|
|
|
58
58
|
self.no_prompt = no_prompt
|
|
59
59
|
self.token_path = token_path
|
|
60
60
|
|
|
61
|
+
self.totp = totp
|
|
62
|
+
|
|
61
63
|
# Keyboardinterrupt
|
|
62
64
|
self.stop_doing = False
|
|
63
65
|
|
|
@@ -67,8 +69,8 @@ class DDSBaseClass:
|
|
|
67
69
|
force_renew_token=force_renew_token,
|
|
68
70
|
no_prompt=no_prompt,
|
|
69
71
|
token_path=token_path,
|
|
70
|
-
totp=totp,
|
|
71
72
|
allow_group=allow_group,
|
|
73
|
+
totp=totp,
|
|
72
74
|
)
|
|
73
75
|
self.token = dds_user.token_dict
|
|
74
76
|
|
|
@@ -113,6 +115,13 @@ class DDSBaseClass:
|
|
|
113
115
|
|
|
114
116
|
# Public methods ############################### Public methods #
|
|
115
117
|
|
|
118
|
+
def set_token(self, token_dict: dict) -> None:
|
|
119
|
+
"""Sets the token for the base class.
|
|
120
|
+
Called from auth class to set the token when authenticating
|
|
121
|
+
in the gui without running the base class with authenticate set to true.
|
|
122
|
+
"""
|
|
123
|
+
self.token = token_dict
|
|
124
|
+
|
|
116
125
|
def get_project_info(self):
|
|
117
126
|
"""Collect project information from API."""
|
|
118
127
|
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Shared constants used in the DDS CLI.
|
|
2
|
+
|
|
3
|
+
Using this will avoid e.g. circular imports and
|
|
4
|
+
make it easier to maintain.
|
|
5
|
+
|
|
6
|
+
TODO: Move other constants here from __init__.py.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
# Timeout settings for upload and download
|
|
10
|
+
READ_TIMEOUT = 300
|
|
11
|
+
CONNECT_TIMEOUT = 60
|
|
12
|
+
|
|
13
|
+
# Import these constants when using '*'
|
|
14
|
+
__all__ = ["READ_TIMEOUT", "CONNECT_TIMEOUT"]
|
|
@@ -40,7 +40,7 @@ def verify_proceed(func):
|
|
|
40
40
|
# Check if keyboardinterrupt in dds
|
|
41
41
|
if self.stop_doing:
|
|
42
42
|
# TODO (ina): Add save to status here
|
|
43
|
-
message = "
|
|
43
|
+
message = f"KeyboardInterrupt - cancelling file {escape(file)}"
|
|
44
44
|
LOG.warning(message)
|
|
45
45
|
return False # Do not proceed
|
|
46
46
|
|
|
@@ -52,7 +52,6 @@ def verify_proceed(func):
|
|
|
52
52
|
|
|
53
53
|
# Mark as started
|
|
54
54
|
self.status[file]["started"] = True
|
|
55
|
-
LOG.debug("File '%s' started: %s", escape(str(file)), func.__name__)
|
|
56
55
|
|
|
57
56
|
# Run function
|
|
58
57
|
ok_to_proceed, message = func(self, file=file, *args, **kwargs)
|
|
@@ -101,7 +100,6 @@ def update_status(func):
|
|
|
101
100
|
|
|
102
101
|
# Update status to started
|
|
103
102
|
self.status[file][func.__name__].update({"started": True})
|
|
104
|
-
LOG.debug("File '%s' status updated to %s: started", escape(str(file)), func.__name__)
|
|
105
103
|
|
|
106
104
|
# Run function
|
|
107
105
|
ok_to_continue, message, *_ = func(self, file=file, *args, **kwargs)
|
|
@@ -109,14 +107,12 @@ def update_status(func):
|
|
|
109
107
|
# ok_to_continue = False
|
|
110
108
|
if not ok_to_continue:
|
|
111
109
|
# Save info about which operation failed
|
|
112
|
-
|
|
113
110
|
self.status[file]["failed_op"] = func.__name__
|
|
114
111
|
LOG.warning("%s failed: %s", func.__name__, message)
|
|
115
112
|
|
|
116
113
|
else:
|
|
117
114
|
# Update status to done
|
|
118
115
|
self.status[file][func.__name__].update({"done": True})
|
|
119
|
-
LOG.debug("File %s status updated to %s: done", escape(str(file)), func.__name__)
|
|
120
116
|
|
|
121
117
|
return ok_to_continue, message
|
|
122
118
|
|
|
@@ -14,6 +14,7 @@ from rich.markup import escape
|
|
|
14
14
|
from rich.progress import Progress, SpinnerColumn
|
|
15
15
|
|
|
16
16
|
# Own modules
|
|
17
|
+
from dds_cli import constants
|
|
17
18
|
from dds_cli import DDSEndpoint, FileSegment
|
|
18
19
|
from dds_cli import file_handler_remote as fhr
|
|
19
20
|
from dds_cli import data_remover as dr
|
|
@@ -115,7 +116,9 @@ class DataGetter(base.DDSBaseClass):
|
|
|
115
116
|
"""Download the file, reveals the original data and verifies the integrity."""
|
|
116
117
|
all_ok, message = (False, "")
|
|
117
118
|
file_info = self.filehandler.data[file]
|
|
119
|
+
file_name_in_db = escape(str(file_info["name_in_db"]))
|
|
118
120
|
|
|
121
|
+
LOG.debug("Step 'download_and_verify': started file '%s'", file_name_in_db)
|
|
119
122
|
# File task for downloading
|
|
120
123
|
task = progress.add_task(
|
|
121
124
|
description=txt.TextHandler.task_name(file=escape(str(file)), step="get"),
|
|
@@ -133,20 +136,49 @@ class DataGetter(base.DDSBaseClass):
|
|
|
133
136
|
total=file_info["size_original"],
|
|
134
137
|
)
|
|
135
138
|
|
|
136
|
-
LOG.debug("File '%s' downloaded: %s",
|
|
139
|
+
LOG.debug("File '%s' downloaded: %s", file_name_in_db, file_downloaded)
|
|
140
|
+
|
|
141
|
+
file_size_verified = False
|
|
137
142
|
|
|
138
143
|
if file_downloaded:
|
|
144
|
+
## File size verification
|
|
145
|
+
expected_size = file_info["size_stored"]
|
|
146
|
+
actual_size = file_info["path_downloaded"].stat().st_size
|
|
147
|
+
|
|
148
|
+
if actual_size == expected_size:
|
|
149
|
+
file_size_verified = True
|
|
150
|
+
LOG.debug(
|
|
151
|
+
"Downloaded file '%s' size matches expected size: %s bytes.",
|
|
152
|
+
file_name_in_db,
|
|
153
|
+
expected_size,
|
|
154
|
+
)
|
|
155
|
+
else:
|
|
156
|
+
LOG.debug(
|
|
157
|
+
"Downloaded file '%s' size mismatch: expected %s bytes, got %s bytes. Not decrypting.",
|
|
158
|
+
file_name_in_db,
|
|
159
|
+
expected_size,
|
|
160
|
+
actual_size,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
if file_size_verified:
|
|
139
164
|
db_updated, message = self.update_db(file=file)
|
|
140
|
-
LOG.debug(
|
|
165
|
+
LOG.debug(
|
|
166
|
+
"API call: database updated for file '%s': %s",
|
|
167
|
+
file_name_in_db,
|
|
168
|
+
db_updated,
|
|
169
|
+
)
|
|
141
170
|
|
|
142
|
-
LOG.debug("Beginning decryption of file '%s'...",
|
|
171
|
+
LOG.debug("Beginning decryption of file '%s'...", file_name_in_db)
|
|
143
172
|
file_saved = False
|
|
144
173
|
with fe.Decryptor(
|
|
145
174
|
project_keys=self.keys,
|
|
146
175
|
peer_public=file_info["public_key"],
|
|
147
176
|
key_salt=file_info["salt"],
|
|
177
|
+
files_directory=self.dds_directory.directories["FILES"],
|
|
148
178
|
) as decryptor:
|
|
149
|
-
streamed_chunks = decryptor.decrypt_file(
|
|
179
|
+
streamed_chunks = decryptor.decrypt_file(
|
|
180
|
+
infile=file_info["path_downloaded"], outfile=file
|
|
181
|
+
)
|
|
150
182
|
|
|
151
183
|
stream_to_file_func = (
|
|
152
184
|
fc.Compressor.decompress_filechunks
|
|
@@ -157,14 +189,35 @@ class DataGetter(base.DDSBaseClass):
|
|
|
157
189
|
file_saved, message = stream_to_file_func(
|
|
158
190
|
chunks=streamed_chunks,
|
|
159
191
|
outfile=file,
|
|
192
|
+
files_directory=self.dds_directory.directories["FILES"],
|
|
160
193
|
)
|
|
161
194
|
|
|
162
|
-
LOG.debug("File saved? %s", file_saved)
|
|
195
|
+
LOG.debug("File '%s' saved? %s", file_name_in_db, file_saved)
|
|
163
196
|
if file_saved:
|
|
197
|
+
# Check file size post-decryption and post-decompression
|
|
198
|
+
expected_size = file_info["size_original"]
|
|
199
|
+
actual_size = pathlib.Path(file).stat().st_size
|
|
200
|
+
if actual_size == expected_size:
|
|
201
|
+
LOG.debug(
|
|
202
|
+
"Decrypted file '%s' size matches expected size: %s bytes.",
|
|
203
|
+
file_name_in_db,
|
|
204
|
+
expected_size,
|
|
205
|
+
)
|
|
206
|
+
else:
|
|
207
|
+
LOG.debug(
|
|
208
|
+
"Decrypted file '%s' size mismatch: expected %s bytes, got %s bytes",
|
|
209
|
+
file_name_in_db,
|
|
210
|
+
expected_size,
|
|
211
|
+
actual_size,
|
|
212
|
+
)
|
|
164
213
|
# TODO (ina): decide on checksum verification method --
|
|
165
214
|
# this checks original, the other is generated from compressed
|
|
166
215
|
all_ok, message = (
|
|
167
|
-
fe.Encryptor.verify_checksum(
|
|
216
|
+
fe.Encryptor.verify_checksum(
|
|
217
|
+
file=file,
|
|
218
|
+
correct_checksum=file_info["checksum"],
|
|
219
|
+
files_directory=self.dds_directory.directories["FILES"],
|
|
220
|
+
)
|
|
168
221
|
if self.verify_checksum
|
|
169
222
|
else (True, "")
|
|
170
223
|
)
|
|
@@ -183,8 +236,11 @@ class DataGetter(base.DDSBaseClass):
|
|
|
183
236
|
file_remote = self.filehandler.data[file]["url"]
|
|
184
237
|
|
|
185
238
|
try:
|
|
186
|
-
|
|
187
|
-
|
|
239
|
+
with requests.get(
|
|
240
|
+
file_remote,
|
|
241
|
+
stream=True,
|
|
242
|
+
timeout=(constants.CONNECT_TIMEOUT, constants.READ_TIMEOUT),
|
|
243
|
+
) as req:
|
|
188
244
|
req.raise_for_status()
|
|
189
245
|
with file_local.open(mode="wb") as new_file:
|
|
190
246
|
for chunk in req.iter_content(chunk_size=FileSegment.SEGMENT_SIZE_CIPHER):
|
|
@@ -95,7 +95,6 @@ def put(
|
|
|
95
95
|
|
|
96
96
|
# Schedule the first num_threads futures for upload
|
|
97
97
|
for file in itertools.islice(iterator, num_threads):
|
|
98
|
-
LOG.debug("Starting: '%s'", escape(file))
|
|
99
98
|
upload_threads[
|
|
100
99
|
texec.submit(
|
|
101
100
|
putter.protect_and_upload,
|
|
@@ -119,7 +118,7 @@ def put(
|
|
|
119
118
|
# Get result from future and schedule database update
|
|
120
119
|
for fut in done:
|
|
121
120
|
uploaded_file = upload_threads.pop(fut)
|
|
122
|
-
LOG.debug("Future done for file: %s", escape(uploaded_file))
|
|
121
|
+
LOG.debug("Future done for file: '%s'", escape(uploaded_file))
|
|
123
122
|
|
|
124
123
|
# Get result
|
|
125
124
|
try:
|
|
@@ -291,6 +290,8 @@ class DataPutter(base.DDSBaseClass):
|
|
|
291
290
|
all_ok, saved, message = (False, False, "") # Error catching
|
|
292
291
|
file_info = self.filehandler.data[file] # Info on current file
|
|
293
292
|
file_public_key, salt = ("", "") # Crypto info
|
|
293
|
+
file_path_raw = escape(str(file_info["path_raw"]))
|
|
294
|
+
LOG.debug("Step '%s': started file '%s'", self.method, file_path_raw)
|
|
294
295
|
|
|
295
296
|
# Progress bar for processing
|
|
296
297
|
task = progress.add_task(
|
|
@@ -305,6 +306,7 @@ class DataPutter(base.DDSBaseClass):
|
|
|
305
306
|
# Stream the chunks into the encryptor to save the encrypted chunks
|
|
306
307
|
with fe.Encryptor(project_keys=self.keys) as encryptor:
|
|
307
308
|
# Encrypt and save chunks
|
|
309
|
+
LOG.debug("Encrypting file '%s'", file_path_raw)
|
|
308
310
|
saved, message = encryptor.encrypt_filechunks(
|
|
309
311
|
chunks=streamed_chunks,
|
|
310
312
|
outfile=file_info["path_processed"],
|
|
@@ -315,18 +317,21 @@ class DataPutter(base.DDSBaseClass):
|
|
|
315
317
|
file_public_key = encryptor.get_public_component_hex(private_key=encryptor.my_private)
|
|
316
318
|
salt = encryptor.salt
|
|
317
319
|
|
|
318
|
-
LOG.debug("Updating file processed size: %s", file_info["path_processed"])
|
|
319
|
-
|
|
320
320
|
# Update file info incl size, public key, salt
|
|
321
321
|
self.filehandler.data[file]["public_key"] = file_public_key
|
|
322
322
|
self.filehandler.data[file]["salt"] = salt
|
|
323
323
|
self.filehandler.data[file]["size_processed"] = file_info["path_processed"].stat().st_size
|
|
324
324
|
|
|
325
|
+
LOG.debug(
|
|
326
|
+
"File '%s' processed size: %s",
|
|
327
|
+
file_path_raw,
|
|
328
|
+
file_info["path_processed"].stat().st_size,
|
|
329
|
+
)
|
|
330
|
+
|
|
325
331
|
if saved:
|
|
326
332
|
LOG.debug(
|
|
327
|
-
"File successfully encrypted: '%s'
|
|
328
|
-
|
|
329
|
-
escape(str(file_info["path_processed"])),
|
|
333
|
+
"File successfully encrypted: '%s'",
|
|
334
|
+
file_path_raw,
|
|
330
335
|
)
|
|
331
336
|
# Update progress bar for upload
|
|
332
337
|
progress.reset(
|
|
@@ -346,7 +351,8 @@ class DataPutter(base.DDSBaseClass):
|
|
|
346
351
|
if db_updated:
|
|
347
352
|
all_ok = True
|
|
348
353
|
LOG.debug(
|
|
349
|
-
"File successfully uploaded and added to the database: '%s'",
|
|
354
|
+
"File successfully uploaded and added to the database: '%s'",
|
|
355
|
+
file_path_raw,
|
|
350
356
|
)
|
|
351
357
|
|
|
352
358
|
if not saved or all_ok:
|
|
@@ -373,6 +379,8 @@ class DataPutter(base.DDSBaseClass):
|
|
|
373
379
|
# File info
|
|
374
380
|
file_local = str(self.filehandler.data[file]["path_processed"])
|
|
375
381
|
file_remote = self.filehandler.data[file]["path_remote"]
|
|
382
|
+
file_path_raw = self.filehandler.data[file]["path_raw"]
|
|
383
|
+
LOG.debug("Step '%s': started file '%s'", self.method, file_path_raw)
|
|
376
384
|
|
|
377
385
|
try:
|
|
378
386
|
with self.s3connector as conn:
|
|
@@ -441,6 +449,7 @@ class DataPutter(base.DDSBaseClass):
|
|
|
441
449
|
error_message=f"Failed to add file '{file}' to database",
|
|
442
450
|
)
|
|
443
451
|
added_to_db, message = (True, response_json)
|
|
452
|
+
LOG.debug("API call for file '%s: Adding to database'", fileinfo["path_raw"])
|
|
444
453
|
except (
|
|
445
454
|
dds_cli.exceptions.ApiRequestError,
|
|
446
455
|
dds_cli.exceptions.ApiResponseError,
|