dds-cli 2.12.0__tar.gz → 2.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dds_cli-2.12.0 → dds_cli-2.14.0}/PKG-INFO +7 -8
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/__init__.py +1 -1
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/__main__.py +37 -36
- dds_cli-2.14.0/dds_cli/constants.py +14 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/custom_decorators.py +2 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/data_getter.py +17 -4
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/data_lister.py +20 -23
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/data_putter.py +18 -11
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/file_compressor.py +1 -2
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/file_handler_local.py +5 -7
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/project_status.py +11 -2
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/s3_connector.py +9 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/user.py +4 -4
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/version.py +1 -1
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/PKG-INFO +7 -8
- dds_cli-2.14.0/dds_cli.egg-info/SOURCES.txt +62 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/requires.txt +7 -5
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/top_level.txt +0 -1
- {dds_cli-2.12.0 → dds_cli-2.14.0}/setup.py +1 -2
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_auth.py +5 -3
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_base.py +9 -7
- dds_cli-2.14.0/tests/test_commands.py +185 -0
- dds_cli-2.14.0/tests/test_data_getter.py +133 -0
- dds_cli-2.14.0/tests/test_data_putter.py +85 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_data_remover.py +17 -20
- dds_cli-2.14.0/tests/test_decorators.py +128 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_file_compressor.py +17 -25
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_file_encryptor.py +1 -2
- dds_cli-2.14.0/tests/test_file_handler_local.py +222 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_motd_manager.py +15 -13
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_project_status.py +152 -28
- dds_cli-2.14.0/tests/test_s3_connector.py +112 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_superadmin_helper.py +3 -3
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_user.py +97 -20
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_utils.py +26 -16
- dds_cli-2.12.0/dds_cli/dds_gui/__init__.py +0 -6
- dds_cli-2.12.0/dds_cli/dds_gui/app.py +0 -71
- dds_cli-2.12.0/dds_cli/dds_gui/components/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_button.py +0 -65
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_container.py +0 -80
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_footer.py +0 -17
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_form.py +0 -27
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_input.py +0 -19
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_modal.py +0 -138
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_select.py +0 -23
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_status_chip.py +0 -61
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_text_item.py +0 -17
- dds_cli-2.12.0/dds_cli/dds_gui/components/dds_tree_view.py +0 -55
- dds_cli-2.12.0/dds_cli/dds_gui/dds_state_manager.py +0 -202
- dds_cli-2.12.0/dds_cli/dds_gui/pages/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/authentication.py +0 -56
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/authentication_form.py +0 -122
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/modals/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/modals/login_modal.py +0 -35
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/modals/logout_modal.py +0 -28
- dds_cli-2.12.0/dds_cli/dds_gui/pages/authentication/modals/reauthenticate_modal.py +0 -35
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view.py +0 -72
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_actions.py +0 -32
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/download_data.py +0 -61
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_actions_tabs/user_access.py +0 -41
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_content.py +0 -31
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_information.py +0 -99
- dds_cli-2.12.0/dds_cli/dds_gui/pages/project_view_mode/project_list.py +0 -37
- dds_cli-2.12.0/dds_cli/dds_gui/types/__init__.py +0 -1
- dds_cli-2.12.0/dds_cli/dds_gui/types/dds_severity_types.py +0 -12
- dds_cli-2.12.0/dds_cli/dds_gui/types/dds_status_types.py +0 -14
- dds_cli-2.12.0/dds_cli.egg-info/SOURCES.txt +0 -95
- dds_cli-2.12.0/gui_build/gui_standalone.py +0 -9
- dds_cli-2.12.0/tests/__init__.py +0 -0
- dds_cli-2.12.0/tests/gui_tests/__init__.py +0 -0
- dds_cli-2.12.0/tests/gui_tests/test_authentication.py +0 -244
- dds_cli-2.12.0/tests/gui_tests/test_important_information.py +0 -455
- dds_cli-2.12.0/tests/test_file_handler_local.py +0 -90
- {dds_cli-2.12.0 → dds_cli-2.14.0}/LICENSE +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/README.md +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/account_manager.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/auth.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/base.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/data_remover.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/directory.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/exceptions.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/file_encryptor.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/file_handler.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/file_handler_remote.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/message_helper.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/motd_manager.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/options.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/project_creator.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/project_info.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/status.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/superadmin_helper.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/text_handler.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/timestamp.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/unit_manager.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli/utils.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/dependency_links.txt +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/entry_points.txt +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/dds_cli.egg-info/not-zip-safe +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/pyproject.toml +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/setup.cfg +0 -0
- {dds_cli-2.12.0/gui_build → dds_cli-2.14.0/tests}/__init__.py +0 -0
- {dds_cli-2.12.0 → dds_cli-2.14.0}/tests/test_account_manager.py +0 -0
|
@@ -1,39 +1,38 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dds_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.14.0
|
|
4
4
|
Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
|
|
5
5
|
Home-page: https://github.com/ScilifelabDataCentre/dds_cli
|
|
6
6
|
Author: SciLifeLab Data Centre
|
|
7
7
|
License: MIT
|
|
8
8
|
Classifier: Development Status :: 4 - Beta
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
14
|
Description-Content-Type: text/markdown
|
|
16
15
|
License-File: LICENSE
|
|
17
16
|
Requires-Dist: boto3==1.24.73
|
|
18
17
|
Requires-Dist: botocore==1.27.73
|
|
19
18
|
Requires-Dist: click==8.1.3
|
|
20
19
|
Requires-Dist: click-pathlib==2020.3.13.0
|
|
21
|
-
Requires-Dist: cryptography==
|
|
20
|
+
Requires-Dist: cryptography==44.0.1
|
|
22
21
|
Requires-Dist: immutabledict==2.2.1
|
|
23
22
|
Requires-Dist: jwcrypto==1.5.6
|
|
24
23
|
Requires-Dist: prettytable==3.7.0
|
|
25
24
|
Requires-Dist: prompt-toolkit==3.0.40
|
|
26
|
-
Requires-Dist: PyNaCl==1.
|
|
25
|
+
Requires-Dist: PyNaCl==1.6.2
|
|
27
26
|
Requires-Dist: pytz==2022.2.1
|
|
28
27
|
Requires-Dist: PyYAML==6.0.2
|
|
29
|
-
Requires-Dist: questionary==1.
|
|
30
|
-
Requires-Dist: requests==2.32.
|
|
28
|
+
Requires-Dist: questionary==2.1.1
|
|
29
|
+
Requires-Dist: requests==2.32.4
|
|
31
30
|
Requires-Dist: rich==13.6.0
|
|
32
31
|
Requires-Dist: rich-click==1.5.2
|
|
33
32
|
Requires-Dist: simplejson==3.17.6
|
|
34
33
|
Requires-Dist: tzlocal==4.2
|
|
35
34
|
Requires-Dist: zstandard==0.23.0
|
|
36
|
-
Requires-Dist:
|
|
35
|
+
Requires-Dist: legacy-cgi==2.6.1; python_version >= "3.13"
|
|
37
36
|
Dynamic: author
|
|
38
37
|
Dynamic: classifier
|
|
39
38
|
Dynamic: description
|
|
@@ -44,7 +44,7 @@ DDS_DIR_REQUIRED_METHODS = ["put", "get"]
|
|
|
44
44
|
DDS_KEYS_REQUIRED_METHODS = ["put", "get"]
|
|
45
45
|
|
|
46
46
|
# Token related variables
|
|
47
|
-
TOKEN_FILE = pathlib.Path
|
|
47
|
+
TOKEN_FILE = pathlib.Path.home() / ".dds_cli_token"
|
|
48
48
|
TOKEN_EXPIRATION_WARNING_THRESHOLD = datetime.timedelta(hours=6)
|
|
49
49
|
|
|
50
50
|
|
|
@@ -8,61 +8,60 @@
|
|
|
8
8
|
import concurrent.futures
|
|
9
9
|
import itertools
|
|
10
10
|
import logging
|
|
11
|
+
import pathlib
|
|
11
12
|
import sys
|
|
12
13
|
|
|
13
14
|
# Installed
|
|
14
|
-
|
|
15
|
-
import rich_click as click
|
|
15
|
+
|
|
16
16
|
import click_pathlib
|
|
17
|
+
import questionary
|
|
17
18
|
import rich
|
|
18
19
|
import rich.logging
|
|
19
20
|
import rich.markup
|
|
20
21
|
import rich.progress
|
|
21
22
|
import rich.prompt
|
|
22
|
-
import
|
|
23
|
+
import rich_click as click
|
|
23
24
|
|
|
24
25
|
# Own modules
|
|
25
26
|
import dds_cli
|
|
26
27
|
import dds_cli.account_manager
|
|
27
|
-
import dds_cli.
|
|
28
|
-
import dds_cli.motd_manager
|
|
29
|
-
import dds_cli.superadmin_helper
|
|
28
|
+
import dds_cli.auth
|
|
30
29
|
import dds_cli.data_getter
|
|
31
30
|
import dds_cli.data_lister
|
|
32
31
|
import dds_cli.data_putter
|
|
33
32
|
import dds_cli.data_remover
|
|
34
33
|
import dds_cli.directory
|
|
34
|
+
import dds_cli.message_helper
|
|
35
|
+
import dds_cli.motd_manager
|
|
35
36
|
import dds_cli.project_creator
|
|
36
|
-
import dds_cli.auth
|
|
37
|
-
import dds_cli.project_status
|
|
38
37
|
import dds_cli.project_info
|
|
38
|
+
import dds_cli.project_status
|
|
39
|
+
import dds_cli.superadmin_helper
|
|
40
|
+
import dds_cli.unit_manager
|
|
39
41
|
import dds_cli.user
|
|
40
42
|
import dds_cli.utils
|
|
41
|
-
import dds_cli.message_helper
|
|
42
43
|
from dds_cli.options import (
|
|
44
|
+
break_on_fail_flag,
|
|
43
45
|
destination_option,
|
|
44
46
|
email_arg,
|
|
45
47
|
email_option,
|
|
46
48
|
folder_option,
|
|
49
|
+
json_flag,
|
|
50
|
+
nomail_flag,
|
|
47
51
|
num_threads_option,
|
|
48
52
|
project_option,
|
|
53
|
+
silent_flag,
|
|
54
|
+
size_flag,
|
|
49
55
|
sort_projects_option,
|
|
50
56
|
source_option,
|
|
51
57
|
source_path_file_option,
|
|
52
58
|
token_path_option,
|
|
53
|
-
username_option,
|
|
54
|
-
break_on_fail_flag,
|
|
55
|
-
json_flag,
|
|
56
|
-
nomail_flag,
|
|
57
|
-
silent_flag,
|
|
58
|
-
size_flag,
|
|
59
59
|
tree_flag,
|
|
60
60
|
usage_flag,
|
|
61
|
+
username_option,
|
|
61
62
|
users_flag,
|
|
62
63
|
)
|
|
63
64
|
|
|
64
|
-
# import dds_cli.dds_gui.app
|
|
65
|
-
|
|
66
65
|
####################################################################################################
|
|
67
66
|
# START LOGGING CONFIG ###################################################### START LOGGING CONFIG #
|
|
68
67
|
####################################################################################################
|
|
@@ -209,21 +208,6 @@ def dds_main(click_ctx, verbose, force_no_log, log_file, no_prompt, token_path):
|
|
|
209
208
|
click_ctx.obj.update({"DEFAULT_LOG": False})
|
|
210
209
|
|
|
211
210
|
|
|
212
|
-
### GUI COMMAND ###
|
|
213
|
-
|
|
214
|
-
# TODO: Should totp be passed to the gui?
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
# @dds_main.command(name="gui")
|
|
218
|
-
# @click.pass_obj
|
|
219
|
-
# def gui(click_ctx):
|
|
220
|
-
# """Start the DDS GUI."""
|
|
221
|
-
# gui_app = dds_cli.dds_gui.app.DDSApp(token_path=click_ctx.get("TOKEN_PATH"))
|
|
222
|
-
# gui_app.title = "SciLifeLab Data Delivery System"
|
|
223
|
-
# gui_app.sub_title = "CLI Version: " + dds_cli.__version__
|
|
224
|
-
# gui_app.run()
|
|
225
|
-
|
|
226
|
-
|
|
227
211
|
# ************************************************************************************************ #
|
|
228
212
|
# MAIN DDS COMMANDS ************************************************************ MAIN DDS COMMANDS #
|
|
229
213
|
# ************************************************************************************************ #
|
|
@@ -383,6 +367,7 @@ def list_projects_and_contents(
|
|
|
383
367
|
dds_cli.exceptions.AuthenticationError,
|
|
384
368
|
dds_cli.exceptions.ApiResponseError,
|
|
385
369
|
dds_cli.exceptions.ApiRequestError,
|
|
370
|
+
dds_cli.exceptions.DDSCLIException,
|
|
386
371
|
) as err:
|
|
387
372
|
LOG.error(err)
|
|
388
373
|
sys.exit(1)
|
|
@@ -544,13 +529,14 @@ def configure():
|
|
|
544
529
|
"Which method would you like to use?", choices=["Email", "Authenticator App", "Cancel"]
|
|
545
530
|
).ask()
|
|
546
531
|
|
|
532
|
+
auth_method: str | None = None # type hint, initialized
|
|
547
533
|
if auth_method_choice == "Cancel":
|
|
548
534
|
LOG.info("Two-factor authentication method not configured.")
|
|
549
535
|
sys.exit(0)
|
|
550
536
|
elif auth_method_choice == "Authenticator App":
|
|
551
|
-
auth_method
|
|
537
|
+
auth_method = "totp"
|
|
552
538
|
elif auth_method_choice == "Email":
|
|
553
|
-
auth_method
|
|
539
|
+
auth_method = "hotp"
|
|
554
540
|
|
|
555
541
|
with dds_cli.auth.Auth(authenticate=True, force_renew_token=False) as authenticator:
|
|
556
542
|
authenticator.twofactor(auth_method=auth_method)
|
|
@@ -885,6 +871,8 @@ def activate_user(click_ctx, email):
|
|
|
885
871
|
Super Admins: All users
|
|
886
872
|
Unit Admins: Unit Admins / Personnel
|
|
887
873
|
"""
|
|
874
|
+
proceed_activation = False # default assignment
|
|
875
|
+
|
|
888
876
|
if click_ctx.get("NO_PROMPT", False):
|
|
889
877
|
pass
|
|
890
878
|
else:
|
|
@@ -925,6 +913,8 @@ def deactivate_user(click_ctx, email):
|
|
|
925
913
|
Super Admins: All users
|
|
926
914
|
Unit Admins: Unit Admins / Personnel
|
|
927
915
|
"""
|
|
916
|
+
proceed_deactivation = False # default assignment
|
|
917
|
+
|
|
928
918
|
if click_ctx.get("NO_PROMPT", False):
|
|
929
919
|
pass
|
|
930
920
|
else:
|
|
@@ -1166,6 +1156,13 @@ def display_project_status(click_ctx, project, show_history):
|
|
|
1166
1156
|
sys.exit(1)
|
|
1167
1157
|
|
|
1168
1158
|
|
|
1159
|
+
def validate_deadline(_ctx, _param, value):
|
|
1160
|
+
"""Validate that the deadline is a positive number of days between 1 and 90."""
|
|
1161
|
+
if value is not None and value not in range(1, 91):
|
|
1162
|
+
raise click.BadParameter("Deadline must be a positive number of days between 1 and 90.")
|
|
1163
|
+
return value
|
|
1164
|
+
|
|
1165
|
+
|
|
1169
1166
|
# -- dds project status release -- #
|
|
1170
1167
|
@project_status.command(name="release", no_args_is_help=True)
|
|
1171
1168
|
# Options
|
|
@@ -1174,7 +1171,8 @@ def display_project_status(click_ctx, project, show_history):
|
|
|
1174
1171
|
"--deadline",
|
|
1175
1172
|
required=False,
|
|
1176
1173
|
type=int,
|
|
1177
|
-
|
|
1174
|
+
callback=validate_deadline,
|
|
1175
|
+
help="Deadline in days when releasing a project. Must be a positive number of days (maximum 90 days).",
|
|
1178
1176
|
)
|
|
1179
1177
|
@nomail_flag(help_message="Do not send e-mail notifications regarding project updates.")
|
|
1180
1178
|
@click.pass_obj
|
|
@@ -1315,7 +1313,8 @@ def delete_project(click_ctx, project: str):
|
|
|
1315
1313
|
"--new-deadline",
|
|
1316
1314
|
required=False,
|
|
1317
1315
|
type=int,
|
|
1318
|
-
|
|
1316
|
+
callback=validate_deadline,
|
|
1317
|
+
help="Number of days to extend the deadline. Must be a positive number of days (maximum 90 days).",
|
|
1319
1318
|
)
|
|
1320
1319
|
@click.pass_obj
|
|
1321
1320
|
def extend_deadline(click_ctx, project: str, new_deadline: int):
|
|
@@ -1739,6 +1738,7 @@ def put_data(
|
|
|
1739
1738
|
dds_cli.exceptions.ApiRequestError,
|
|
1740
1739
|
dds_cli.exceptions.NoKeyError,
|
|
1741
1740
|
dds_cli.exceptions.NoDataError,
|
|
1741
|
+
dds_cli.exceptions.DDSCLIException,
|
|
1742
1742
|
) as err:
|
|
1743
1743
|
LOG.error(err)
|
|
1744
1744
|
sys.exit(1)
|
|
@@ -2269,6 +2269,7 @@ def get_stats(click_ctx):
|
|
|
2269
2269
|
dds_cli.exceptions.AuthenticationError,
|
|
2270
2270
|
dds_cli.exceptions.ApiResponseError,
|
|
2271
2271
|
dds_cli.exceptions.ApiRequestError,
|
|
2272
|
+
dds_cli.exceptions.DDSCLIException,
|
|
2272
2273
|
) as err:
|
|
2273
2274
|
LOG.error(err)
|
|
2274
2275
|
sys.exit(1)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Shared constants used in the DDS CLI.
|
|
2
|
+
|
|
3
|
+
Using this will avoid e.g. circular imports and
|
|
4
|
+
make it easier to maintain.
|
|
5
|
+
|
|
6
|
+
TODO: Move other constants here from __init__.py.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
# Timeout settings for upload and download
|
|
10
|
+
READ_TIMEOUT = 300
|
|
11
|
+
CONNECT_TIMEOUT = 60
|
|
12
|
+
|
|
13
|
+
# Import these constants when using '*'
|
|
14
|
+
__all__ = ["READ_TIMEOUT", "CONNECT_TIMEOUT"]
|
|
@@ -154,6 +154,8 @@ def removal_spinner(func):
|
|
|
154
154
|
SpinnerColumn(spinner_name="dots12", style="white"),
|
|
155
155
|
console=dds_cli.utils.stderr_console,
|
|
156
156
|
) as progress:
|
|
157
|
+
|
|
158
|
+
description: str | None = None # type hint, initialized
|
|
157
159
|
# Determine spinner text
|
|
158
160
|
if func.__name__ == "remove_all":
|
|
159
161
|
description = f"Removing all files in project {self.project}"
|
|
@@ -14,6 +14,7 @@ from rich.markup import escape
|
|
|
14
14
|
from rich.progress import Progress, SpinnerColumn
|
|
15
15
|
|
|
16
16
|
# Own modules
|
|
17
|
+
from dds_cli import constants
|
|
17
18
|
from dds_cli import DDSEndpoint, FileSegment
|
|
18
19
|
from dds_cli import file_handler_remote as fhr
|
|
19
20
|
from dds_cli import data_remover as dr
|
|
@@ -100,8 +101,17 @@ class DataGetter(base.DDSBaseClass):
|
|
|
100
101
|
|
|
101
102
|
if not self.filehandler.data:
|
|
102
103
|
if self.temporary_directory and self.temporary_directory.is_dir():
|
|
103
|
-
LOG.debug("Deleting
|
|
104
|
-
|
|
104
|
+
LOG.debug("Deleting staging directory '%s'.", self.temporary_directory)
|
|
105
|
+
try:
|
|
106
|
+
dds_cli.utils.delete_folder(self.temporary_directory)
|
|
107
|
+
except OSError as err:
|
|
108
|
+
# Folder deletion may fail if log file is still being written to
|
|
109
|
+
# This is not critical - the important thing is to show the error message
|
|
110
|
+
LOG.error(
|
|
111
|
+
"Could not delete staging directory %s: %s",
|
|
112
|
+
self.temporary_directory,
|
|
113
|
+
err,
|
|
114
|
+
)
|
|
105
115
|
raise dds_cli.exceptions.DownloadError("No files to download.")
|
|
106
116
|
|
|
107
117
|
self.status = self.filehandler.create_download_status_dict()
|
|
@@ -235,8 +245,11 @@ class DataGetter(base.DDSBaseClass):
|
|
|
235
245
|
file_remote = self.filehandler.data[file]["url"]
|
|
236
246
|
|
|
237
247
|
try:
|
|
238
|
-
|
|
239
|
-
|
|
248
|
+
with requests.get(
|
|
249
|
+
file_remote,
|
|
250
|
+
stream=True,
|
|
251
|
+
timeout=(constants.CONNECT_TIMEOUT, constants.READ_TIMEOUT),
|
|
252
|
+
) as req:
|
|
240
253
|
req.raise_for_status()
|
|
241
254
|
with file_local.open(mode="wb") as new_file:
|
|
242
255
|
for chunk in req.iter_content(chunk_size=FileSegment.SEGMENT_SIZE_CIPHER):
|
|
@@ -5,26 +5,24 @@
|
|
|
5
5
|
###############################################################################
|
|
6
6
|
|
|
7
7
|
# Standard library
|
|
8
|
-
from dataclasses import dataclass
|
|
9
|
-
import logging
|
|
10
|
-
from typing import Tuple, Union, List
|
|
11
8
|
import datetime
|
|
9
|
+
import logging
|
|
12
10
|
import pathlib
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import List, Tuple, Union
|
|
13
13
|
|
|
14
14
|
# Installed
|
|
15
|
-
|
|
15
|
+
import pytz
|
|
16
|
+
import tzlocal
|
|
16
17
|
from rich.markup import escape
|
|
18
|
+
from rich.padding import Padding
|
|
17
19
|
from rich.table import Table
|
|
18
20
|
from rich.tree import Tree
|
|
19
|
-
import pytz
|
|
20
|
-
import tzlocal
|
|
21
21
|
|
|
22
22
|
# Own modules
|
|
23
|
-
from dds_cli import base
|
|
24
|
-
from dds_cli import exceptions
|
|
25
|
-
import dds_cli.utils
|
|
26
|
-
from dds_cli import DDSEndpoint
|
|
23
|
+
from dds_cli import DDSEndpoint, base, exceptions
|
|
27
24
|
from dds_cli import text_handler as th
|
|
25
|
+
import dds_cli.utils
|
|
28
26
|
|
|
29
27
|
|
|
30
28
|
###############################################################################
|
|
@@ -268,7 +266,11 @@ class DataLister(base.DDSBaseClass):
|
|
|
268
266
|
# Get max length of size string
|
|
269
267
|
max_size = max(
|
|
270
268
|
(
|
|
271
|
-
len(
|
|
269
|
+
len(
|
|
270
|
+
dds_cli.utils.format_api_response(
|
|
271
|
+
response=x["size"], key="Size", binary=self.binary
|
|
272
|
+
).split(" ", maxsplit=1)[0]
|
|
273
|
+
)
|
|
272
274
|
for x in sorted_files_folders
|
|
273
275
|
if show_size and "size" in x
|
|
274
276
|
),
|
|
@@ -280,9 +282,12 @@ class DataLister(base.DDSBaseClass):
|
|
|
280
282
|
is_folder = item.pop("folder")
|
|
281
283
|
|
|
282
284
|
if not is_folder:
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
285
|
+
formatted_size = None
|
|
286
|
+
if show_size and "size" in item:
|
|
287
|
+
formatted_size = dds_cli.utils.format_api_response(
|
|
288
|
+
response=item["size"], key="Size", binary=self.binary
|
|
289
|
+
)
|
|
290
|
+
tree.subtrees.append((escape(item["name"]), formatted_size))
|
|
286
291
|
else:
|
|
287
292
|
subtree, _max_string, _max_size = __construct_file_tree(
|
|
288
293
|
pathlib.Path(folder, item["name"]).as_posix() if folder else item["name"],
|
|
@@ -348,15 +353,7 @@ class DataLister(base.DDSBaseClass):
|
|
|
348
353
|
string_len=len(node[0]),
|
|
349
354
|
max_string_len=max_str - 4 * depth,
|
|
350
355
|
)
|
|
351
|
-
line += f"{tab}{node[1].split()[0]}"
|
|
352
|
-
|
|
353
|
-
# Define space between number and size format
|
|
354
|
-
tabs_bf_format = th.TextHandler.format_tabs(
|
|
355
|
-
string_len=len(node[1].split()[1]),
|
|
356
|
-
max_string_len=max_size,
|
|
357
|
-
tab_len=2,
|
|
358
|
-
)
|
|
359
|
-
line += f"{tabs_bf_format}{node[1].split()[1]}"
|
|
356
|
+
line += f"{tab}{node[1].split()[0]} {node[1].split()[1]}"
|
|
360
357
|
tree.add(line)
|
|
361
358
|
|
|
362
359
|
return tree, tree_length
|
|
@@ -7,30 +7,28 @@
|
|
|
7
7
|
# Standard library
|
|
8
8
|
import concurrent.futures
|
|
9
9
|
import itertools
|
|
10
|
+
import json
|
|
10
11
|
import logging
|
|
11
12
|
import pathlib
|
|
12
|
-
import json
|
|
13
13
|
|
|
14
14
|
# Installed
|
|
15
15
|
import boto3
|
|
16
16
|
import botocore
|
|
17
17
|
from rich.markup import escape
|
|
18
|
-
from rich.progress import Progress, SpinnerColumn
|
|
18
|
+
from rich.progress import BarColumn, Progress, SpinnerColumn
|
|
19
19
|
|
|
20
20
|
# Own modules
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
import dds_cli
|
|
22
|
+
import dds_cli.directory
|
|
23
|
+
import dds_cli.utils
|
|
24
|
+
from dds_cli import DDSEndpoint, base
|
|
23
25
|
from dds_cli import data_remover as dr
|
|
24
|
-
from dds_cli import
|
|
26
|
+
from dds_cli import exceptions
|
|
25
27
|
from dds_cli import file_encryptor as fe
|
|
26
28
|
from dds_cli import file_handler_local as fhl
|
|
27
29
|
from dds_cli import status
|
|
28
30
|
from dds_cli import text_handler as txt
|
|
29
|
-
from dds_cli.custom_decorators import
|
|
30
|
-
|
|
31
|
-
import dds_cli
|
|
32
|
-
import dds_cli.directory
|
|
33
|
-
import dds_cli.utils
|
|
31
|
+
from dds_cli.custom_decorators import subpath_required, update_status, verify_proceed
|
|
34
32
|
|
|
35
33
|
###############################################################################
|
|
36
34
|
# START LOGGING CONFIG ################################# START LOGGING CONFIG #
|
|
@@ -274,7 +272,16 @@ class DataPutter(base.DDSBaseClass):
|
|
|
274
272
|
if not self.filehandler.data:
|
|
275
273
|
if self.temporary_directory and self.temporary_directory.is_dir():
|
|
276
274
|
LOG.debug("Deleting temporary folder %s.", self.temporary_directory)
|
|
277
|
-
|
|
275
|
+
try:
|
|
276
|
+
dds_cli.utils.delete_folder(self.temporary_directory)
|
|
277
|
+
except OSError as err:
|
|
278
|
+
# Folder deletion may fail if log file is still being written to
|
|
279
|
+
# This is not critical - the important thing is to show the error message
|
|
280
|
+
LOG.debug(
|
|
281
|
+
"Could not delete staging directory %s: %s",
|
|
282
|
+
self.temporary_directory,
|
|
283
|
+
err,
|
|
284
|
+
)
|
|
278
285
|
raise exceptions.UploadError(
|
|
279
286
|
"The specified data has already been uploaded. If you wish to redo the upload, "
|
|
280
287
|
"use the '--overwrite' flag. Please use with caution as previously uploaded data "
|
|
@@ -104,8 +104,7 @@ class Compressor:
|
|
|
104
104
|
# if not chunk:
|
|
105
105
|
# break
|
|
106
106
|
# yield
|
|
107
|
-
|
|
108
|
-
yield chunk
|
|
107
|
+
yield from iter(lambda: compressor.read(chunk_size), b"")
|
|
109
108
|
except Exception as err: # pylint: disable=broad-exception-caught
|
|
110
109
|
LOG.warning(str(err))
|
|
111
110
|
else:
|
|
@@ -92,8 +92,7 @@ class LocalFileHandler(fh.FileHandler):
|
|
|
92
92
|
|
|
93
93
|
try:
|
|
94
94
|
with file.open(mode="rb") as infile:
|
|
95
|
-
|
|
96
|
-
yield chunk
|
|
95
|
+
yield from iter(lambda: infile.read(chunk_size), b"")
|
|
97
96
|
except OSError as err:
|
|
98
97
|
LOG.warning(str(err))
|
|
99
98
|
|
|
@@ -278,11 +277,10 @@ class LocalFileHandler(fh.FileHandler):
|
|
|
278
277
|
# LOG.debug(
|
|
279
278
|
# "Test: %s",
|
|
280
279
|
# )
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
yield chunk
|
|
280
|
+
yield from fc.Compressor.compress_file(file=file_info["path_raw"])
|
|
281
|
+
# LOG.debug("Chunk type: %s", type(chunk))
|
|
282
|
+
# checksum.update(chunk)
|
|
283
|
+
# break
|
|
286
284
|
|
|
287
285
|
# LOG.debug("Streaming file finished.")
|
|
288
286
|
# Add checksum to file info
|
|
@@ -102,6 +102,9 @@ class ProjectStatusManager(base.DDSBaseClass):
|
|
|
102
102
|
def update_status(self, new_status, deadline=None, is_aborted=False, no_mail=False):
|
|
103
103
|
"""Update project status"""
|
|
104
104
|
|
|
105
|
+
if deadline is not None and deadline <= 0:
|
|
106
|
+
raise exceptions.DDSCLIException("Deadline must be a positive number of days.")
|
|
107
|
+
|
|
105
108
|
extra_params = {"new_status": new_status, "send_email": not no_mail}
|
|
106
109
|
if deadline:
|
|
107
110
|
extra_params["deadline"] = deadline
|
|
@@ -219,7 +222,7 @@ class ProjectStatusManager(base.DDSBaseClass):
|
|
|
219
222
|
dds_cli.utils.console.print(print_info)
|
|
220
223
|
|
|
221
224
|
# If it wasnt provided during the command click, ask the user for the new deadline
|
|
222
|
-
if
|
|
225
|
+
if new_deadline is None:
|
|
223
226
|
# Question number of days to extend the deadline
|
|
224
227
|
prompt_question = (
|
|
225
228
|
"How many days would you like to extend the project deadline with? "
|
|
@@ -227,6 +230,12 @@ class ProjectStatusManager(base.DDSBaseClass):
|
|
|
227
230
|
)
|
|
228
231
|
new_deadline = rich.prompt.IntPrompt.ask(prompt_question, default=default_unit_days)
|
|
229
232
|
|
|
233
|
+
# Validate that the deadline extension is positive
|
|
234
|
+
if new_deadline <= 0:
|
|
235
|
+
raise exceptions.DDSCLIException(
|
|
236
|
+
"Deadline extension must be a positive number of days."
|
|
237
|
+
)
|
|
238
|
+
|
|
230
239
|
# Confirm operation question
|
|
231
240
|
new_deadline_date = parse(current_deadline) + datetime.timedelta(days=new_deadline)
|
|
232
241
|
new_deadline_date = new_deadline_date.strftime("%a,%d %b %Y %H:%M:%S")
|
|
@@ -234,7 +243,7 @@ class ProjectStatusManager(base.DDSBaseClass):
|
|
|
234
243
|
f"\nThe new deadline for project {project_id} will be: [b][blue]{new_deadline_date}[/b][/blue]"
|
|
235
244
|
"\n\n[b][blue]Are you sure [/b][/blue]you want to perform this operation? "
|
|
236
245
|
"\nYou can only extend the data availability a maximum of "
|
|
237
|
-
"[b][blue]
|
|
246
|
+
"[b][blue]2 times[/b][/blue], this consumes one of those times."
|
|
238
247
|
)
|
|
239
248
|
|
|
240
249
|
if not rich.prompt.Confirm.ask(prompt_question):
|
|
@@ -15,6 +15,7 @@ import botocore
|
|
|
15
15
|
|
|
16
16
|
# Own modules
|
|
17
17
|
import dds_cli.utils
|
|
18
|
+
from dds_cli import constants
|
|
18
19
|
from dds_cli import DDSEndpoint
|
|
19
20
|
|
|
20
21
|
###############################################################################
|
|
@@ -75,6 +76,14 @@ class S3Connector:
|
|
|
75
76
|
endpoint_url=self.url,
|
|
76
77
|
aws_access_key_id=self.keys["access_key"],
|
|
77
78
|
aws_secret_access_key=self.keys["secret_key"],
|
|
79
|
+
config=botocore.client.Config(
|
|
80
|
+
read_timeout=constants.READ_TIMEOUT,
|
|
81
|
+
connect_timeout=constants.CONNECT_TIMEOUT,
|
|
82
|
+
retries={
|
|
83
|
+
"max_attempts": 10,
|
|
84
|
+
# TODO: Add retry strategy mode="standard" when boto3 version >= 1.26.0
|
|
85
|
+
},
|
|
86
|
+
),
|
|
78
87
|
)
|
|
79
88
|
except (boto3.exceptions.Boto3Error, botocore.exceptions.BotoCoreError) as err:
|
|
80
89
|
LOG.warning("S3 connection failed: %s", err)
|
|
@@ -76,7 +76,7 @@ class User:
|
|
|
76
76
|
LOG.debug("Starting authentication on the API...")
|
|
77
77
|
|
|
78
78
|
# If no username or password is provided, prompt for them
|
|
79
|
-
if not username
|
|
79
|
+
if not username or not password:
|
|
80
80
|
if self.no_prompt:
|
|
81
81
|
raise exceptions.AuthenticationError(
|
|
82
82
|
message=(
|
|
@@ -88,12 +88,12 @@ class User:
|
|
|
88
88
|
username = Prompt.ask("DDS username")
|
|
89
89
|
password = getpass.getpass(prompt="DDS password: ")
|
|
90
90
|
|
|
91
|
-
if username
|
|
91
|
+
if not username:
|
|
92
92
|
raise exceptions.AuthenticationError(
|
|
93
93
|
message="Non-empty username needed to be able to authenticate."
|
|
94
94
|
)
|
|
95
95
|
|
|
96
|
-
if password
|
|
96
|
+
if not password:
|
|
97
97
|
raise exceptions.AuthenticationError(
|
|
98
98
|
message="Non-empty password needed to be able to authenticate."
|
|
99
99
|
)
|
|
@@ -289,7 +289,7 @@ class TokenFile:
|
|
|
289
289
|
if token_path is None:
|
|
290
290
|
self.token_file = dds_cli.TOKEN_FILE
|
|
291
291
|
else:
|
|
292
|
-
self.token_file = pathlib.Path(
|
|
292
|
+
self.token_file = pathlib.Path(token_path).expanduser()
|
|
293
293
|
|
|
294
294
|
def read_token(self):
|
|
295
295
|
"""Attempts to fetch a valid token from the token file.
|
|
@@ -1,39 +1,38 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dds_cli
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.14.0
|
|
4
4
|
Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
|
|
5
5
|
Home-page: https://github.com/ScilifelabDataCentre/dds_cli
|
|
6
6
|
Author: SciLifeLab Data Centre
|
|
7
7
|
License: MIT
|
|
8
8
|
Classifier: Development Status :: 4 - Beta
|
|
9
9
|
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
12
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
13
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
14
|
Description-Content-Type: text/markdown
|
|
16
15
|
License-File: LICENSE
|
|
17
16
|
Requires-Dist: boto3==1.24.73
|
|
18
17
|
Requires-Dist: botocore==1.27.73
|
|
19
18
|
Requires-Dist: click==8.1.3
|
|
20
19
|
Requires-Dist: click-pathlib==2020.3.13.0
|
|
21
|
-
Requires-Dist: cryptography==
|
|
20
|
+
Requires-Dist: cryptography==44.0.1
|
|
22
21
|
Requires-Dist: immutabledict==2.2.1
|
|
23
22
|
Requires-Dist: jwcrypto==1.5.6
|
|
24
23
|
Requires-Dist: prettytable==3.7.0
|
|
25
24
|
Requires-Dist: prompt-toolkit==3.0.40
|
|
26
|
-
Requires-Dist: PyNaCl==1.
|
|
25
|
+
Requires-Dist: PyNaCl==1.6.2
|
|
27
26
|
Requires-Dist: pytz==2022.2.1
|
|
28
27
|
Requires-Dist: PyYAML==6.0.2
|
|
29
|
-
Requires-Dist: questionary==1.
|
|
30
|
-
Requires-Dist: requests==2.32.
|
|
28
|
+
Requires-Dist: questionary==2.1.1
|
|
29
|
+
Requires-Dist: requests==2.32.4
|
|
31
30
|
Requires-Dist: rich==13.6.0
|
|
32
31
|
Requires-Dist: rich-click==1.5.2
|
|
33
32
|
Requires-Dist: simplejson==3.17.6
|
|
34
33
|
Requires-Dist: tzlocal==4.2
|
|
35
34
|
Requires-Dist: zstandard==0.23.0
|
|
36
|
-
Requires-Dist:
|
|
35
|
+
Requires-Dist: legacy-cgi==2.6.1; python_version >= "3.13"
|
|
37
36
|
Dynamic: author
|
|
38
37
|
Dynamic: classifier
|
|
39
38
|
Dynamic: description
|