dds-cli 2.14.0__tar.gz → 2.14.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dds_cli-2.14.0 → dds_cli-2.14.2}/PKG-INFO +9 -10
- {dds_cli-2.14.0 → dds_cli-2.14.2}/README.md +4 -4
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/__init__.py +1 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/__main__.py +44 -46
- dds_cli-2.14.2/dds_cli/constants.py +25 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/custom_decorators.py +0 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/data_getter.py +57 -23
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/data_lister.py +1 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/data_putter.py +0 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/file_compressor.py +4 -5
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/file_encryptor.py +1 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/file_handler_local.py +17 -18
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/file_handler_remote.py +2 -4
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/project_info.py +1 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/project_status.py +2 -2
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/user.py +4 -4
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/utils.py +4 -6
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/version.py +1 -1
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/PKG-INFO +9 -10
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/requires.txt +4 -5
- dds_cli-2.14.2/pyproject.toml +16 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/setup.py +4 -6
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_auth.py +18 -16
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_commands.py +24 -24
- dds_cli-2.14.2/tests/test_data_getter.py +314 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_decorators.py +13 -12
- dds_cli-2.14.0/dds_cli/constants.py +0 -14
- dds_cli-2.14.0/pyproject.toml +0 -3
- dds_cli-2.14.0/tests/test_data_getter.py +0 -133
- {dds_cli-2.14.0 → dds_cli-2.14.2}/LICENSE +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/account_manager.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/auth.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/base.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/data_remover.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/directory.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/exceptions.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/file_handler.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/message_helper.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/motd_manager.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/options.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/project_creator.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/s3_connector.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/status.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/superadmin_helper.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/text_handler.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/timestamp.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli/unit_manager.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/SOURCES.txt +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/dependency_links.txt +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/entry_points.txt +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/not-zip-safe +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/dds_cli.egg-info/top_level.txt +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/setup.cfg +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/__init__.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_account_manager.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_base.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_data_putter.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_data_remover.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_file_compressor.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_file_encryptor.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_file_handler_local.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_motd_manager.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_project_status.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_s3_connector.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_superadmin_helper.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_user.py +0 -0
- {dds_cli-2.14.0 → dds_cli-2.14.2}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dds_cli
|
|
3
|
-
Version: 2.14.
|
|
3
|
+
Version: 2.14.2
|
|
4
4
|
Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
|
|
5
5
|
Home-page: https://github.com/ScilifelabDataCentre/dds_cli
|
|
6
6
|
Author: SciLifeLab Data Centre
|
|
@@ -16,18 +16,17 @@ License-File: LICENSE
|
|
|
16
16
|
Requires-Dist: boto3==1.24.73
|
|
17
17
|
Requires-Dist: botocore==1.27.73
|
|
18
18
|
Requires-Dist: click==8.1.3
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist: cryptography==44.0.1
|
|
19
|
+
Requires-Dist: cryptography==46.0.6
|
|
21
20
|
Requires-Dist: immutabledict==2.2.1
|
|
22
21
|
Requires-Dist: jwcrypto==1.5.6
|
|
23
22
|
Requires-Dist: prettytable==3.7.0
|
|
24
23
|
Requires-Dist: prompt-toolkit==3.0.40
|
|
25
24
|
Requires-Dist: PyNaCl==1.6.2
|
|
26
|
-
Requires-Dist: pytz==
|
|
25
|
+
Requires-Dist: pytz==2026.1.post1
|
|
27
26
|
Requires-Dist: PyYAML==6.0.2
|
|
28
27
|
Requires-Dist: questionary==2.1.1
|
|
29
|
-
Requires-Dist: requests==2.
|
|
30
|
-
Requires-Dist: rich==
|
|
28
|
+
Requires-Dist: requests==2.33.0
|
|
29
|
+
Requires-Dist: rich==14.3.3
|
|
31
30
|
Requires-Dist: rich-click==1.5.2
|
|
32
31
|
Requires-Dist: simplejson==3.17.6
|
|
33
32
|
Requires-Dist: tzlocal==4.2
|
|
@@ -60,15 +59,15 @@ Dynamic: summary
|
|
|
60
59
|
<a href="https://opensource.org/licenses/MIT">
|
|
61
60
|
<img alt="Licence: MIT" src="https://img.shields.io/badge/License-MIT-yellow.svg">
|
|
62
61
|
</a>
|
|
63
|
-
<a href="https://github.com/
|
|
64
|
-
<img alt="
|
|
62
|
+
<a href="https://github.com/astral-sh/ruff">
|
|
63
|
+
<img alt="Linting: ruff" src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json">
|
|
65
64
|
</a>
|
|
66
65
|
<a href="https://prettier.io/">
|
|
67
66
|
<img alt="Code style: prettier" src="https://img.shields.io/badge/code_style-prettier-ff69b4.svg">
|
|
68
67
|
</a>
|
|
69
68
|
<br />
|
|
70
|
-
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-
|
|
71
|
-
<img alt="
|
|
69
|
+
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml">
|
|
70
|
+
<img alt="Linting: ruff" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml/badge.svg?event=push">
|
|
72
71
|
</a>
|
|
73
72
|
<img alt="CodeQL" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/codeql-cli.yml/badge.svg">
|
|
74
73
|
<a href="https://codecov.io/github/ScilifelabDataCentre/dds_cli" >
|
|
@@ -15,15 +15,15 @@
|
|
|
15
15
|
<a href="https://opensource.org/licenses/MIT">
|
|
16
16
|
<img alt="Licence: MIT" src="https://img.shields.io/badge/License-MIT-yellow.svg">
|
|
17
17
|
</a>
|
|
18
|
-
<a href="https://github.com/
|
|
19
|
-
<img alt="
|
|
18
|
+
<a href="https://github.com/astral-sh/ruff">
|
|
19
|
+
<img alt="Linting: ruff" src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json">
|
|
20
20
|
</a>
|
|
21
21
|
<a href="https://prettier.io/">
|
|
22
22
|
<img alt="Code style: prettier" src="https://img.shields.io/badge/code_style-prettier-ff69b4.svg">
|
|
23
23
|
</a>
|
|
24
24
|
<br />
|
|
25
|
-
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-
|
|
26
|
-
<img alt="
|
|
25
|
+
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml">
|
|
26
|
+
<img alt="Linting: ruff" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml/badge.svg?event=push">
|
|
27
27
|
</a>
|
|
28
28
|
<img alt="CodeQL" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/codeql-cli.yml/badge.svg">
|
|
29
29
|
<a href="https://codecov.io/github/ScilifelabDataCentre/dds_cli" >
|
|
@@ -13,7 +13,6 @@ import sys
|
|
|
13
13
|
|
|
14
14
|
# Installed
|
|
15
15
|
|
|
16
|
-
import click_pathlib
|
|
17
16
|
import questionary
|
|
18
17
|
import rich
|
|
19
18
|
import rich.logging
|
|
@@ -200,12 +199,11 @@ def dds_main(click_ctx, verbose, force_no_log, log_file, no_prompt, token_path):
|
|
|
200
199
|
else:
|
|
201
200
|
file_handler = dds_cli.utils.setup_logging_to_file(filename=log_file)
|
|
202
201
|
LOG.addHandler(file_handler)
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
click_ctx.obj.update({"DEFAULT_LOG": False})
|
|
202
|
+
elif force_no_log:
|
|
203
|
+
LOG.warning(
|
|
204
|
+
"You have chosen to turn off the recommended default logging with the '--force-no-log' option."
|
|
205
|
+
)
|
|
206
|
+
click_ctx.obj.update({"DEFAULT_LOG": False})
|
|
209
207
|
|
|
210
208
|
|
|
211
209
|
# ************************************************************************************************ #
|
|
@@ -269,29 +267,27 @@ def list_projects_and_contents(
|
|
|
269
267
|
projects = lister.list_projects(sort_by=sort, show_all=show_all)
|
|
270
268
|
if json:
|
|
271
269
|
dds_cli.utils.console.print_json(data=projects)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
LOG.debug("No project entered, exiting.")
|
|
294
|
-
break
|
|
270
|
+
# If an interactive terminal, ask user if they want to view files for a project
|
|
271
|
+
elif sys.stdout.isatty() and not lister.no_prompt:
|
|
272
|
+
project_ids = [p["Project ID"] for p in projects]
|
|
273
|
+
LOG.info(
|
|
274
|
+
"Would you like to view files in a specific project? Leave blank to exit."
|
|
275
|
+
)
|
|
276
|
+
# Keep asking until we get a valid response
|
|
277
|
+
while project not in project_ids:
|
|
278
|
+
try:
|
|
279
|
+
project = questionary.autocomplete(
|
|
280
|
+
"Project ID:",
|
|
281
|
+
choices=project_ids,
|
|
282
|
+
validate=lambda x: x in project_ids or x == "",
|
|
283
|
+
style=dds_cli.dds_questionary_styles,
|
|
284
|
+
).unsafe_ask()
|
|
285
|
+
assert project and project != ""
|
|
286
|
+
|
|
287
|
+
# If didn't enter anything, convert to None and exit
|
|
288
|
+
except (KeyboardInterrupt, AssertionError):
|
|
289
|
+
LOG.debug("No project entered, exiting.")
|
|
290
|
+
break
|
|
295
291
|
|
|
296
292
|
# List all files in a project if we know a project ID
|
|
297
293
|
if project:
|
|
@@ -783,15 +779,14 @@ def delete_user(click_ctx, email, self, is_invite):
|
|
|
783
779
|
proceed_deletion = rich.prompt.Confirm.ask(
|
|
784
780
|
f"Delete invitation of {email} to Data Delivery System?"
|
|
785
781
|
)
|
|
782
|
+
elif self:
|
|
783
|
+
proceed_deletion = rich.prompt.Confirm.ask(
|
|
784
|
+
"Are you sure? Deleted accounts can't be restored!"
|
|
785
|
+
)
|
|
786
786
|
else:
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
)
|
|
791
|
-
else:
|
|
792
|
-
proceed_deletion = rich.prompt.Confirm.ask(
|
|
793
|
-
f"Delete Data Delivery System user account associated with {email}"
|
|
794
|
-
)
|
|
787
|
+
proceed_deletion = rich.prompt.Confirm.ask(
|
|
788
|
+
f"Delete Data Delivery System user account associated with {email}"
|
|
789
|
+
)
|
|
795
790
|
|
|
796
791
|
if proceed_deletion:
|
|
797
792
|
try:
|
|
@@ -1630,7 +1625,9 @@ def data_group_command(_):
|
|
|
1630
1625
|
"--mount-dir",
|
|
1631
1626
|
"-md",
|
|
1632
1627
|
required=False,
|
|
1633
|
-
type=
|
|
1628
|
+
type=click.Path(
|
|
1629
|
+
exists=False, file_okay=False, dir_okay=True, resolve_path=True, path_type=pathlib.Path
|
|
1630
|
+
),
|
|
1634
1631
|
help=(
|
|
1635
1632
|
"New directory where the files will be mounted before upload "
|
|
1636
1633
|
"and any error log files will be saved for a specific upload."
|
|
@@ -1753,7 +1750,9 @@ def put_data(
|
|
|
1753
1750
|
@source_path_file_option()
|
|
1754
1751
|
@destination_option(
|
|
1755
1752
|
help_message="Destination of downloaded data.",
|
|
1756
|
-
option_type=
|
|
1753
|
+
option_type=click.Path(
|
|
1754
|
+
exists=False, file_okay=False, dir_okay=True, resolve_path=True, path_type=pathlib.Path
|
|
1755
|
+
),
|
|
1757
1756
|
)
|
|
1758
1757
|
# Flags
|
|
1759
1758
|
@break_on_fail_flag(help_message="Cancel download of all files if one fails.")
|
|
@@ -2020,12 +2019,11 @@ def rm_data(click_ctx, project, file, folder, rm_all):
|
|
|
2020
2019
|
if rm_all:
|
|
2021
2020
|
if no_prompt:
|
|
2022
2021
|
LOG.warning("Deleting all files within project '%s'", project)
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
)
|
|
2027
|
-
|
|
2028
|
-
sys.exit(0)
|
|
2022
|
+
elif not rich.prompt.Confirm.ask(
|
|
2023
|
+
f"Are you sure you want to delete all files within project '{project}'?"
|
|
2024
|
+
):
|
|
2025
|
+
LOG.info("Probably for the best. Exiting.")
|
|
2026
|
+
sys.exit(0)
|
|
2029
2027
|
|
|
2030
2028
|
try:
|
|
2031
2029
|
with dds_cli.data_remover.DataRemover(
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Shared constants used in the DDS CLI.
|
|
2
|
+
|
|
3
|
+
Using this will avoid e.g. circular imports and
|
|
4
|
+
make it easier to maintain.
|
|
5
|
+
|
|
6
|
+
TODO: Move other constants here from __init__.py.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
# Timeout settings for upload and download
|
|
10
|
+
READ_TIMEOUT = 300
|
|
11
|
+
CONNECT_TIMEOUT = 60
|
|
12
|
+
|
|
13
|
+
# Retry settings for download
|
|
14
|
+
DOWNLOAD_MAX_RETRIES = 5
|
|
15
|
+
DOWNLOAD_BACKOFF_FACTOR = 2
|
|
16
|
+
DOWNLOAD_INITIAL_WAIT = 1 # seconds
|
|
17
|
+
|
|
18
|
+
# Import these constants when using '*'
|
|
19
|
+
__all__ = [
|
|
20
|
+
"READ_TIMEOUT",
|
|
21
|
+
"CONNECT_TIMEOUT",
|
|
22
|
+
"DOWNLOAD_MAX_RETRIES",
|
|
23
|
+
"DOWNLOAD_BACKOFF_FACTOR",
|
|
24
|
+
"DOWNLOAD_INITIAL_WAIT",
|
|
25
|
+
]
|
|
@@ -154,7 +154,6 @@ def removal_spinner(func):
|
|
|
154
154
|
SpinnerColumn(spinner_name="dots12", style="white"),
|
|
155
155
|
console=dds_cli.utils.stderr_console,
|
|
156
156
|
) as progress:
|
|
157
|
-
|
|
158
157
|
description: str | None = None # type hint, initialized
|
|
159
158
|
# Determine spinner text
|
|
160
159
|
if func.__name__ == "remove_all":
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
# Standard library
|
|
8
8
|
import logging
|
|
9
9
|
import pathlib
|
|
10
|
+
import time
|
|
10
11
|
|
|
11
12
|
# Installed
|
|
12
13
|
import requests
|
|
@@ -243,35 +244,68 @@ class DataGetter(base.DDSBaseClass):
|
|
|
243
244
|
error = ""
|
|
244
245
|
file_local = self.filehandler.data[file]["path_downloaded"]
|
|
245
246
|
file_remote = self.filehandler.data[file]["url"]
|
|
247
|
+
file_name_in_db = escape(str(self.filehandler.data[file]["name_in_db"]))
|
|
246
248
|
|
|
247
|
-
|
|
248
|
-
with requests.get(
|
|
249
|
-
file_remote,
|
|
250
|
-
stream=True,
|
|
251
|
-
timeout=(constants.CONNECT_TIMEOUT, constants.READ_TIMEOUT),
|
|
252
|
-
) as req:
|
|
253
|
-
req.raise_for_status()
|
|
254
|
-
with file_local.open(mode="wb") as new_file:
|
|
255
|
-
for chunk in req.iter_content(chunk_size=FileSegment.SEGMENT_SIZE_CIPHER):
|
|
256
|
-
progress.update(task, advance=len(chunk))
|
|
257
|
-
new_file.write(chunk)
|
|
258
|
-
except (
|
|
249
|
+
retryable_exceptions = (
|
|
259
250
|
requests.exceptions.ConnectTimeout,
|
|
260
|
-
requests.exceptions.HTTPError,
|
|
261
251
|
requests.exceptions.ReadTimeout,
|
|
262
252
|
requests.exceptions.Timeout,
|
|
263
253
|
requests.exceptions.ConnectionError,
|
|
264
|
-
)
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
max_retries = constants.DOWNLOAD_MAX_RETRIES
|
|
257
|
+
backoff_factor = constants.DOWNLOAD_BACKOFF_FACTOR
|
|
258
|
+
wait = constants.DOWNLOAD_INITIAL_WAIT
|
|
259
|
+
retry_messages = []
|
|
260
|
+
|
|
261
|
+
for attempt in range(1, max_retries + 1):
|
|
262
|
+
error = ""
|
|
263
|
+
if attempt > 1:
|
|
264
|
+
progress.reset(task, completed=0)
|
|
265
|
+
|
|
266
|
+
try:
|
|
267
|
+
with requests.get(
|
|
268
|
+
file_remote,
|
|
269
|
+
stream=True,
|
|
270
|
+
timeout=(constants.CONNECT_TIMEOUT, constants.READ_TIMEOUT),
|
|
271
|
+
) as req:
|
|
272
|
+
req.raise_for_status()
|
|
273
|
+
with file_local.open(mode="wb") as new_file:
|
|
274
|
+
for chunk in req.iter_content(chunk_size=FileSegment.SEGMENT_SIZE_CIPHER):
|
|
275
|
+
progress.update(task, advance=len(chunk))
|
|
276
|
+
new_file.write(chunk)
|
|
277
|
+
except (requests.exceptions.HTTPError, *retryable_exceptions) as err:
|
|
278
|
+
if (
|
|
279
|
+
isinstance(err, requests.exceptions.HTTPError)
|
|
280
|
+
and hasattr(err, "response")
|
|
281
|
+
and hasattr(err.response, "status_code")
|
|
282
|
+
and err.response.status_code == 404
|
|
283
|
+
):
|
|
284
|
+
error = "File not found! Please contact support."
|
|
285
|
+
break
|
|
272
286
|
error = str(err)
|
|
273
|
-
|
|
274
|
-
|
|
287
|
+
if attempt < max_retries:
|
|
288
|
+
retry_msg = (
|
|
289
|
+
f"Download attempt {attempt}/{max_retries} failed for "
|
|
290
|
+
f"'{file_name_in_db}': {error}. Retrying in {wait}s..."
|
|
291
|
+
)
|
|
292
|
+
retry_messages.append(retry_msg)
|
|
293
|
+
LOG.warning(retry_msg)
|
|
294
|
+
time.sleep(wait)
|
|
295
|
+
wait *= backoff_factor
|
|
296
|
+
else:
|
|
297
|
+
downloaded = True
|
|
298
|
+
break
|
|
299
|
+
|
|
300
|
+
if not downloaded and error:
|
|
301
|
+
if retry_messages:
|
|
302
|
+
error = " | ".join(retry_messages) + f" | Final error: {error}"
|
|
303
|
+
LOG.error(
|
|
304
|
+
"Download failed for '%s' after %d attempts: %s",
|
|
305
|
+
file_name_in_db,
|
|
306
|
+
max_retries,
|
|
307
|
+
error,
|
|
308
|
+
)
|
|
275
309
|
|
|
276
310
|
return downloaded, error
|
|
277
311
|
|
|
@@ -233,7 +233,7 @@ class DataLister(base.DDSBaseClass):
|
|
|
233
233
|
error_message="Failed to list the project's directory tree",
|
|
234
234
|
)
|
|
235
235
|
|
|
236
|
-
if
|
|
236
|
+
if "files_folders" not in resp_json:
|
|
237
237
|
raise exceptions.NoDataError(f"Could not find folder: '{folder}'")
|
|
238
238
|
|
|
239
239
|
sorted_files_folders = sorted(resp_json["files_folders"], key=lambda f: f["name"])
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Compressor module. Handles the compression of files."""
|
|
2
2
|
|
|
3
3
|
###############################################################################
|
|
4
4
|
# IMPORTS ########################################################### IMPORTS #
|
|
@@ -10,7 +10,6 @@ import logging
|
|
|
10
10
|
import pathlib
|
|
11
11
|
import traceback
|
|
12
12
|
|
|
13
|
-
|
|
14
13
|
# Installed
|
|
15
14
|
import zstandard as zstd
|
|
16
15
|
from rich.markup import escape
|
|
@@ -36,7 +35,7 @@ class CompressionMagic:
|
|
|
36
35
|
LZIP = b"LZIP"
|
|
37
36
|
RAR4 = b"Rar!\x1a\x07\x00"
|
|
38
37
|
RAR5 = b"Rar!\x1a\x07\x01\x00"
|
|
39
|
-
GZIP = b"\
|
|
38
|
+
GZIP = b"\x1f\x8b"
|
|
40
39
|
ZSTANDARD = b"(\xb5/\xfd"
|
|
41
40
|
|
|
42
41
|
|
|
@@ -55,7 +54,7 @@ class Compressor:
|
|
|
55
54
|
b"_'\xa8\x89": "jar",
|
|
56
55
|
b"ZOO ": "zoo",
|
|
57
56
|
b"PK\x03\x04": "zip",
|
|
58
|
-
b"\
|
|
57
|
+
b"\x1f\x8b": "gzip",
|
|
59
58
|
b"UFA\xc6\xd2\xc1": "ufa",
|
|
60
59
|
b"StuffIt ": "sit",
|
|
61
60
|
b"Rar!\x1a\x07\x00": "rar v4.x",
|
|
@@ -105,7 +104,7 @@ class Compressor:
|
|
|
105
104
|
# break
|
|
106
105
|
# yield
|
|
107
106
|
yield from iter(lambda: compressor.read(chunk_size), b"")
|
|
108
|
-
except Exception as err:
|
|
107
|
+
except Exception as err:
|
|
109
108
|
LOG.warning(str(err))
|
|
110
109
|
else:
|
|
111
110
|
LOG.debug("Compression of '%s' finished.", file)
|
|
@@ -83,7 +83,7 @@ class LocalFileHandler(fh.FileHandler):
|
|
|
83
83
|
called in the bucket."""
|
|
84
84
|
|
|
85
85
|
# Generate new file name
|
|
86
|
-
new_name = f"{'%020x' % random.randrange(16**20)}_{uuid.uuid5(uuid.NAMESPACE_X500, str(folder))}{uuid.uuid5(uuid.NAMESPACE_X500, filename)}"
|
|
86
|
+
new_name = f"{'%020x' % random.randrange(16**20)}_{uuid.uuid5(uuid.NAMESPACE_X500, str(folder))}{uuid.uuid5(uuid.NAMESPACE_X500, filename)}"
|
|
87
87
|
return new_name
|
|
88
88
|
|
|
89
89
|
@staticmethod
|
|
@@ -147,26 +147,25 @@ class LocalFileHandler(fh.FileHandler):
|
|
|
147
147
|
folder=path_key,
|
|
148
148
|
)
|
|
149
149
|
file_info.update({**content_info})
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
)
|
|
160
|
-
else:
|
|
161
|
-
LOG.warning(
|
|
162
|
-
"IGNORED: Link: '%s' -> '%s' seems to be broken, will be ignored.",
|
|
163
|
-
path,
|
|
164
|
-
resolved,
|
|
165
|
-
)
|
|
150
|
+
# Symlinks are also identified as files - if here and symlink --> broken
|
|
151
|
+
elif path.is_symlink():
|
|
152
|
+
try:
|
|
153
|
+
resolved = path.resolve()
|
|
154
|
+
except RuntimeError:
|
|
155
|
+
LOG.warning(
|
|
156
|
+
"IGNORED: Link: '%s' seems to contain infinite loop, will be ignored.",
|
|
157
|
+
path,
|
|
158
|
+
)
|
|
166
159
|
else:
|
|
167
160
|
LOG.warning(
|
|
168
|
-
"IGNORED:
|
|
161
|
+
"IGNORED: Link: '%s' -> '%s' seems to be broken, will be ignored.",
|
|
162
|
+
path,
|
|
163
|
+
resolved,
|
|
169
164
|
)
|
|
165
|
+
else:
|
|
166
|
+
LOG.warning(
|
|
167
|
+
"IGNORED: Path of unsupported/unknown type: '%s', will be ignored.", path
|
|
168
|
+
)
|
|
170
169
|
|
|
171
170
|
return file_info, progress_tasks
|
|
172
171
|
|
|
@@ -100,8 +100,7 @@ class RemoteFileHandler(fh.FileHandler):
|
|
|
100
100
|
|
|
101
101
|
# Save info on files in dict and return
|
|
102
102
|
data = {
|
|
103
|
-
self.local_destination
|
|
104
|
-
/ pathlib.Path(x): {
|
|
103
|
+
self.local_destination / pathlib.Path(x): {
|
|
105
104
|
**y,
|
|
106
105
|
"name_in_db": x,
|
|
107
106
|
"path_downloaded": self.local_destination
|
|
@@ -115,8 +114,7 @@ class RemoteFileHandler(fh.FileHandler):
|
|
|
115
114
|
for _, folder_item in folder_contents.items():
|
|
116
115
|
data.update(
|
|
117
116
|
{
|
|
118
|
-
self.local_destination
|
|
119
|
-
/ pathlib.Path(j): {
|
|
117
|
+
self.local_destination / pathlib.Path(j): {
|
|
120
118
|
**k,
|
|
121
119
|
"name_in_db": j,
|
|
122
120
|
"path_downloaded": self.local_destination
|
|
@@ -57,7 +57,7 @@ class ProjectInfoManager(base.DDSBaseClass):
|
|
|
57
57
|
dds_cli.utils.console.print(f"[b]Project title:[/b] {project_info['Title']}")
|
|
58
58
|
dds_cli.utils.console.print(f"[b]Project description:[/b] {project_info['Description']}")
|
|
59
59
|
|
|
60
|
-
def update_info(self, title=None, description=None, pi=None):
|
|
60
|
+
def update_info(self, title=None, description=None, pi=None):
|
|
61
61
|
"""Update project info"""
|
|
62
62
|
|
|
63
63
|
if all(item is None for item in [title, description, pi]):
|
|
@@ -307,7 +307,7 @@ class ProjectBusyStatusManager(base.DDSBaseClass):
|
|
|
307
307
|
else:
|
|
308
308
|
projects: typing.Dict = response_json.get("projects")
|
|
309
309
|
LOG.info("The following projects are busy:")
|
|
310
|
-
for proj in projects:
|
|
311
|
-
dds_cli.utils.console.print(f"{proj}: updated on {
|
|
310
|
+
for proj, updated_on in projects.items():
|
|
311
|
+
dds_cli.utils.console.print(f"{proj}: updated on {updated_on}")
|
|
312
312
|
else:
|
|
313
313
|
LOG.info("There are no busy projects at the moment.")
|
|
@@ -275,7 +275,7 @@ class User:
|
|
|
275
275
|
)
|
|
276
276
|
# Get response
|
|
277
277
|
username = response_json["info"]["username"]
|
|
278
|
-
except: #
|
|
278
|
+
except: # noqa: E722
|
|
279
279
|
pass
|
|
280
280
|
return username
|
|
281
281
|
|
|
@@ -307,7 +307,7 @@ class TokenFile:
|
|
|
307
307
|
self.check_token_file_permissions()
|
|
308
308
|
|
|
309
309
|
# Read token from file
|
|
310
|
-
with self.token_file.open(mode="r") as file: #
|
|
310
|
+
with self.token_file.open(mode="r") as file: # noqa: PLW1514
|
|
311
311
|
token = file.read()
|
|
312
312
|
if not token:
|
|
313
313
|
raise exceptions.TokenNotFoundError(message="Token file is empty.")
|
|
@@ -331,7 +331,7 @@ class TokenFile:
|
|
|
331
331
|
|
|
332
332
|
self.check_token_file_permissions()
|
|
333
333
|
|
|
334
|
-
with self.token_file.open("w") as file: #
|
|
334
|
+
with self.token_file.open("w") as file: # noqa: PLW1514
|
|
335
335
|
file.write(token)
|
|
336
336
|
|
|
337
337
|
if os.name == "nt":
|
|
@@ -421,7 +421,7 @@ class TokenFile:
|
|
|
421
421
|
return expiration_time
|
|
422
422
|
|
|
423
423
|
# Private methods ############################################################ Private methods #
|
|
424
|
-
def __token_dates(self, token): #
|
|
424
|
+
def __token_dates(self, token): # noqa: PLR1710
|
|
425
425
|
"""Returns the expiration time in UTC that is extracted from the token jose header."""
|
|
426
426
|
|
|
427
427
|
expiration_time = get_token_expiration_time(token=token)
|
|
@@ -49,9 +49,9 @@ class HumanBytes:
|
|
|
49
49
|
"""
|
|
50
50
|
assert isinstance(num, (int, float)), "num must be an int or float"
|
|
51
51
|
assert isinstance(metric, bool), "metric must be a bool"
|
|
52
|
-
assert (
|
|
53
|
-
|
|
54
|
-
)
|
|
52
|
+
assert isinstance(precision, int) and 0 <= precision <= 3, (
|
|
53
|
+
"precision must be an int (range 0-3)"
|
|
54
|
+
)
|
|
55
55
|
|
|
56
56
|
unit_labels = HumanBytes.METRIC_LABELS if metric else HumanBytes.BINARY_LABELS
|
|
57
57
|
last_label = unit_labels[-1]
|
|
@@ -324,9 +324,7 @@ def get_json_response(response):
|
|
|
324
324
|
return json_response
|
|
325
325
|
|
|
326
326
|
|
|
327
|
-
def format_api_response(
|
|
328
|
-
response, key: str, binary: bool = False, always_show: bool = False
|
|
329
|
-
): # pylint: disable=unused-argument
|
|
327
|
+
def format_api_response(response, key: str, binary: bool = False, always_show: bool = False):
|
|
330
328
|
"""Take a value e.g. bytes and reformat it to include a unit prefix."""
|
|
331
329
|
formatted_response = response
|
|
332
330
|
if isinstance(response, bool):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dds_cli
|
|
3
|
-
Version: 2.14.
|
|
3
|
+
Version: 2.14.2
|
|
4
4
|
Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
|
|
5
5
|
Home-page: https://github.com/ScilifelabDataCentre/dds_cli
|
|
6
6
|
Author: SciLifeLab Data Centre
|
|
@@ -16,18 +16,17 @@ License-File: LICENSE
|
|
|
16
16
|
Requires-Dist: boto3==1.24.73
|
|
17
17
|
Requires-Dist: botocore==1.27.73
|
|
18
18
|
Requires-Dist: click==8.1.3
|
|
19
|
-
Requires-Dist:
|
|
20
|
-
Requires-Dist: cryptography==44.0.1
|
|
19
|
+
Requires-Dist: cryptography==46.0.6
|
|
21
20
|
Requires-Dist: immutabledict==2.2.1
|
|
22
21
|
Requires-Dist: jwcrypto==1.5.6
|
|
23
22
|
Requires-Dist: prettytable==3.7.0
|
|
24
23
|
Requires-Dist: prompt-toolkit==3.0.40
|
|
25
24
|
Requires-Dist: PyNaCl==1.6.2
|
|
26
|
-
Requires-Dist: pytz==
|
|
25
|
+
Requires-Dist: pytz==2026.1.post1
|
|
27
26
|
Requires-Dist: PyYAML==6.0.2
|
|
28
27
|
Requires-Dist: questionary==2.1.1
|
|
29
|
-
Requires-Dist: requests==2.
|
|
30
|
-
Requires-Dist: rich==
|
|
28
|
+
Requires-Dist: requests==2.33.0
|
|
29
|
+
Requires-Dist: rich==14.3.3
|
|
31
30
|
Requires-Dist: rich-click==1.5.2
|
|
32
31
|
Requires-Dist: simplejson==3.17.6
|
|
33
32
|
Requires-Dist: tzlocal==4.2
|
|
@@ -60,15 +59,15 @@ Dynamic: summary
|
|
|
60
59
|
<a href="https://opensource.org/licenses/MIT">
|
|
61
60
|
<img alt="Licence: MIT" src="https://img.shields.io/badge/License-MIT-yellow.svg">
|
|
62
61
|
</a>
|
|
63
|
-
<a href="https://github.com/
|
|
64
|
-
<img alt="
|
|
62
|
+
<a href="https://github.com/astral-sh/ruff">
|
|
63
|
+
<img alt="Linting: ruff" src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json">
|
|
65
64
|
</a>
|
|
66
65
|
<a href="https://prettier.io/">
|
|
67
66
|
<img alt="Code style: prettier" src="https://img.shields.io/badge/code_style-prettier-ff69b4.svg">
|
|
68
67
|
</a>
|
|
69
68
|
<br />
|
|
70
|
-
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-
|
|
71
|
-
<img alt="
|
|
69
|
+
<a href="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml">
|
|
70
|
+
<img alt="Linting: ruff" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/lint-ruff.yml/badge.svg?event=push">
|
|
72
71
|
</a>
|
|
73
72
|
<img alt="CodeQL" src="https://github.com/ScilifelabDataCentre/dds_cli/actions/workflows/codeql-cli.yml/badge.svg">
|
|
74
73
|
<a href="https://codecov.io/github/ScilifelabDataCentre/dds_cli" >
|