dds-cli 2.2.63__tar.gz → 2.2.65__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {dds_cli-2.2.63 → dds_cli-2.2.65}/PKG-INFO +1 -1
  2. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/__init__.py +2 -1
  3. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/__main__.py +72 -23
  4. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/account_manager.py +15 -12
  5. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/auth.py +8 -3
  6. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/base.py +21 -23
  7. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/custom_decorators.py +13 -11
  8. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_getter.py +6 -7
  9. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_lister.py +64 -64
  10. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_putter.py +19 -14
  11. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/data_remover.py +14 -12
  12. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/directory.py +5 -4
  13. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_compressor.py +5 -5
  14. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_encryptor.py +7 -7
  15. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler.py +4 -5
  16. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler_local.py +20 -18
  17. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/file_handler_remote.py +3 -9
  18. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/motd_manager.py +2 -7
  19. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/options.py +9 -2
  20. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_creator.py +3 -4
  21. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_info.py +19 -11
  22. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/project_status.py +19 -20
  23. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/s3_connector.py +5 -6
  24. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/status.py +1 -1
  25. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/timestamp.py +7 -5
  26. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/unit_manager.py +0 -5
  27. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/user.py +34 -25
  28. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/utils.py +18 -13
  29. dds_cli-2.2.65/dds_cli/version.py +3 -0
  30. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/PKG-INFO +1 -1
  31. {dds_cli-2.2.63 → dds_cli-2.2.65}/setup.py +4 -2
  32. dds_cli-2.2.63/dds_cli/version.py +0 -1
  33. {dds_cli-2.2.63 → dds_cli-2.2.65}/LICENSE +0 -0
  34. {dds_cli-2.2.63 → dds_cli-2.2.65}/README.md +0 -0
  35. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/exceptions.py +0 -0
  36. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/maintenance_manager.py +0 -0
  37. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli/text_handler.py +0 -0
  38. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/SOURCES.txt +0 -0
  39. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/dependency_links.txt +0 -0
  40. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/entry_points.txt +0 -0
  41. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/not-zip-safe +0 -0
  42. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/requires.txt +0 -0
  43. {dds_cli-2.2.63 → dds_cli-2.2.65}/dds_cli.egg-info/top_level.txt +0 -0
  44. {dds_cli-2.2.63 → dds_cli-2.2.65}/pyproject.toml +0 -0
  45. {dds_cli-2.2.63 → dds_cli-2.2.65}/setup.cfg +0 -0
  46. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/__init__.py +0 -0
  47. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_account_manager.py +0 -0
  48. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_data_remover.py +0 -0
  49. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_compressor.py +0 -0
  50. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_encryptor.py +0 -0
  51. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_file_handler_local.py +0 -0
  52. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_maintenance_manager.py +0 -0
  53. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_motd_manager.py +0 -0
  54. {dds_cli-2.2.63 → dds_cli-2.2.65}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dds_cli
3
- Version: 2.2.63
3
+ Version: 2.2.65
4
4
  Summary: A command line tool to manage data and projects in the SciLifeLab Data Delivery System.
5
5
  Home-page: https://github.com/ScilifelabDataCentre/dds_cli
6
6
  Author: SciLifeLab Data Centre
@@ -1,3 +1,4 @@
1
+ # pylint: skip-file
1
2
  """DDS CLI."""
2
3
 
3
4
  import datetime
@@ -59,7 +60,7 @@ class DDSEndpoint:
59
60
  BASE_ENDPOINT_LOCAL = "http://127.0.0.1:5000/api/v1"
60
61
  BASE_ENDPOINT_DOCKER = "http://dds_backend:5000/api/v1"
61
62
  BASE_ENDPOINT_REMOTE = "https://delivery.scilifelab.se/api/v1"
62
- BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.dckube.scilifelab.se/api/v1"
63
+ BASE_ENDPOINT_REMOTE_TEST = "https://dds-dev.dckube3.scilifelab.se/api/v1"
63
64
  if os.getenv("DDS_CLI_ENV") == "development":
64
65
  BASE_ENDPOINT = BASE_ENDPOINT_LOCAL
65
66
  elif os.getenv("DDS_CLI_ENV") == "docker-dev":
@@ -8,8 +8,8 @@
8
8
  import concurrent.futures
9
9
  import itertools
10
10
  import logging
11
- import os
12
11
  import sys
12
+ import typing
13
13
 
14
14
  # Installed
15
15
  import pathlib
@@ -70,7 +70,6 @@ LOG = logging.getLogger("dds_cli")
70
70
  # Configuration for rich-click output
71
71
  click.rich_click.MAX_WIDTH = 100
72
72
 
73
-
74
73
  ## # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
75
74
  # #
76
75
  # MMMM MMMM AAAA II NNNN NN #
@@ -82,15 +81,15 @@ click.rich_click.MAX_WIDTH = 100
82
81
  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ##
83
82
 
84
83
 
85
- dds_url = dds_cli.DDSEndpoint.BASE_ENDPOINT
84
+ DDS_URL = dds_cli.DDSEndpoint.BASE_ENDPOINT
85
+ DDS_URL_BASE = DDS_URL[: DDS_URL.index("/", 8)]
86
+
86
87
  # Print header to STDERR
87
88
  dds_cli.utils.stderr_console.print(
88
89
  "[green] ︵",
89
90
  "\n[green] ︵ ( ) ︵",
90
91
  "\n[green]( ) ) ( ( )[/] [bold]SciLifeLab Data Delivery System",
91
- "\n[green] ︶ ( ) ) ([/] [blue][link={0}]{0}/[/link]".format(
92
- dds_url[: dds_url.index("/", 8)]
93
- ),
92
+ f"\n[green] ︶ ( ) ) ([/] [blue][link={DDS_URL_BASE}]{DDS_URL_BASE}/[/link]",
94
93
  f"\n[green] ︶ ( )[/] [dim]CLI Version {dds_cli.__version__}",
95
94
  "\n[green] ︶",
96
95
  highlight=False,
@@ -99,7 +98,7 @@ dds_cli.utils.stderr_console.print(
99
98
  if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] != "motd"):
100
99
  motds = dds_cli.motd_manager.MotdManager.list_all_active_motds(table=False)
101
100
  if motds:
102
- dds_cli.utils.stderr_console.print(f"[bold]Important information:[/bold]")
101
+ dds_cli.utils.stderr_console.print("[bold]Important information:[/bold]")
103
102
  for motd in motds:
104
103
  dds_cli.utils.stderr_console.print(f"{motd['Created']} - {motd['Message']} \n")
105
104
 
@@ -379,7 +378,10 @@ def auth_group_command(_):
379
378
  is_flag=True,
380
379
  required=False,
381
380
  default=False,
382
- help="[Not recommended, use with care] Allow read permissions to group. Sets 640 permission instead of 600, allowing others to access your authenticated session token.",
381
+ help=(
382
+ "[Not recommended, use with care] Allow read permissions to group. Sets 640 permission instead of 600, "
383
+ "allowing others to access your authenticated session token."
384
+ ),
383
385
  )
384
386
  @click.pass_obj
385
387
  def login(click_ctx, totp, allow_group):
@@ -403,7 +405,7 @@ def login(click_ctx, totp, allow_group):
403
405
  token_path=click_ctx.get("TOKEN_PATH"), totp=totp, allow_group=allow_group
404
406
  ):
405
407
  # Authentication token renewed in the init method.
406
- LOG.info(f"[green] :white_check_mark: Authentication successful![/green]")
408
+ LOG.info("[green] :white_check_mark: Authentication successful![/green]")
407
409
  except (
408
410
  dds_cli.exceptions.APIError,
409
411
  dds_cli.exceptions.AuthenticationError,
@@ -561,7 +563,7 @@ def list_users(click_ctx, unit, invites):
561
563
  token_path=click_ctx.get("TOKEN_PATH"),
562
564
  ) as lister:
563
565
  if invites:
564
- lister.list_invites(unit=unit, invites=invites)
566
+ lister.list_invites(invites=invites)
565
567
  else:
566
568
  lister.list_users(unit=unit)
567
569
 
@@ -582,7 +584,7 @@ def list_users(click_ctx, unit, invites):
582
584
  required=True, help_message="[Super Admins only] The username of the account you want to check."
583
585
  )
584
586
  @click.pass_obj
585
- def list_users(click_ctx, username):
587
+ def find_users(click_ctx, username):
586
588
  """Check if a username is registered to an account in the DDS."""
587
589
  try:
588
590
  with dds_cli.account_manager.AccountManager(
@@ -620,7 +622,7 @@ def list_users(click_ctx, username):
620
622
  ),
621
623
  help=(
622
624
  "Type of account. To include a space in the chosen role, use quotes "
623
- '(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
625
+ r'(e.g. "Unit Personnel") or escape the space (e.g. Unit\ Personnel)'
624
626
  ),
625
627
  )
626
628
  @click.option(
@@ -998,8 +1000,9 @@ def create(
998
1000
  email_overlap = set(owner) & set(researcher)
999
1001
  if email_overlap:
1000
1002
  LOG.info(
1001
- f"The email(s) {email_overlap} specified as both owner and researcher! "
1002
- "Please specify a unique role for each email."
1003
+ "The email(s) %s specified as both owner and researcher! "
1004
+ "Please specify a unique role for each email.",
1005
+ email_overlap,
1003
1006
  )
1004
1007
  sys.exit(1)
1005
1008
  if owner:
@@ -1686,7 +1689,8 @@ def get_data(
1686
1689
  sys.exit(1)
1687
1690
  elif not get_all and not (source or source_path_file):
1688
1691
  LOG.error(
1689
- "Specify either '--source' or '--source-path-file' to download specific directories/files, or '--get-all' to download all project contents."
1692
+ "Specify either '--source' or '--source-path-file' to download specific directories/files, "
1693
+ "or '--get-all' to download all project contents."
1690
1694
  )
1691
1695
  sys.exit(1)
1692
1696
 
@@ -1725,7 +1729,7 @@ def get_data(
1725
1729
 
1726
1730
  # Schedule the first num_threads futures for upload
1727
1731
  for file in itertools.islice(iterator, num_threads):
1728
- LOG.debug(f"Starting: {rich.markup.escape(str(file))}")
1732
+ LOG.debug("Starting: %s", rich.markup.escape(str(file)))
1729
1733
  # Execute download
1730
1734
  download_threads[
1731
1735
  texec.submit(getter.download_and_verify, file=file, progress=progress)
@@ -1741,19 +1745,21 @@ def get_data(
1741
1745
 
1742
1746
  for dfut in ddone:
1743
1747
  downloaded_file = download_threads.pop(dfut)
1744
- LOG.debug(
1745
- f"Future done: {rich.markup.escape(str(downloaded_file))}",
1746
- )
1748
+ LOG.debug("Future done: %s", rich.markup.escape(str(downloaded_file)))
1747
1749
 
1748
1750
  # Get result
1749
1751
  try:
1750
1752
  file_downloaded = dfut.result()
1751
1753
  LOG.debug(
1752
- f"Download of {rich.markup.escape(str(downloaded_file))} successful: {file_downloaded}"
1754
+ "Download of %s successful: %s",
1755
+ rich.markup.escape(str(downloaded_file)),
1756
+ file_downloaded,
1753
1757
  )
1754
1758
  except concurrent.futures.BrokenExecutor as err:
1755
1759
  LOG.critical(
1756
- f"Download of file {rich.markup.escape(str(downloaded_file))} failed! Error: {err}"
1760
+ "Download of file %s failed! Error: %s",
1761
+ rich.markup.escape(str(downloaded_file)),
1762
+ err,
1757
1763
  )
1758
1764
  continue
1759
1765
 
@@ -1762,7 +1768,7 @@ def get_data(
1762
1768
 
1763
1769
  # Schedule the next set of futures for download
1764
1770
  for next_file in itertools.islice(iterator, new_tasks):
1765
- LOG.debug(f"Starting: {rich.markup.escape(str(next_file))}")
1771
+ LOG.debug("Starting: %s", rich.markup.escape(str(next_file)))
1766
1772
  # Execute download
1767
1773
  download_threads[
1768
1774
  texec.submit(
@@ -1867,7 +1873,7 @@ def rm_data(click_ctx, project, file, folder, rm_all):
1867
1873
  # Warn if trying to remove all contents
1868
1874
  if rm_all:
1869
1875
  if no_prompt:
1870
- LOG.warning(f"Deleting all files within project '{project}'")
1876
+ LOG.warning("Deleting all files within project '%s'", project)
1871
1877
  else:
1872
1878
  if not rich.prompt.Confirm.ask(
1873
1879
  f"Are you sure you want to delete all files within project '{project}'?"
@@ -2081,3 +2087,46 @@ def set_maintenance_mode(click_ctx, setting):
2081
2087
  ) as err:
2082
2088
  LOG.error(err)
2083
2089
  sys.exit(1)
2090
+
2091
+
2092
+ # Stats
2093
+
2094
+
2095
+ @dds_main.command(name="stats", no_args_is_help=False)
2096
+ @click.argument(
2097
+ "stat_type", nargs=1, type=click.Choice(["active", "all", "size"], case_sensitive=True)
2098
+ )
2099
+ @click.pass_obj
2100
+ def get_stats(click_ctx, stat_type):
2101
+ """Get statistics in the DDS."""
2102
+ try:
2103
+ # Num projects
2104
+ with dds_cli.data_lister.DataLister(
2105
+ show_usage=True,
2106
+ no_prompt=click_ctx.get("NO_PROMPT", False),
2107
+ json=True,
2108
+ token_path=click_ctx.get("TOKEN_PATH"),
2109
+ ) as lister:
2110
+ # Get projects, only active by default
2111
+ projects: typing.List = lister.list_projects(show_all=stat_type == "all")
2112
+
2113
+ if stat_type == "size":
2114
+ # Calculate total amount of saved data in active projects
2115
+ title_bold_part: str = "Bytes"
2116
+ title_rest: str = "currently stored in DDS"
2117
+ value: int = sum(x["Size"] for x in projects)
2118
+ else:
2119
+ # Get number of projects
2120
+ title_bold_part: str = "Active" if stat_type == "active" else "Total"
2121
+ title_rest: str = "projects"
2122
+ value: int = len(projects)
2123
+
2124
+ LOG.info("[bold]%s[/bold] %s: %s", title_bold_part, title_rest, value)
2125
+ except (
2126
+ dds_cli.exceptions.APIError,
2127
+ dds_cli.exceptions.AuthenticationError,
2128
+ dds_cli.exceptions.ApiResponseError,
2129
+ dds_cli.exceptions.ApiRequestError,
2130
+ ) as err:
2131
+ LOG.error(err)
2132
+ sys.exit(1)
@@ -5,12 +5,10 @@
5
5
  ###################################################################################################
6
6
 
7
7
  # Standard library
8
- from email import header
9
8
  import logging
10
9
 
11
10
  # Installed
12
11
  import rich.markup
13
- from rich.table import Table
14
12
 
15
13
  # Own modules
16
14
  import dds_cli
@@ -139,11 +137,16 @@ class AccountManager(dds_cli.base.DDSBaseClass):
139
137
  info = response.get("info")
140
138
  if info:
141
139
  LOG.info(
142
- f"Username: {info['username']} \
143
- \nRole: {info['role']} \
144
- \nName: {info['name']} \
145
- \nPrimary Email: {info['email_primary']} \
146
- \nAssociated Emails: {', '.join(str(x) for x in info['emails_all'])}"
140
+ "Username: %s \n"
141
+ "Role: %s \n"
142
+ "Name: %s \n"
143
+ "Primary Email: %s \n"
144
+ "Associated Emails: %s \n",
145
+ info["username"],
146
+ info["role"],
147
+ info["name"],
148
+ info["email_primary"],
149
+ ", ".join(str(x) for x in info["emails_all"]),
147
150
  )
148
151
 
149
152
  def user_activation(self, email, action):
@@ -173,7 +176,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
173
176
  )
174
177
 
175
178
  if project_errors:
176
- LOG.warning(f"Could not fix user '{email}' access to the following projects:")
179
+ LOG.warning("Could not fix user '%s' access to the following projects:", email)
177
180
  msg = project_errors
178
181
  else:
179
182
  msg = response_json.get(
@@ -197,7 +200,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
197
200
  )
198
201
 
199
202
  if response.get("empty"):
200
- LOG.info(f"There are no Unit Admins or Unit Personnel connected to unit '{unit}'")
203
+ LOG.info("There are no Unit Admins or Unit Personnel connected to unit '%s'", unit)
201
204
  return
202
205
 
203
206
  users, keys = dds_cli.utils.get_required_in_response(
@@ -226,7 +229,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
226
229
  # Print out table
227
230
  dds_cli.utils.print_or_page(item=table)
228
231
 
229
- def list_invites(self, unit: str = None, invites: bool = None) -> None:
232
+ def list_invites(self, invites: bool = None) -> None:
230
233
  """List all unit users within a specific unit."""
231
234
  response, _ = dds_cli.utils.perform_request(
232
235
  endpoint=dds_cli.DDSEndpoint.LIST_INVITED_USERS,
@@ -239,7 +242,7 @@ class AccountManager(dds_cli.base.DDSBaseClass):
239
242
  invites = response.get("invites")
240
243
 
241
244
  if not invites:
242
- LOG.info(f"There are no current invites")
245
+ LOG.info("There are no current invites")
243
246
  return
244
247
 
245
248
  table = dds_cli.utils.create_table(
@@ -269,5 +272,5 @@ class AccountManager(dds_cli.base.DDSBaseClass):
269
272
  )
270
273
 
271
274
  LOG.info(
272
- f"Account exists: {'[blue][bold]Yes[/bold][/blue]' if exists else '[red][bold]No[/bold][/red]'}"
275
+ "Account exists: [bold]%s[/bold]", "[blue]Yes[/blue]" if exists else "[red]No[/red]"
273
276
  )
@@ -4,7 +4,7 @@ import logging
4
4
  import getpass
5
5
 
6
6
  # Installed
7
- import rich
7
+ from rich.prompt import Prompt
8
8
 
9
9
  # Own modules
10
10
  import dds_cli
@@ -48,6 +48,7 @@ class Auth(base.DDSBaseClass):
48
48
  )
49
49
 
50
50
  def check(self):
51
+ """Check if token exists and return info."""
51
52
  token_file = user.TokenFile(token_path=self.token_path)
52
53
  if token_file.file_exists():
53
54
  token = token_file.read_token()
@@ -55,10 +56,12 @@ class Auth(base.DDSBaseClass):
55
56
  token_file.token_report(token=token)
56
57
  else:
57
58
  LOG.info(
58
- "[red]No saved token found, or token has expired. Authenticate yourself with `dds auth login` to use this functionality![/red]"
59
+ "[red]No saved token found, or token has expired. "
60
+ "Authenticate yourself with `dds auth login` to use this functionality![/red]"
59
61
  )
60
62
 
61
63
  def logout(self):
64
+ """Logout user by removing authenticated token."""
62
65
  token_file = user.TokenFile(token_path=self.token_path)
63
66
  if token_file.file_exists():
64
67
  token_file.delete_token()
@@ -67,6 +70,7 @@ class Auth(base.DDSBaseClass):
67
70
  LOG.info("[green]Already logged out![/green]")
68
71
 
69
72
  def twofactor(self, auth_method: str = None):
73
+ """Perform 2FA for user."""
70
74
  if auth_method == "totp":
71
75
  response_json, _ = dds_cli.utils.perform_request(
72
76
  endpoint=dds_cli.DDSEndpoint.USER_ACTIVATE_TOTP,
@@ -78,7 +82,7 @@ class Auth(base.DDSBaseClass):
78
82
  LOG.info(
79
83
  "Activating authentication via email, please (re-)enter your username and password:"
80
84
  )
81
- username: str = rich.prompt.Prompt.ask("DDS username")
85
+ username: str = Prompt.ask("DDS username")
82
86
  password: str = getpass.getpass(prompt="DDS password: ")
83
87
 
84
88
  if password == "":
@@ -95,6 +99,7 @@ class Auth(base.DDSBaseClass):
95
99
  LOG.info(response_json.get("message"))
96
100
 
97
101
  def deactivate(self, username: str = None):
102
+ """Deactivate TOTP for user."""
98
103
  response_json, _ = dds_cli.utils.perform_request(
99
104
  endpoint=dds_cli.DDSEndpoint.TOTP_DEACTIVATE,
100
105
  headers=self.token,
@@ -6,14 +6,10 @@
6
6
 
7
7
  # Standard library
8
8
  import logging
9
- import os
10
9
  import pathlib
11
10
  import typing
12
11
 
13
12
  # Installed
14
- import http
15
- import time
16
- import simplejson
17
13
  from rich.progress import Progress, SpinnerColumn
18
14
 
19
15
  # Own modules
@@ -71,7 +67,7 @@ class DDSBaseClass:
71
67
  # Get attempted operation e.g. put/ls/rm/get
72
68
  if self.method not in DDS_METHODS:
73
69
  raise exceptions.InvalidMethodError(attempted_method=self.method)
74
- LOG.debug(f"Attempted operation: {self.method}")
70
+ LOG.debug("Attempted operation: %s", self.method)
75
71
 
76
72
  # Use user defined destination if any specified
77
73
  if self.method in DDS_DIR_REQUIRED_METHODS:
@@ -114,14 +110,14 @@ class DDSBaseClass:
114
110
 
115
111
  self.keys = self.__get_project_keys()
116
112
 
117
- self.status = dict()
113
+ self.status: typing.Dict = {}
118
114
  self.filehandler = None
119
115
 
120
116
  def __enter__(self):
121
117
  """Return self when using context manager."""
122
118
  return self
123
119
 
124
- def __exit__(self, exc_type, exc_value, tb, max_fileerrs: int = 40):
120
+ def __exit__(self, exception_type, exception_value, traceback, max_fileerrs: int = 40):
125
121
  """Finish and print out delivery summary.
126
122
 
127
123
  This is not entered if there's an error during __init__.
@@ -131,8 +127,8 @@ class DDSBaseClass:
131
127
  self.__printout_delivery_summary()
132
128
 
133
129
  # Exception is not handled
134
- if exc_type is not None:
135
- LOG.debug(f"Exception: {exc_type} with value {exc_value}")
130
+ if exception_type is not None:
131
+ LOG.debug("Exception: %s with value %s", exception_type, exception_value)
136
132
  return False
137
133
 
138
134
  return True
@@ -188,7 +184,7 @@ class DDSBaseClass:
188
184
  def __printout_delivery_summary(self):
189
185
  """Print out the delivery summary if any files were cancelled."""
190
186
  if self.stop_doing:
191
- LOG.info(f"{'Upload' if self.method == 'put' else 'Download'} cancelled.\n")
187
+ LOG.info("%s cancelled.\n", "Upload" if self.method == "put" else "Download")
192
188
  return
193
189
 
194
190
  # TODO: Look into a better summary print out - old deleted for now
@@ -210,21 +206,23 @@ class DDSBaseClass:
210
206
  f"Please verify that the following error log has been generated: {self.failed_delivery_log}\n"
211
207
  "[red][bold]Do not[/bold][/red] delete this file; The Data Centre may need it during DDS support."
212
208
  )
213
- else:
214
- # TODO: --destination should be able to >at least< overwrite the files in the
215
- # previously created download location.
216
- raise exceptions.DownloadError(
217
- "Errors occurred during download.\n"
218
- "If you wish to retry the download, re-run the `dds data get` command again, "
219
- "specifying the same options as you did now. A new directory will "
220
- "automatically be created and all files will be downloaded again.\n\n"
221
- f"See {self.failed_delivery_log} for more information."
222
- )
223
209
 
224
- elif nr_uploaded:
210
+ # TODO: --destination should be able to >at least< overwrite the files in the
211
+ # previously created download location.
212
+ raise exceptions.DownloadError(
213
+ "Errors occurred during download.\n"
214
+ "If you wish to retry the download, re-run the `dds data get` command again, "
215
+ "specifying the same options as you did now. A new directory will "
216
+ "automatically be created and all files will be downloaded again.\n\n"
217
+ f"See {self.failed_delivery_log} for more information."
218
+ )
219
+
220
+ if nr_uploaded:
225
221
  # Raise exception in order to give exit code 1
226
222
  LOG.warning(
227
- f"{nr_uploaded} files have already been uploaded to this project.\nUpload [bold]partially[/bold] completed!\n"
223
+ "%s files have already been uploaded to this project.\n"
224
+ "Upload [bold]partially[/bold] completed!\n",
225
+ nr_uploaded,
228
226
  )
229
227
 
230
228
  else:
@@ -234,7 +232,7 @@ class DDSBaseClass:
234
232
  )
235
233
 
236
234
  if self.method == "get" and len(self.filehandler.data) > len(any_failed):
237
- LOG.info(f"Any downloaded files are located: {self.filehandler.local_destination}.")
235
+ LOG.info("Any downloaded files are located at: %s.", self.filehandler.local_destination)
238
236
 
239
237
  def __collect_all_failed(self, sort: bool = True) -> list:
240
238
  """Put cancelled files from status in to failed dict and sort the output."""
@@ -52,13 +52,13 @@ def verify_proceed(func):
52
52
 
53
53
  # Mark as started
54
54
  self.status[file]["started"] = True
55
- LOG.debug(f"File {escape(str(file))} started {func.__name__}")
55
+ LOG.debug("File '%s' started: %s", escape(str(file)), func.__name__)
56
56
 
57
57
  # Run function
58
58
  ok_to_proceed, message = func(self, file=file, *args, **kwargs)
59
59
  # Cancel file(s) if something failed
60
60
  if not ok_to_proceed:
61
- LOG.warning(f"{func.__name__} failed: {message}")
61
+ LOG.warning("%s failed: %s", func.__name__, message)
62
62
  self.status[file].update({"cancel": True, "message": message})
63
63
  if self.status[file].get("failed_op") is None:
64
64
  self.status[file]["failed_op"] = "crypto"
@@ -91,13 +91,17 @@ def update_status(func):
91
91
  def wrapped(self, file, *args, **kwargs):
92
92
  # TODO (ina): add processing?
93
93
  if func.__name__ not in ["put", "add_file_db", "get", "update_db"]:
94
- raise Exception(f"The function {func.__name__} cannot be used with this decorator.")
94
+ raise dds_cli.exceptions.DDSCLIException(
95
+ f"The function {func.__name__} cannot be used with this decorator."
96
+ )
95
97
  if func.__name__ not in self.status[file]:
96
- raise Exception(f"No status found for function {func.__name__}.")
98
+ raise dds_cli.exceptions.DDSCLIException(
99
+ f"No status found for function {func.__name__}."
100
+ )
97
101
 
98
102
  # Update status to started
99
103
  self.status[file][func.__name__].update({"started": True})
100
- LOG.debug(f"File {escape(str(file))} status updated to {func.__name__}: started")
104
+ LOG.debug("File '%s' status updated to %s: started", escape(str(file)), func.__name__)
101
105
 
102
106
  # Run function
103
107
  ok_to_continue, message, *_ = func(self, file=file, *args, **kwargs)
@@ -107,12 +111,12 @@ def update_status(func):
107
111
  # Save info about which operation failed
108
112
 
109
113
  self.status[file]["failed_op"] = func.__name__
110
- LOG.warning(f"{func.__name__} failed: {message}")
114
+ LOG.warning("%s failed: %s", func.__name__, message)
111
115
 
112
116
  else:
113
117
  # Update status to done
114
118
  self.status[file][func.__name__].update({"done": True})
115
- LOG.debug(f"File {escape(str(file))} status updated to {func.__name__}: done")
119
+ LOG.debug("File %s status updated to %s: done", escape(str(file)), func.__name__)
116
120
 
117
121
  return ok_to_continue, message
118
122
 
@@ -137,7 +141,7 @@ def subpath_required(func):
137
141
  except OSError as err:
138
142
  return False, str(err)
139
143
 
140
- LOG.debug(f"New directory created: {full_subpath}")
144
+ LOG.debug("New directory created: '%s'", full_subpath)
141
145
 
142
146
  return func(self, file=file, *args, **kwargs)
143
147
 
@@ -149,8 +153,6 @@ def removal_spinner(func):
149
153
 
150
154
  @functools.wraps(func)
151
155
  def create_and_remove_task(self, *args, **kwargs):
152
- message = ""
153
-
154
156
  with Progress(
155
157
  "[bold]{task.description}",
156
158
  SpinnerColumn(spinner_name="dots12", style="white"),
@@ -186,7 +188,7 @@ def removal_spinner(func):
186
188
  dds_cli.utils.console.print(self.failed_table)
187
189
  else:
188
190
  dds_cli.utils.console.print(self.failed_table)
189
- LOG.warning(f"Finished {description_lc} with errors, see table above")
191
+ LOG.warning("Finished %s with errors, see table above", description_lc)
190
192
  elif self.failed_files is not None:
191
193
  self.failed_files["result"] = f"Finished {description_lc} with errors"
192
194
  dds_cli.utils.console.print(self.failed_files)
@@ -10,7 +10,6 @@ import pathlib
10
10
 
11
11
  # Installed
12
12
  import requests
13
- import simplejson
14
13
  from rich.markup import escape
15
14
  from rich.progress import Progress, SpinnerColumn
16
15
 
@@ -101,7 +100,7 @@ class DataGetter(base.DDSBaseClass):
101
100
 
102
101
  if not self.filehandler.data:
103
102
  if self.temporary_directory and self.temporary_directory.is_dir():
104
- LOG.debug(f"Deleting temporary folder {self.temporary_directory}.")
103
+ LOG.debug("Deleting temporary folder '%s'.", self.temporary_directory)
105
104
  dds_cli.utils.delete_folder(self.temporary_directory)
106
105
  raise dds_cli.exceptions.DownloadError("No files to download.")
107
106
 
@@ -134,13 +133,13 @@ class DataGetter(base.DDSBaseClass):
134
133
  total=file_info["size_original"],
135
134
  )
136
135
 
137
- LOG.debug(f"File {escape(str(file))} downloaded: {file_downloaded}")
136
+ LOG.debug("File '%s' downloaded: %s", escape(str(file)), file_downloaded)
138
137
 
139
138
  if file_downloaded:
140
139
  db_updated, message = self.update_db(file=file)
141
- LOG.debug(f"Database updated: {db_updated}")
140
+ LOG.debug("Database updated: %s", db_updated)
142
141
 
143
- LOG.debug(f"Beginning decryption of file {escape(str(file))}...")
142
+ LOG.debug("Beginning decryption of file '%s'...", escape(str(file)))
144
143
  file_saved = False
145
144
  with fe.Decryptor(
146
145
  project_keys=self.keys,
@@ -160,7 +159,7 @@ class DataGetter(base.DDSBaseClass):
160
159
  outfile=file,
161
160
  )
162
161
 
163
- LOG.debug(f"file saved? {file_saved}")
162
+ LOG.debug("File saved? %s", file_saved)
164
163
  if file_saved:
165
164
  # TODO (ina): decide on checksum verification method --
166
165
  # this checks original, the other is generated from compressed
@@ -184,6 +183,7 @@ class DataGetter(base.DDSBaseClass):
184
183
  file_remote = self.filehandler.data[file]["url"]
185
184
 
186
185
  try:
186
+ # TODO: Set timeout? (pylint)
187
187
  with requests.get(file_remote, stream=True) as req:
188
188
  req.raise_for_status()
189
189
  with file_local.open(mode="wb") as new_file:
@@ -214,7 +214,6 @@ class DataGetter(base.DDSBaseClass):
214
214
  def update_db(self, file):
215
215
  """Update file info in db."""
216
216
  updated_in_db = False
217
- error = ""
218
217
 
219
218
  # Get file info
220
219
  fileinfo = self.filehandler.data[file]