rucio-clients 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (86) hide show
  1. rucio/alembicrevision.py +1 -1
  2. rucio/cli/__init__.py +14 -0
  3. rucio/cli/account.py +216 -0
  4. rucio/cli/bin_legacy/__init__.py +13 -0
  5. rucio_clients-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
  6. rucio_clients-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
  7. rucio/cli/command.py +272 -0
  8. rucio/cli/config.py +72 -0
  9. rucio/cli/did.py +191 -0
  10. rucio/cli/download.py +128 -0
  11. rucio/cli/lifetime_exception.py +33 -0
  12. rucio/cli/replica.py +162 -0
  13. rucio/cli/rse.py +293 -0
  14. rucio/cli/rule.py +158 -0
  15. rucio/cli/scope.py +40 -0
  16. rucio/cli/subscription.py +73 -0
  17. rucio/cli/upload.py +60 -0
  18. rucio/cli/utils.py +226 -0
  19. rucio/client/accountclient.py +0 -1
  20. rucio/client/baseclient.py +33 -24
  21. rucio/client/client.py +45 -1
  22. rucio/client/didclient.py +5 -3
  23. rucio/client/downloadclient.py +6 -8
  24. rucio/client/replicaclient.py +0 -2
  25. rucio/client/richclient.py +317 -0
  26. rucio/client/rseclient.py +4 -4
  27. rucio/client/uploadclient.py +26 -12
  28. rucio/common/bittorrent.py +234 -0
  29. rucio/common/cache.py +66 -29
  30. rucio/common/checksum.py +168 -0
  31. rucio/common/client.py +122 -0
  32. rucio/common/config.py +22 -35
  33. rucio/common/constants.py +61 -3
  34. rucio/common/didtype.py +72 -24
  35. rucio/common/exception.py +65 -8
  36. rucio/common/extra.py +5 -10
  37. rucio/common/logging.py +13 -13
  38. rucio/common/pcache.py +8 -7
  39. rucio/common/plugins.py +59 -27
  40. rucio/common/policy.py +12 -3
  41. rucio/common/schema/__init__.py +84 -34
  42. rucio/common/schema/generic.py +0 -17
  43. rucio/common/schema/generic_multi_vo.py +0 -17
  44. rucio/common/stomp_utils.py +383 -119
  45. rucio/common/test_rucio_server.py +12 -6
  46. rucio/common/types.py +132 -52
  47. rucio/common/utils.py +93 -643
  48. rucio/rse/__init__.py +3 -3
  49. rucio/rse/protocols/bittorrent.py +11 -1
  50. rucio/rse/protocols/cache.py +0 -11
  51. rucio/rse/protocols/dummy.py +0 -11
  52. rucio/rse/protocols/gfal.py +14 -9
  53. rucio/rse/protocols/globus.py +1 -1
  54. rucio/rse/protocols/http_cache.py +1 -1
  55. rucio/rse/protocols/posix.py +2 -2
  56. rucio/rse/protocols/protocol.py +84 -317
  57. rucio/rse/protocols/rclone.py +2 -1
  58. rucio/rse/protocols/rfio.py +10 -1
  59. rucio/rse/protocols/ssh.py +2 -1
  60. rucio/rse/protocols/storm.py +2 -13
  61. rucio/rse/protocols/webdav.py +74 -30
  62. rucio/rse/protocols/xrootd.py +2 -1
  63. rucio/rse/rsemanager.py +170 -53
  64. rucio/rse/translation.py +260 -0
  65. rucio/vcsversion.py +4 -4
  66. rucio/version.py +7 -0
  67. {rucio_clients-35.7.0.data → rucio_clients-37.0.0rc2.data}/data/etc/rucio.cfg.atlas.client.template +3 -2
  68. {rucio_clients-35.7.0.data → rucio_clients-37.0.0rc2.data}/data/etc/rucio.cfg.template +3 -19
  69. {rucio_clients-35.7.0.data → rucio_clients-37.0.0rc2.data}/data/requirements.client.txt +11 -7
  70. rucio_clients-37.0.0rc2.data/scripts/rucio +133 -0
  71. rucio_clients-37.0.0rc2.data/scripts/rucio-admin +97 -0
  72. {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0rc2.dist-info}/METADATA +18 -14
  73. rucio_clients-37.0.0rc2.dist-info/RECORD +104 -0
  74. {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0rc2.dist-info}/licenses/AUTHORS.rst +3 -0
  75. rucio/common/schema/atlas.py +0 -413
  76. rucio/common/schema/belleii.py +0 -408
  77. rucio/common/schema/domatpc.py +0 -401
  78. rucio/common/schema/escape.py +0 -426
  79. rucio/common/schema/icecube.py +0 -406
  80. rucio/rse/protocols/gsiftp.py +0 -92
  81. rucio_clients-35.7.0.dist-info/RECORD +0 -88
  82. {rucio_clients-35.7.0.data → rucio_clients-37.0.0rc2.data}/data/etc/rse-accounts.cfg.template +0 -0
  83. {rucio_clients-35.7.0.data → rucio_clients-37.0.0rc2.data}/data/rucio_client/merge_rucio_configs.py +0 -0
  84. {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0rc2.dist-info}/WHEEL +0 -0
  85. {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0rc2.dist-info}/licenses/LICENSE +0 -0
  86. {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0rc2.dist-info}/top_level.txt +0 -0
rucio/common/config.py CHANGED
@@ -17,18 +17,23 @@
17
17
  import configparser
18
18
  import json
19
19
  import os
20
- from collections.abc import Callable
20
+ from functools import cache
21
21
  from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload
22
22
 
23
23
  from rucio.common import exception
24
- from rucio.common.exception import ConfigNotFound, DatabaseException
24
+ from rucio.common.exception import ConfigLoadingError, ConfigNotFound, DatabaseException
25
25
 
26
26
  _T = TypeVar('_T')
27
27
  _U = TypeVar('_U')
28
28
 
29
29
  if TYPE_CHECKING:
30
+ from collections.abc import Callable
31
+
30
32
  from sqlalchemy.orm import Session
31
33
 
34
+ LEGACY_SECTION_NAME = {}
35
+ LEGACY_OPTION_NAME = {}
36
+
32
37
 
33
38
  def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
34
39
  if value.lower() in ['true', 'yes', 'on']:
@@ -110,7 +115,7 @@ def config_get(
110
115
  session: "Optional[Session]" = ...,
111
116
  use_cache: bool = ...,
112
117
  expiration_time: int = ...,
113
- convert_type_fnc: Callable[[str], _T],
118
+ convert_type_fnc: 'Callable[[str], _T]',
114
119
  ) -> _T:
115
120
  ...
116
121
 
@@ -126,7 +131,7 @@ def config_get(
126
131
  session: "Optional[Session]" = ...,
127
132
  use_cache: bool = ...,
128
133
  expiration_time: int = ...,
129
- convert_type_fnc: Callable[[str], _U],
134
+ convert_type_fnc: 'Callable[[str], _U]',
130
135
  ) -> Union[_T, _U]:
131
136
  ...
132
137
 
@@ -143,7 +148,7 @@ def config_get(
143
148
  session: "Optional[Session]" = ...,
144
149
  use_cache: bool = ...,
145
150
  expiration_time: int = ...,
146
- convert_type_fnc: Callable[[str], _U],
151
+ convert_type_fnc: 'Callable[[str], _U]',
147
152
  ) -> Union[_T, _U]:
148
153
  ...
149
154
 
@@ -158,7 +163,7 @@ def config_get(
158
163
  session: "Optional[Session]" = None,
159
164
  use_cache: bool = True,
160
165
  expiration_time: int = 900,
161
- convert_type_fnc: Callable[[str], _T] = lambda x: x,
166
+ convert_type_fnc: 'Callable[[str], _T]' = lambda x: x,
162
167
  ) -> Union[_T, _U]:
163
168
  """
164
169
  Return the string value for a given option in a section
@@ -196,7 +201,7 @@ def config_get(
196
201
  except ConfigNotFound:
197
202
  pass
198
203
 
199
- from rucio.common.utils import is_client
204
+ from rucio.common.client import is_client
200
205
  client_mode = is_client()
201
206
 
202
207
  if not client_mode and check_config_table:
@@ -223,8 +228,6 @@ def get_legacy_config(section: str, option: str):
223
228
  :param option: The option of the new config.
224
229
  :returns: The string value of the legacy option if one is found, None otherwise.
225
230
  """
226
- LEGACY_SECTION_NAME = {}
227
- LEGACY_OPTION_NAME = {}
228
231
 
229
232
  section = LEGACY_SECTION_NAME.get(section, section)
230
233
  option = LEGACY_OPTION_NAME.get(option, option)
@@ -544,12 +547,12 @@ def config_get_list(
544
547
  section: str,
545
548
  option: str,
546
549
  *,
547
- default: list[str] = ...,
550
+ default: _T = ...,
548
551
  check_config_table: bool = ...,
549
552
  session: "Optional[Session]" = ...,
550
553
  use_cache: bool = ...,
551
554
  expiration_time: int = ...,
552
- ) -> list[str]:
555
+ ) -> Union[list[str], _T]:
553
556
  ...
554
557
 
555
558
 
@@ -637,7 +640,7 @@ def __config_get_table(
637
640
  session: "Optional[Session]" = None,
638
641
  use_cache: bool = True,
639
642
  expiration_time: int = 900,
640
- convert_type_fnc: Optional[Callable[[str], _T]],
643
+ convert_type_fnc: Optional['Callable[[str], _T]'],
641
644
  ) -> _T:
642
645
  """
643
646
  Search for a section-option configuration parameter in the configuration table
@@ -655,7 +658,6 @@ def __config_get_table(
655
658
  :raises ConfigNotFound
656
659
  :raises DatabaseException
657
660
  """
658
- global __CONFIG
659
661
  try:
660
662
  from rucio.core.config import get as core_config_get
661
663
  return core_config_get(section, option, default=default, session=session, use_cache=use_cache,
@@ -664,7 +666,7 @@ def __config_get_table(
664
666
  if raise_exception and default is None:
665
667
  raise err
666
668
  if clean_cached:
667
- __CONFIG = None
669
+ clean_cached_config()
668
670
  return default
669
671
 
670
672
 
@@ -714,14 +716,8 @@ def get_config_dirs() -> list[str]:
714
716
  """
715
717
  configdirs = []
716
718
 
717
- if 'RUCIO_HOME' in os.environ:
718
- configdirs.append('%s/etc/' % os.environ['RUCIO_HOME'])
719
-
720
- if 'VIRTUAL_ENV' in os.environ:
721
- configdirs.append('%s/etc/' % os.environ['VIRTUAL_ENV'])
722
-
723
- if 'CONDA_PREFIX' in os.environ:
724
- configdirs.append('%s/etc/' % os.environ['CONDA_PREFIX'])
719
+ env_vars = ['RUCIO_HOME', 'VIRTUAL_ENV', 'CONDA_PREFIX']
720
+ configdirs.extend([os.path.join(os.environ[var], 'etc', '') for var in env_vars if var in os.environ])
725
721
 
726
722
  configdirs.append('/opt/rucio/etc/')
727
723
 
@@ -758,21 +754,15 @@ def get_rse_credentials(path_to_credentials_file: Optional[Union[str, os.PathLik
758
754
  return credentials
759
755
 
760
756
 
761
- __CONFIG = None
762
-
763
-
757
+ @cache
764
758
  def get_config() -> configparser.ConfigParser:
765
759
  """Factory function for the configuration class. Returns the ConfigParser instance."""
766
- global __CONFIG
767
- if __CONFIG is None:
768
- __CONFIG = Config()
769
- return __CONFIG.parser
760
+ return Config().parser
770
761
 
771
762
 
772
763
  def clean_cached_config() -> None:
773
764
  """Deletes the cached config singleton instance."""
774
- global __CONFIG
775
- __CONFIG = None
765
+ get_config.cache_clear()
776
766
 
777
767
 
778
768
  class Config:
@@ -795,7 +785,4 @@ class Config:
795
785
  '\n\t' + '\n\t'.join(configs))
796
786
 
797
787
  if not self.parser.read(self.configfile) == [self.configfile]:
798
- raise ConfigNotFound(
799
- 'Could not load Rucio configuration file. '
800
- 'Rucio tried loading the following configuration file:'
801
- '\n\t' + self.configfile)
788
+ raise ConfigLoadingError(self.configfile)
rucio/common/constants.py CHANGED
@@ -48,7 +48,7 @@ if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False
48
48
  SCHEME_MAP['srm'].append('davs')
49
49
  SCHEME_MAP['davs'].append('srm')
50
50
 
51
- SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'closeness', 'custom_table', 'dynamic', 'ranking', 'random']
51
+ SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
52
52
  SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
53
53
 
54
54
  SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
@@ -56,8 +56,11 @@ SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
56
56
 
57
57
  RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']
58
58
 
59
- RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete', 'third_party_copy_read', 'third_party_copy_write']
60
- RSE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
59
+ RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete']
60
+ RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
61
+
62
+ RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal[RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, 'third_party_copy_read', 'third_party_copy_write']
63
+ RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
61
64
 
62
65
  FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
63
66
  'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
@@ -74,12 +77,14 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
74
77
  MAX_MESSAGE_LENGTH = 4000
75
78
 
76
79
 
80
+ @enum.unique
77
81
  class SuspiciousAvailability(enum.Enum):
78
82
  ALL = 0
79
83
  EXIST_COPIES = 1
80
84
  LAST_COPY = 2
81
85
 
82
86
 
87
+ @enum.unique
83
88
  class ReplicaState(enum.Enum):
84
89
  # From rucio.db.sqla.constants, update that file at the same time as this
85
90
  AVAILABLE = 'A'
@@ -157,3 +162,56 @@ class RseAttr:
157
162
  DEFAULT_LIMIT_FILES = 'default_limit_files'
158
163
  QUOTA_APPROVERS = 'quota_approvers'
159
164
  RULE_DELETERS = 'rule_deleters'
165
+
166
+
167
+ # Literal types to allow overloading of functions with RSE attributes in their signature.
168
+ # RSE attributes are encoded via the BooleanString decorator as VARCHAR in the database,
169
+ # but they are used as either bool or string in the code.
170
+ # This is only determined at runtime, so for static type checking
171
+ # we need to manually specify which attrs are string and which are bool.
172
+ # In future, we could refactor RseAttr to avoid code duplication.
173
+ RSE_ATTRS_STR = Literal[
174
+ 'archive_timeout',
175
+ 'associated_sites',
176
+ 'bittorrent_tracker_addr',
177
+ 'country',
178
+ 'decommission',
179
+ 'default_account_limit_bytes',
180
+ 'fts',
181
+ 'globus_endpoint_id',
182
+ 'lfn2pfn_algorithm',
183
+ 'maximum_pin_lifetime',
184
+ 'multihop_tombstone_delay',
185
+ 'naming_convention',
186
+ 'oidc_base_path',
187
+ 'oidc_support'
188
+ 'physgroup',
189
+ 'qbittorrent_management_address'
190
+ 'rule_approvers',
191
+ 's3_url_style',
192
+ 'simulate_multirange',
193
+ 'site',
194
+ 'source_for_total_space',
195
+ 'source_for_used_space',
196
+ 'staging_buffer',
197
+ 'tombstone_delay',
198
+ 'type'
199
+ ]
200
+
201
+ RSE_ATTRS_BOOL = Literal[
202
+ 'auto_approve_bytes',
203
+ 'auto_approve_files',
204
+ 'block_manual_approval',
205
+ 'greedyDeletion',
206
+ 'is_object_store',
207
+ 'restricted_read',
208
+ 'restricted_write',
209
+ 'skip_upload_stat',
210
+ 'staging_required',
211
+ 'strict_copy',
212
+ 'use_ipv4',
213
+ 'verify_checksum'
214
+ ]
215
+
216
+ SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
217
+ SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))
rucio/common/didtype.py CHANGED
@@ -16,7 +16,7 @@
16
16
  DID type to represent a did and to simplify operations on it
17
17
  """
18
18
 
19
- from typing import Union
19
+ from typing import Any, Union
20
20
 
21
21
  from rucio.common.exception import DIDError
22
22
 
@@ -58,6 +58,22 @@ class DID:
58
58
  self.scope: str = ''
59
59
  self.name: str = ''
60
60
 
61
+ did = self._parse_did_from_args(*args, **kwargs)
62
+
63
+ self._construct_did(did)
64
+
65
+ if not self.is_valid_format():
66
+ raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
67
+
68
+ def _parse_did_from_args(
69
+ self,
70
+ *args,
71
+ **kwargs
72
+ ) -> Union["DID", str, tuple[str, str], list[str], dict[str, str]]:
73
+ """
74
+ Parse the DID object from the given arguments
75
+ :return: DID object
76
+ """
61
77
  num_args = len(args)
62
78
  num_kwargs = len(kwargs)
63
79
  if (num_args + num_kwargs) > 2:
@@ -77,46 +93,78 @@ class DID:
77
93
  else:
78
94
  raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
79
95
  else:
80
- raise DIDError('First argument of constructor is expected to be string type'
96
+ raise DIDError('First argument of constructor is expected to be string type '
81
97
  'when keyword argument is given. Given type: {}'.format(type(did)))
82
98
  elif num_args == 0:
83
99
  did = kwargs.get('did', kwargs)
84
100
  else:
85
101
  did = args
102
+ return did
86
103
 
104
+ def _construct_did(self, did: Any) -> None:
105
+ """
106
+ Construct the DID object from the given input.
107
+
108
+ :param did: input to construct the DID object from
109
+ """
87
110
  if isinstance(did, dict):
88
- self.scope = did.get('scope', '')
89
- self.name = did.get('name', '')
90
- if not self.has_scope():
91
- self.update_implicit_scope()
111
+ self._did_from_dict(did)
92
112
  elif isinstance(did, tuple) or isinstance(did, list):
93
- if len(did) != 2:
94
- raise DIDError('Construction from tuple or list requires exactly 2 elements')
95
- self.scope = did[0]
96
- self.name = did[1]
113
+ self._did_from_list_or_tuple(did)
97
114
  elif isinstance(did, str):
98
- did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
99
- if len(did_parts) == 1:
100
- self.name = did
101
- self.update_implicit_scope()
102
- if not self.has_scope():
103
- raise DIDError('Object construction from non-splitable string is ambigious')
104
- else:
105
- self.scope = did_parts[0]
106
- self.name = did_parts[1]
115
+ self._did_from_str(did)
107
116
  elif isinstance(did, DID):
108
- self.scope = did.scope
109
- self.name = did.name
117
+ self._did_from_did_object(did)
110
118
  else:
111
119
  raise DIDError('Cannot build object from: {}'.format(type(did)))
112
120
 
113
121
  if self.name.endswith('/'):
114
122
  self.name = self.name[:-1]
115
123
 
116
- if not self.is_valid_format():
117
- raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
124
+ def _did_from_str(self, did: str) -> None:
125
+ """
126
+ Construct the DID from a string.
127
+ :param did: string containing the DID information
128
+ """
129
+ did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
130
+ if len(did_parts) == 1:
131
+ self.name = did
132
+ self._update_implicit_scope()
133
+ if not self.has_scope():
134
+ raise DIDError('Object construction from non-splitable string is ambigious')
135
+ else:
136
+ self.scope = did_parts[0]
137
+ self.name = did_parts[1]
138
+
139
+ def _did_from_dict(self, did: dict[str, str]) -> None:
140
+ """
141
+ Construct the DID from a dictionary.
142
+ :param did: dictionary optionally containing the keys 'scope' and 'name'
143
+ """
144
+ self.scope = did.get('scope', '')
145
+ self.name = did.get('name', '')
146
+ if not self.has_scope():
147
+ self._update_implicit_scope()
148
+
149
+ def _did_from_list_or_tuple(self, did: Union[list[str], tuple[str, str]]) -> None:
150
+ """
151
+ Construct the DID from a list or tuple.
152
+ :param did: list or tuple with expected length of 2
153
+ """
154
+ if len(did) != 2:
155
+ raise DIDError('Construction from tuple or list requires exactly 2 elements. Number of elements passed: %i' % len(did))
156
+ self.scope = did[0]
157
+ self.name = did[1]
158
+
159
+ def _did_from_did_object(self, did: "DID") -> None:
160
+ """
161
+ Construct the DID from another DID object.
162
+ :param did: DID object
163
+ """
164
+ self.scope = did.scope
165
+ self.name = did.name
118
166
 
119
- def update_implicit_scope(self) -> None:
167
+ def _update_implicit_scope(self) -> None:
120
168
  """
121
169
  This method sets the scope if it is implicitly given in self.name
122
170
  """
rucio/common/exception.py CHANGED
@@ -20,6 +20,8 @@
20
20
 
21
21
  """
22
22
 
23
+ from typing import Optional
24
+
23
25
  from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
24
26
 
25
27
 
@@ -93,11 +95,12 @@ class ClientParameterMismatch(RucioException):
93
95
 
94
96
  class ClientProtocolNotSupported(RucioException):
95
97
  """
96
- RucioException
98
+ Client protocol not supported
97
99
  """
98
- def __init__(self, *args):
100
+
101
+ def __init__(self, host: str, protocol: str, protocols_allowed: Optional[list[str]] = None, *args):
99
102
  super(ClientProtocolNotSupported, self).__init__(*args)
100
- self._message = "Client protocol not supported."
103
+ self._message = f"Client protocol '{protocol}' not supported when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
101
104
  self.error_code = 6
102
105
 
103
106
 
@@ -1068,14 +1071,14 @@ class PolicyPackageVersionError(PolicyPackageBaseException):
1068
1071
  """
1069
1072
  Policy package is not compatible with this version of Rucio.
1070
1073
  """
1071
- def __init__(self, package: str, rucio_version: str, supported_versions: list[str], *args):
1074
+ def __init__(self, package: str, rucio_version: str, supported_versionset: str, *args):
1072
1075
  super(PolicyPackageVersionError, self).__init__(package, *args)
1073
1076
  self.rucio_version = rucio_version
1074
- self.supported_versions = supported_versions
1077
+ self.supported_versionset = supported_versionset
1075
1078
  self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
1076
1079
  self.package,
1077
1080
  self.rucio_version,
1078
- self.supported_versions
1081
+ self.supported_versionset
1079
1082
  )
1080
1083
  self.error_code = 103
1081
1084
 
@@ -1125,8 +1128,8 @@ class TraceValidationSchemaNotFound(RucioException):
1125
1128
  """
1126
1129
  Trace validation schema not found.
1127
1130
  """
1128
- def __init__(self, *args, **kwargs):
1129
- super(TraceValidationSchemaNotFound, self).__init__(*args, **kwargs)
1131
+ def __init__(self, *args):
1132
+ super(TraceValidationSchemaNotFound, self).__init__(*args)
1130
1133
  self._message = 'Trace validation schema not found.'
1131
1134
  self.error_code = 108
1132
1135
 
@@ -1149,3 +1152,57 @@ class UnsupportedMetadataPlugin(RucioException):
1149
1152
  super(UnsupportedMetadataPlugin, self).__init__(*args)
1150
1153
  self._message = "The requested metadata plugin is not enabled on the server."
1151
1154
  self.error_code = 110
1155
+
1156
+
1157
+ class ChecksumCalculationError(RucioException):
1158
+ """
1159
+ An error occurred while calculating the checksum.
1160
+ """
1161
+ def __init__(
1162
+ self,
1163
+ algorithm_name: str,
1164
+ filepath: str,
1165
+ *args,
1166
+ **kwargs
1167
+ ):
1168
+ super(ChecksumCalculationError, self).__init__(*args, **kwargs)
1169
+ self.algorithm_name = algorithm_name
1170
+ self.filepath = filepath
1171
+ self._message = 'An error occurred while calculating the %s checksum of file %s.' % (self.algorithm_name, self.filepath)
1172
+ self.error_code = 111
1173
+
1174
+
1175
+ class ConfigLoadingError(RucioException):
1176
+ """
1177
+ An error occurred while loading the configuration.
1178
+ """
1179
+ def __init__(
1180
+ self,
1181
+ config_file: str,
1182
+ *args,
1183
+ **kwargs
1184
+ ):
1185
+ super(ConfigLoadingError, self).__init__(*args, **kwargs)
1186
+ self._message = 'Could not load Rucio configuration file. Rucio tried loading the following configuration file:\n\t %s' % (config_file)
1187
+ self.error_code = 112
1188
+
1189
+
1190
+ class ClientProtocolNotFound(RucioException):
1191
+ """
1192
+ Missing protocol in client configuration (e.g. no http/https in url).
1193
+ """
1194
+
1195
+ def __init__(self, host: str, protocols_allowed: Optional[list[str]] = None, *args):
1196
+ super(ClientProtocolNotFound, self).__init__(*args)
1197
+ self._message = f"Client protocol missing when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
1198
+ self.error_code = 113
1199
+
1200
+
1201
+ class ConnectionParameterNotFound(RucioException):
1202
+ """
1203
+ Thrown when a required connection parameter is missing.
1204
+ """
1205
+ def __init__(self, param: str, *args):
1206
+ super(ConnectionParameterNotFound, self).__init__(*args)
1207
+ self._message = f"Required connection parameter '{param}' is not provided."
1208
+ self.error_code = 114
rucio/common/extra.py CHANGED
@@ -13,24 +13,19 @@
13
13
  # limitations under the License.
14
14
 
15
15
  import importlib
16
- import warnings
17
- from typing import TYPE_CHECKING
16
+ from typing import TYPE_CHECKING, Optional
18
17
 
19
18
  if TYPE_CHECKING:
20
- from typing import Any
19
+ from collections.abc import Iterable
20
+ from types import ModuleType
21
21
 
22
22
 
23
- def import_extras(module_list: list[str]) -> "dict[str, Any]":
23
+ def import_extras(module_list: 'Iterable[str]') -> dict[str, "Optional[ModuleType]"]:
24
24
  out = dict()
25
25
  for mod in module_list:
26
26
  out[mod] = None
27
27
  try:
28
- with warnings.catch_warnings():
29
- # TODO: remove when https://github.com/paramiko/paramiko/issues/2038 is fixed
30
- warnings.filterwarnings('ignore', 'Blowfish has been deprecated', module='paramiko')
31
- # TODO: deprecated python 2 and 3.6 too ...
32
- warnings.filterwarnings('ignore', 'Python .* is no longer supported', module='paramiko')
33
- out[mod] = importlib.import_module(mod)
28
+ out[mod] = importlib.import_module(mod)
34
29
  except ImportError:
35
30
  pass
36
31
  return out
rucio/common/logging.py CHANGED
@@ -18,13 +18,13 @@ import json
18
18
  import logging
19
19
  import re
20
20
  import sys
21
- from collections.abc import Callable, Iterator, Mapping, Sequence
22
21
  from traceback import format_tb
23
22
  from typing import TYPE_CHECKING, Any, Literal, Optional, Union, get_args
24
23
 
25
24
  from rucio.common.config import config_get, config_get_bool
26
25
 
27
26
  if TYPE_CHECKING:
27
+ from collections.abc import Callable, Iterator, Mapping, Sequence
28
28
  from logging import LogRecord, _SysExcInfoType
29
29
 
30
30
  from _typeshed import OptExcInfo
@@ -76,7 +76,7 @@ def _json_serializable(obj: Any) -> Union[dict[Any, Any], str]:
76
76
  return str(obj)
77
77
 
78
78
 
79
- def _navigate_path(obj: Any, path: Sequence[str]) -> Optional[Any]:
79
+ def _navigate_path(obj: Any, path: 'Sequence[str]') -> Optional[Any]:
80
80
  """
81
81
  Traverse the path in the given object either via attributes or via dict-like subscriptions.
82
82
  Returns the found value; None if navigation fails
@@ -136,7 +136,7 @@ def _unflatten_dict(dictionary: dict[str, Any]) -> dict[str, Any]:
136
136
  return ret
137
137
 
138
138
 
139
- def _get_request_data(request_path: Sequence[str]) -> "Callable[[LogDataSource, LogRecord], Iterator[tuple[str, Optional[Any]]]]":
139
+ def _get_request_data(request_path: 'Sequence[str]') -> "Callable[[LogDataSource, LogRecord], Iterator[tuple[str, Optional[Any]]]]":
140
140
  """
141
141
  Returns a function which, when called, will resolve the value
142
142
  in the flask request object at request_path
@@ -146,7 +146,7 @@ def _get_request_data(request_path: Sequence[str]) -> "Callable[[LogDataSource,
146
146
  # TODO: move to top of file once we got rid of/refactored rsemanager
147
147
  from flask import has_request_context, request
148
148
 
149
- def _request_data_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
149
+ def _request_data_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
150
150
  value = None
151
151
  if has_request_context() and request_path:
152
152
  value = _navigate_path(request, request_path)
@@ -161,7 +161,7 @@ def _get_record_attribute(attribute: str) -> "Callable[[LogDataSource, LogRecord
161
161
  the record passed in argument.
162
162
  """
163
163
 
164
- def _record_attribute_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
164
+ def _record_attribute_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
165
165
  value = None
166
166
  try:
167
167
  value = getattr(record, attribute)
@@ -172,7 +172,7 @@ def _get_record_attribute(attribute: str) -> "Callable[[LogDataSource, LogRecord
172
172
  return _record_attribute_formatter
173
173
 
174
174
 
175
- def _timestamp_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
175
+ def _timestamp_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
176
176
  """
177
177
  Format a timestamp
178
178
  """
@@ -214,7 +214,7 @@ class LogDataSource:
214
214
  def __str__(self):
215
215
  return self.__class__.__name__ + '(' + ', '.join(self.ecs_fields) + ')'
216
216
 
217
- def format(self, record: "LogRecord") -> Optional[Iterator[tuple[str, Any]]]:
217
+ def format(self, record: "LogRecord") -> Optional['Iterator[tuple[str, Any]]']:
218
218
  if not self._formatter:
219
219
  return
220
220
  for field_name, field_value in self._formatter(self, record):
@@ -241,7 +241,7 @@ class MessageLogDataSource(LogDataSource):
241
241
  return exc_info
242
242
  return None
243
243
 
244
- def format(self, record: "LogRecord") -> Iterator[tuple[str, Optional[str]]]:
244
+ def format(self, record: "LogRecord") -> 'Iterator[tuple[str, Optional[str]]]':
245
245
  exc_info = self._get_exc_info(record)
246
246
  message = record.getMessage()
247
247
  error_type, error_message, stack_trace = None, None, None
@@ -276,7 +276,7 @@ class ConstantStrDataSource(LogDataSource):
276
276
  log_record = ECS_TO_LOG_RECORD_MAP.get(ecs_field, None)
277
277
  self._str = _str
278
278
 
279
- def _formatter(data_source: LogDataSource, record: "LogRecord") -> Iterator[tuple[str, str]]:
279
+ def _formatter(data_source: LogDataSource, record: "LogRecord") -> 'Iterator[tuple[str, str]]':
280
280
  yield self.ecs_fields[0], self._str
281
281
 
282
282
  super().__init__(ecs_fields=(ecs_field,), formatter=_formatter, dst_record_attr=log_record)
@@ -303,7 +303,7 @@ class RucioFormatter(logging.Formatter):
303
303
  fmt: Optional[str] = None,
304
304
  validate: Optional[bool] = None,
305
305
  output_json: bool = False,
306
- additional_fields: Optional[Mapping[ECS_FIELDS, str]] = None
306
+ additional_fields: Optional['Mapping[ECS_FIELDS, str]'] = None
307
307
  ):
308
308
  _kwargs = {}
309
309
  if validate is not None:
@@ -384,7 +384,7 @@ class RucioFormatter(logging.Formatter):
384
384
  def rucio_log_formatter(process_name: Optional[str] = None) -> RucioFormatter:
385
385
  config_logformat = config_get('common', 'logformat', raise_exception=False, default='%(asctime)s\t%(name)s\t%(process)d\t%(levelname)s\t%(message)s')
386
386
  output_json = config_get_bool('common', 'logjson', default=False)
387
- additional_fields = {}
387
+ additional_fields: 'Mapping[ECS_FIELDS, str]' = {}
388
388
  if process_name:
389
389
  additional_fields['process.name'] = process_name
390
390
  return RucioFormatter(fmt=config_logformat, output_json=output_json, additional_fields=additional_fields)
@@ -406,7 +406,7 @@ def setup_logging(application: Optional["Flask"] = None, process_name: Optional[
406
406
  application.logger.addHandler(stdouthandler)
407
407
 
408
408
 
409
- def formatted_logger(innerfunc: Callable, formatstr: str = "%s") -> Callable:
409
+ def formatted_logger(innerfunc: 'Callable', formatstr: str = "%s") -> 'Callable':
410
410
  """
411
411
  Decorates the passed function, formatting log input by
412
412
  the passed formatstr. The format string must always include a %s.
@@ -415,6 +415,6 @@ def formatted_logger(innerfunc: Callable, formatstr: str = "%s") -> Callable:
415
415
  :param formatstr: format string with %s as placeholder.
416
416
  """
417
417
  @functools.wraps(innerfunc)
418
- def log_format(level: int, msg: object, *args, **kwargs) -> Callable:
418
+ def log_format(level: int, msg: object, *args, **kwargs) -> 'Callable':
419
419
  return innerfunc(level, formatstr % msg, *args, **kwargs)
420
420
  return log_format