rucio-clients 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/cli/__init__.py +14 -0
- rucio/cli/account.py +216 -0
- rucio/cli/bin_legacy/__init__.py +13 -0
- rucio_clients-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio_clients-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rucio.cfg.template +3 -19
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/requirements.client.txt +11 -7
- rucio_clients-37.0.0.data/scripts/rucio +133 -0
- rucio_clients-37.0.0.data/scripts/rucio-admin +97 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/METADATA +18 -14
- rucio_clients-37.0.0.dist-info/RECORD +104 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio_clients-35.7.0.dist-info/RECORD +0 -88
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/WHEEL +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/top_level.txt +0 -0
rucio/common/config.py
CHANGED
|
@@ -17,18 +17,23 @@
|
|
|
17
17
|
import configparser
|
|
18
18
|
import json
|
|
19
19
|
import os
|
|
20
|
-
from
|
|
20
|
+
from functools import cache
|
|
21
21
|
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload
|
|
22
22
|
|
|
23
23
|
from rucio.common import exception
|
|
24
|
-
from rucio.common.exception import ConfigNotFound, DatabaseException
|
|
24
|
+
from rucio.common.exception import ConfigLoadingError, ConfigNotFound, DatabaseException
|
|
25
25
|
|
|
26
26
|
_T = TypeVar('_T')
|
|
27
27
|
_U = TypeVar('_U')
|
|
28
28
|
|
|
29
29
|
if TYPE_CHECKING:
|
|
30
|
+
from collections.abc import Callable
|
|
31
|
+
|
|
30
32
|
from sqlalchemy.orm import Session
|
|
31
33
|
|
|
34
|
+
LEGACY_SECTION_NAME = {}
|
|
35
|
+
LEGACY_OPTION_NAME = {}
|
|
36
|
+
|
|
32
37
|
|
|
33
38
|
def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
|
|
34
39
|
if value.lower() in ['true', 'yes', 'on']:
|
|
@@ -110,7 +115,7 @@ def config_get(
|
|
|
110
115
|
session: "Optional[Session]" = ...,
|
|
111
116
|
use_cache: bool = ...,
|
|
112
117
|
expiration_time: int = ...,
|
|
113
|
-
convert_type_fnc: Callable[[str], _T],
|
|
118
|
+
convert_type_fnc: 'Callable[[str], _T]',
|
|
114
119
|
) -> _T:
|
|
115
120
|
...
|
|
116
121
|
|
|
@@ -126,7 +131,7 @@ def config_get(
|
|
|
126
131
|
session: "Optional[Session]" = ...,
|
|
127
132
|
use_cache: bool = ...,
|
|
128
133
|
expiration_time: int = ...,
|
|
129
|
-
convert_type_fnc: Callable[[str], _U],
|
|
134
|
+
convert_type_fnc: 'Callable[[str], _U]',
|
|
130
135
|
) -> Union[_T, _U]:
|
|
131
136
|
...
|
|
132
137
|
|
|
@@ -143,7 +148,7 @@ def config_get(
|
|
|
143
148
|
session: "Optional[Session]" = ...,
|
|
144
149
|
use_cache: bool = ...,
|
|
145
150
|
expiration_time: int = ...,
|
|
146
|
-
convert_type_fnc: Callable[[str], _U],
|
|
151
|
+
convert_type_fnc: 'Callable[[str], _U]',
|
|
147
152
|
) -> Union[_T, _U]:
|
|
148
153
|
...
|
|
149
154
|
|
|
@@ -158,7 +163,7 @@ def config_get(
|
|
|
158
163
|
session: "Optional[Session]" = None,
|
|
159
164
|
use_cache: bool = True,
|
|
160
165
|
expiration_time: int = 900,
|
|
161
|
-
convert_type_fnc: Callable[[str], _T] = lambda x: x,
|
|
166
|
+
convert_type_fnc: 'Callable[[str], _T]' = lambda x: x,
|
|
162
167
|
) -> Union[_T, _U]:
|
|
163
168
|
"""
|
|
164
169
|
Return the string value for a given option in a section
|
|
@@ -196,7 +201,7 @@ def config_get(
|
|
|
196
201
|
except ConfigNotFound:
|
|
197
202
|
pass
|
|
198
203
|
|
|
199
|
-
from rucio.common.
|
|
204
|
+
from rucio.common.client import is_client
|
|
200
205
|
client_mode = is_client()
|
|
201
206
|
|
|
202
207
|
if not client_mode and check_config_table:
|
|
@@ -223,8 +228,6 @@ def get_legacy_config(section: str, option: str):
|
|
|
223
228
|
:param option: The option of the new config.
|
|
224
229
|
:returns: The string value of the legacy option if one is found, None otherwise.
|
|
225
230
|
"""
|
|
226
|
-
LEGACY_SECTION_NAME = {}
|
|
227
|
-
LEGACY_OPTION_NAME = {}
|
|
228
231
|
|
|
229
232
|
section = LEGACY_SECTION_NAME.get(section, section)
|
|
230
233
|
option = LEGACY_OPTION_NAME.get(option, option)
|
|
@@ -544,12 +547,12 @@ def config_get_list(
|
|
|
544
547
|
section: str,
|
|
545
548
|
option: str,
|
|
546
549
|
*,
|
|
547
|
-
default:
|
|
550
|
+
default: _T = ...,
|
|
548
551
|
check_config_table: bool = ...,
|
|
549
552
|
session: "Optional[Session]" = ...,
|
|
550
553
|
use_cache: bool = ...,
|
|
551
554
|
expiration_time: int = ...,
|
|
552
|
-
) -> list[str]:
|
|
555
|
+
) -> Union[list[str], _T]:
|
|
553
556
|
...
|
|
554
557
|
|
|
555
558
|
|
|
@@ -637,7 +640,7 @@ def __config_get_table(
|
|
|
637
640
|
session: "Optional[Session]" = None,
|
|
638
641
|
use_cache: bool = True,
|
|
639
642
|
expiration_time: int = 900,
|
|
640
|
-
convert_type_fnc: Optional[Callable[[str], _T]],
|
|
643
|
+
convert_type_fnc: Optional['Callable[[str], _T]'],
|
|
641
644
|
) -> _T:
|
|
642
645
|
"""
|
|
643
646
|
Search for a section-option configuration parameter in the configuration table
|
|
@@ -655,7 +658,6 @@ def __config_get_table(
|
|
|
655
658
|
:raises ConfigNotFound
|
|
656
659
|
:raises DatabaseException
|
|
657
660
|
"""
|
|
658
|
-
global __CONFIG
|
|
659
661
|
try:
|
|
660
662
|
from rucio.core.config import get as core_config_get
|
|
661
663
|
return core_config_get(section, option, default=default, session=session, use_cache=use_cache,
|
|
@@ -664,7 +666,7 @@ def __config_get_table(
|
|
|
664
666
|
if raise_exception and default is None:
|
|
665
667
|
raise err
|
|
666
668
|
if clean_cached:
|
|
667
|
-
|
|
669
|
+
clean_cached_config()
|
|
668
670
|
return default
|
|
669
671
|
|
|
670
672
|
|
|
@@ -714,14 +716,8 @@ def get_config_dirs() -> list[str]:
|
|
|
714
716
|
"""
|
|
715
717
|
configdirs = []
|
|
716
718
|
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
if 'VIRTUAL_ENV' in os.environ:
|
|
721
|
-
configdirs.append('%s/etc/' % os.environ['VIRTUAL_ENV'])
|
|
722
|
-
|
|
723
|
-
if 'CONDA_PREFIX' in os.environ:
|
|
724
|
-
configdirs.append('%s/etc/' % os.environ['CONDA_PREFIX'])
|
|
719
|
+
env_vars = ['RUCIO_HOME', 'VIRTUAL_ENV', 'CONDA_PREFIX']
|
|
720
|
+
configdirs.extend([os.path.join(os.environ[var], 'etc', '') for var in env_vars if var in os.environ])
|
|
725
721
|
|
|
726
722
|
configdirs.append('/opt/rucio/etc/')
|
|
727
723
|
|
|
@@ -758,21 +754,15 @@ def get_rse_credentials(path_to_credentials_file: Optional[Union[str, os.PathLik
|
|
|
758
754
|
return credentials
|
|
759
755
|
|
|
760
756
|
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
757
|
+
@cache
|
|
764
758
|
def get_config() -> configparser.ConfigParser:
|
|
765
759
|
"""Factory function for the configuration class. Returns the ConfigParser instance."""
|
|
766
|
-
|
|
767
|
-
if __CONFIG is None:
|
|
768
|
-
__CONFIG = Config()
|
|
769
|
-
return __CONFIG.parser
|
|
760
|
+
return Config().parser
|
|
770
761
|
|
|
771
762
|
|
|
772
763
|
def clean_cached_config() -> None:
|
|
773
764
|
"""Deletes the cached config singleton instance."""
|
|
774
|
-
|
|
775
|
-
__CONFIG = None
|
|
765
|
+
get_config.cache_clear()
|
|
776
766
|
|
|
777
767
|
|
|
778
768
|
class Config:
|
|
@@ -795,7 +785,4 @@ class Config:
|
|
|
795
785
|
'\n\t' + '\n\t'.join(configs))
|
|
796
786
|
|
|
797
787
|
if not self.parser.read(self.configfile) == [self.configfile]:
|
|
798
|
-
raise
|
|
799
|
-
'Could not load Rucio configuration file. '
|
|
800
|
-
'Rucio tried loading the following configuration file:'
|
|
801
|
-
'\n\t' + self.configfile)
|
|
788
|
+
raise ConfigLoadingError(self.configfile)
|
rucio/common/constants.py
CHANGED
|
@@ -48,7 +48,7 @@ if config_get_bool('transfers', 'srm_https_compatibility', raise_exception=False
|
|
|
48
48
|
SCHEME_MAP['srm'].append('davs')
|
|
49
49
|
SCHEME_MAP['davs'].append('srm')
|
|
50
50
|
|
|
51
|
-
SORTING_ALGORITHMS_LITERAL = Literal['geoip', '
|
|
51
|
+
SORTING_ALGORITHMS_LITERAL = Literal['geoip', 'custom_table', 'random']
|
|
52
52
|
SORTING_ALGORITHMS = list(get_args(SORTING_ALGORITHMS_LITERAL))
|
|
53
53
|
|
|
54
54
|
SUPPORTED_PROTOCOLS_LITERAL = Literal['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 'storm', 'srm+https', 'scp', 'rsync', 'rclone', 'magnet']
|
|
@@ -56,8 +56,11 @@ SUPPORTED_PROTOCOLS: list[str] = list(get_args(SUPPORTED_PROTOCOLS_LITERAL))
|
|
|
56
56
|
|
|
57
57
|
RSE_SUPPORTED_PROTOCOL_DOMAINS_LITERAL = Literal['ALL', 'LAN', 'WAN']
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal['read', 'write', 'delete']
|
|
60
|
+
RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
|
|
61
|
+
|
|
62
|
+
RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL = Literal[RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, 'third_party_copy_read', 'third_party_copy_write']
|
|
63
|
+
RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS: list[str] = list(get_args(RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL))
|
|
61
64
|
|
|
62
65
|
FTS_STATE = namedtuple('FTS_STATE', ['SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY', 'NOT_USED',
|
|
63
66
|
'CANCELED'])('SUBMITTED', 'READY', 'ACTIVE', 'FAILED', 'FINISHED', 'FINISHEDDIRTY',
|
|
@@ -74,12 +77,14 @@ FTS_JOB_TYPE = namedtuple('FTS_JOB_TYPE', ['MULTIPLE_REPLICA', 'MULTI_HOP', 'SES
|
|
|
74
77
|
MAX_MESSAGE_LENGTH = 4000
|
|
75
78
|
|
|
76
79
|
|
|
80
|
+
@enum.unique
|
|
77
81
|
class SuspiciousAvailability(enum.Enum):
|
|
78
82
|
ALL = 0
|
|
79
83
|
EXIST_COPIES = 1
|
|
80
84
|
LAST_COPY = 2
|
|
81
85
|
|
|
82
86
|
|
|
87
|
+
@enum.unique
|
|
83
88
|
class ReplicaState(enum.Enum):
|
|
84
89
|
# From rucio.db.sqla.constants, update that file at the same time as this
|
|
85
90
|
AVAILABLE = 'A'
|
|
@@ -157,3 +162,56 @@ class RseAttr:
|
|
|
157
162
|
DEFAULT_LIMIT_FILES = 'default_limit_files'
|
|
158
163
|
QUOTA_APPROVERS = 'quota_approvers'
|
|
159
164
|
RULE_DELETERS = 'rule_deleters'
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# Literal types to allow overloading of functions with RSE attributes in their signature.
|
|
168
|
+
# RSE attributes are encoded via the BooleanString decorator as VARCHAR in the database,
|
|
169
|
+
# but they are used as either bool or string in the code.
|
|
170
|
+
# This is only determined at runtime, so for static type checking
|
|
171
|
+
# we need to manually specify which attrs are string and which are bool.
|
|
172
|
+
# In future, we could refactor RseAttr to avoid code duplication.
|
|
173
|
+
RSE_ATTRS_STR = Literal[
|
|
174
|
+
'archive_timeout',
|
|
175
|
+
'associated_sites',
|
|
176
|
+
'bittorrent_tracker_addr',
|
|
177
|
+
'country',
|
|
178
|
+
'decommission',
|
|
179
|
+
'default_account_limit_bytes',
|
|
180
|
+
'fts',
|
|
181
|
+
'globus_endpoint_id',
|
|
182
|
+
'lfn2pfn_algorithm',
|
|
183
|
+
'maximum_pin_lifetime',
|
|
184
|
+
'multihop_tombstone_delay',
|
|
185
|
+
'naming_convention',
|
|
186
|
+
'oidc_base_path',
|
|
187
|
+
'oidc_support'
|
|
188
|
+
'physgroup',
|
|
189
|
+
'qbittorrent_management_address'
|
|
190
|
+
'rule_approvers',
|
|
191
|
+
's3_url_style',
|
|
192
|
+
'simulate_multirange',
|
|
193
|
+
'site',
|
|
194
|
+
'source_for_total_space',
|
|
195
|
+
'source_for_used_space',
|
|
196
|
+
'staging_buffer',
|
|
197
|
+
'tombstone_delay',
|
|
198
|
+
'type'
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
RSE_ATTRS_BOOL = Literal[
|
|
202
|
+
'auto_approve_bytes',
|
|
203
|
+
'auto_approve_files',
|
|
204
|
+
'block_manual_approval',
|
|
205
|
+
'greedyDeletion',
|
|
206
|
+
'is_object_store',
|
|
207
|
+
'restricted_read',
|
|
208
|
+
'restricted_write',
|
|
209
|
+
'skip_upload_stat',
|
|
210
|
+
'staging_required',
|
|
211
|
+
'strict_copy',
|
|
212
|
+
'use_ipv4',
|
|
213
|
+
'verify_checksum'
|
|
214
|
+
]
|
|
215
|
+
|
|
216
|
+
SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
|
|
217
|
+
SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))
|
rucio/common/didtype.py
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
DID type to represent a did and to simplify operations on it
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
|
-
from typing import Union
|
|
19
|
+
from typing import Any, Union
|
|
20
20
|
|
|
21
21
|
from rucio.common.exception import DIDError
|
|
22
22
|
|
|
@@ -58,6 +58,22 @@ class DID:
|
|
|
58
58
|
self.scope: str = ''
|
|
59
59
|
self.name: str = ''
|
|
60
60
|
|
|
61
|
+
did = self._parse_did_from_args(*args, **kwargs)
|
|
62
|
+
|
|
63
|
+
self._construct_did(did)
|
|
64
|
+
|
|
65
|
+
if not self.is_valid_format():
|
|
66
|
+
raise DIDError('Object has invalid format after construction: {}'.format(str(self)))
|
|
67
|
+
|
|
68
|
+
def _parse_did_from_args(
|
|
69
|
+
self,
|
|
70
|
+
*args,
|
|
71
|
+
**kwargs
|
|
72
|
+
) -> Union["DID", str, tuple[str, str], list[str], dict[str, str]]:
|
|
73
|
+
"""
|
|
74
|
+
Parse the DID object from the given arguments
|
|
75
|
+
:return: DID object
|
|
76
|
+
"""
|
|
61
77
|
num_args = len(args)
|
|
62
78
|
num_kwargs = len(kwargs)
|
|
63
79
|
if (num_args + num_kwargs) > 2:
|
|
@@ -77,46 +93,78 @@ class DID:
|
|
|
77
93
|
else:
|
|
78
94
|
raise DIDError('Constructor got unexpected keyword argument: {}'.format(k))
|
|
79
95
|
else:
|
|
80
|
-
raise DIDError('First argument of constructor is expected to be string type'
|
|
96
|
+
raise DIDError('First argument of constructor is expected to be string type '
|
|
81
97
|
'when keyword argument is given. Given type: {}'.format(type(did)))
|
|
82
98
|
elif num_args == 0:
|
|
83
99
|
did = kwargs.get('did', kwargs)
|
|
84
100
|
else:
|
|
85
101
|
did = args
|
|
102
|
+
return did
|
|
86
103
|
|
|
104
|
+
def _construct_did(self, did: Any) -> None:
|
|
105
|
+
"""
|
|
106
|
+
Construct the DID object from the given input.
|
|
107
|
+
|
|
108
|
+
:param did: input to construct the DID object from
|
|
109
|
+
"""
|
|
87
110
|
if isinstance(did, dict):
|
|
88
|
-
self.
|
|
89
|
-
self.name = did.get('name', '')
|
|
90
|
-
if not self.has_scope():
|
|
91
|
-
self.update_implicit_scope()
|
|
111
|
+
self._did_from_dict(did)
|
|
92
112
|
elif isinstance(did, tuple) or isinstance(did, list):
|
|
93
|
-
|
|
94
|
-
raise DIDError('Construction from tuple or list requires exactly 2 elements')
|
|
95
|
-
self.scope = did[0]
|
|
96
|
-
self.name = did[1]
|
|
113
|
+
self._did_from_list_or_tuple(did)
|
|
97
114
|
elif isinstance(did, str):
|
|
98
|
-
|
|
99
|
-
if len(did_parts) == 1:
|
|
100
|
-
self.name = did
|
|
101
|
-
self.update_implicit_scope()
|
|
102
|
-
if not self.has_scope():
|
|
103
|
-
raise DIDError('Object construction from non-splitable string is ambigious')
|
|
104
|
-
else:
|
|
105
|
-
self.scope = did_parts[0]
|
|
106
|
-
self.name = did_parts[1]
|
|
115
|
+
self._did_from_str(did)
|
|
107
116
|
elif isinstance(did, DID):
|
|
108
|
-
self.
|
|
109
|
-
self.name = did.name
|
|
117
|
+
self._did_from_did_object(did)
|
|
110
118
|
else:
|
|
111
119
|
raise DIDError('Cannot build object from: {}'.format(type(did)))
|
|
112
120
|
|
|
113
121
|
if self.name.endswith('/'):
|
|
114
122
|
self.name = self.name[:-1]
|
|
115
123
|
|
|
116
|
-
|
|
117
|
-
|
|
124
|
+
def _did_from_str(self, did: str) -> None:
|
|
125
|
+
"""
|
|
126
|
+
Construct the DID from a string.
|
|
127
|
+
:param did: string containing the DID information
|
|
128
|
+
"""
|
|
129
|
+
did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
|
|
130
|
+
if len(did_parts) == 1:
|
|
131
|
+
self.name = did
|
|
132
|
+
self._update_implicit_scope()
|
|
133
|
+
if not self.has_scope():
|
|
134
|
+
raise DIDError('Object construction from non-splitable string is ambigious')
|
|
135
|
+
else:
|
|
136
|
+
self.scope = did_parts[0]
|
|
137
|
+
self.name = did_parts[1]
|
|
138
|
+
|
|
139
|
+
def _did_from_dict(self, did: dict[str, str]) -> None:
|
|
140
|
+
"""
|
|
141
|
+
Construct the DID from a dictionary.
|
|
142
|
+
:param did: dictionary optionally containing the keys 'scope' and 'name'
|
|
143
|
+
"""
|
|
144
|
+
self.scope = did.get('scope', '')
|
|
145
|
+
self.name = did.get('name', '')
|
|
146
|
+
if not self.has_scope():
|
|
147
|
+
self._update_implicit_scope()
|
|
148
|
+
|
|
149
|
+
def _did_from_list_or_tuple(self, did: Union[list[str], tuple[str, str]]) -> None:
|
|
150
|
+
"""
|
|
151
|
+
Construct the DID from a list or tuple.
|
|
152
|
+
:param did: list or tuple with expected length of 2
|
|
153
|
+
"""
|
|
154
|
+
if len(did) != 2:
|
|
155
|
+
raise DIDError('Construction from tuple or list requires exactly 2 elements. Number of elements passed: %i' % len(did))
|
|
156
|
+
self.scope = did[0]
|
|
157
|
+
self.name = did[1]
|
|
158
|
+
|
|
159
|
+
def _did_from_did_object(self, did: "DID") -> None:
|
|
160
|
+
"""
|
|
161
|
+
Construct the DID from another DID object.
|
|
162
|
+
:param did: DID object
|
|
163
|
+
"""
|
|
164
|
+
self.scope = did.scope
|
|
165
|
+
self.name = did.name
|
|
118
166
|
|
|
119
|
-
def
|
|
167
|
+
def _update_implicit_scope(self) -> None:
|
|
120
168
|
"""
|
|
121
169
|
This method sets the scope if it is implicitly given in self.name
|
|
122
170
|
"""
|
rucio/common/exception.py
CHANGED
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
|
|
21
21
|
"""
|
|
22
22
|
|
|
23
|
+
from typing import Optional
|
|
24
|
+
|
|
23
25
|
from rucio.common.constraints import AUTHORIZED_VALUE_TYPES
|
|
24
26
|
|
|
25
27
|
|
|
@@ -93,11 +95,12 @@ class ClientParameterMismatch(RucioException):
|
|
|
93
95
|
|
|
94
96
|
class ClientProtocolNotSupported(RucioException):
|
|
95
97
|
"""
|
|
96
|
-
|
|
98
|
+
Client protocol not supported
|
|
97
99
|
"""
|
|
98
|
-
|
|
100
|
+
|
|
101
|
+
def __init__(self, host: str, protocol: str, protocols_allowed: Optional[list[str]] = None, *args):
|
|
99
102
|
super(ClientProtocolNotSupported, self).__init__(*args)
|
|
100
|
-
self._message = "Client protocol not supported."
|
|
103
|
+
self._message = f"Client protocol '{protocol}' not supported when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
|
|
101
104
|
self.error_code = 6
|
|
102
105
|
|
|
103
106
|
|
|
@@ -1068,14 +1071,14 @@ class PolicyPackageVersionError(PolicyPackageBaseException):
|
|
|
1068
1071
|
"""
|
|
1069
1072
|
Policy package is not compatible with this version of Rucio.
|
|
1070
1073
|
"""
|
|
1071
|
-
def __init__(self, package: str, rucio_version: str,
|
|
1074
|
+
def __init__(self, package: str, rucio_version: str, supported_versionset: str, *args):
|
|
1072
1075
|
super(PolicyPackageVersionError, self).__init__(package, *args)
|
|
1073
1076
|
self.rucio_version = rucio_version
|
|
1074
|
-
self.
|
|
1077
|
+
self.supported_versionset = supported_versionset
|
|
1075
1078
|
self._message = 'Policy package %s is not compatible with this Rucio version.\nRucio version: %s\nVersions supported by the package: %s' % (
|
|
1076
1079
|
self.package,
|
|
1077
1080
|
self.rucio_version,
|
|
1078
|
-
self.
|
|
1081
|
+
self.supported_versionset
|
|
1079
1082
|
)
|
|
1080
1083
|
self.error_code = 103
|
|
1081
1084
|
|
|
@@ -1125,8 +1128,8 @@ class TraceValidationSchemaNotFound(RucioException):
|
|
|
1125
1128
|
"""
|
|
1126
1129
|
Trace validation schema not found.
|
|
1127
1130
|
"""
|
|
1128
|
-
def __init__(self, *args
|
|
1129
|
-
super(TraceValidationSchemaNotFound, self).__init__(*args
|
|
1131
|
+
def __init__(self, *args):
|
|
1132
|
+
super(TraceValidationSchemaNotFound, self).__init__(*args)
|
|
1130
1133
|
self._message = 'Trace validation schema not found.'
|
|
1131
1134
|
self.error_code = 108
|
|
1132
1135
|
|
|
@@ -1149,3 +1152,57 @@ class UnsupportedMetadataPlugin(RucioException):
|
|
|
1149
1152
|
super(UnsupportedMetadataPlugin, self).__init__(*args)
|
|
1150
1153
|
self._message = "The requested metadata plugin is not enabled on the server."
|
|
1151
1154
|
self.error_code = 110
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
class ChecksumCalculationError(RucioException):
|
|
1158
|
+
"""
|
|
1159
|
+
An error occurred while calculating the checksum.
|
|
1160
|
+
"""
|
|
1161
|
+
def __init__(
|
|
1162
|
+
self,
|
|
1163
|
+
algorithm_name: str,
|
|
1164
|
+
filepath: str,
|
|
1165
|
+
*args,
|
|
1166
|
+
**kwargs
|
|
1167
|
+
):
|
|
1168
|
+
super(ChecksumCalculationError, self).__init__(*args, **kwargs)
|
|
1169
|
+
self.algorithm_name = algorithm_name
|
|
1170
|
+
self.filepath = filepath
|
|
1171
|
+
self._message = 'An error occurred while calculating the %s checksum of file %s.' % (self.algorithm_name, self.filepath)
|
|
1172
|
+
self.error_code = 111
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
class ConfigLoadingError(RucioException):
|
|
1176
|
+
"""
|
|
1177
|
+
An error occurred while loading the configuration.
|
|
1178
|
+
"""
|
|
1179
|
+
def __init__(
|
|
1180
|
+
self,
|
|
1181
|
+
config_file: str,
|
|
1182
|
+
*args,
|
|
1183
|
+
**kwargs
|
|
1184
|
+
):
|
|
1185
|
+
super(ConfigLoadingError, self).__init__(*args, **kwargs)
|
|
1186
|
+
self._message = 'Could not load Rucio configuration file. Rucio tried loading the following configuration file:\n\t %s' % (config_file)
|
|
1187
|
+
self.error_code = 112
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
class ClientProtocolNotFound(RucioException):
|
|
1191
|
+
"""
|
|
1192
|
+
Missing protocol in client configuration (e.g. no http/https in url).
|
|
1193
|
+
"""
|
|
1194
|
+
|
|
1195
|
+
def __init__(self, host: str, protocols_allowed: Optional[list[str]] = None, *args):
|
|
1196
|
+
super(ClientProtocolNotFound, self).__init__(*args)
|
|
1197
|
+
self._message = f"Client protocol missing when connecting to host '{host}'.{' Allowed protocols: ' + ', '.join(protocols_allowed) if protocols_allowed else ''}"
|
|
1198
|
+
self.error_code = 113
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
class ConnectionParameterNotFound(RucioException):
|
|
1202
|
+
"""
|
|
1203
|
+
Thrown when a required connection parameter is missing.
|
|
1204
|
+
"""
|
|
1205
|
+
def __init__(self, param: str, *args):
|
|
1206
|
+
super(ConnectionParameterNotFound, self).__init__(*args)
|
|
1207
|
+
self._message = f"Required connection parameter '{param}' is not provided."
|
|
1208
|
+
self.error_code = 114
|
rucio/common/extra.py
CHANGED
|
@@ -13,24 +13,19 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import importlib
|
|
16
|
-
import
|
|
17
|
-
from typing import TYPE_CHECKING
|
|
16
|
+
from typing import TYPE_CHECKING, Optional
|
|
18
17
|
|
|
19
18
|
if TYPE_CHECKING:
|
|
20
|
-
from
|
|
19
|
+
from collections.abc import Iterable
|
|
20
|
+
from types import ModuleType
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
def import_extras(module_list:
|
|
23
|
+
def import_extras(module_list: 'Iterable[str]') -> dict[str, "Optional[ModuleType]"]:
|
|
24
24
|
out = dict()
|
|
25
25
|
for mod in module_list:
|
|
26
26
|
out[mod] = None
|
|
27
27
|
try:
|
|
28
|
-
|
|
29
|
-
# TODO: remove when https://github.com/paramiko/paramiko/issues/2038 is fixed
|
|
30
|
-
warnings.filterwarnings('ignore', 'Blowfish has been deprecated', module='paramiko')
|
|
31
|
-
# TODO: deprecated python 2 and 3.6 too ...
|
|
32
|
-
warnings.filterwarnings('ignore', 'Python .* is no longer supported', module='paramiko')
|
|
33
|
-
out[mod] = importlib.import_module(mod)
|
|
28
|
+
out[mod] = importlib.import_module(mod)
|
|
34
29
|
except ImportError:
|
|
35
30
|
pass
|
|
36
31
|
return out
|
rucio/common/logging.py
CHANGED
|
@@ -18,13 +18,13 @@ import json
|
|
|
18
18
|
import logging
|
|
19
19
|
import re
|
|
20
20
|
import sys
|
|
21
|
-
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
22
21
|
from traceback import format_tb
|
|
23
22
|
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, get_args
|
|
24
23
|
|
|
25
24
|
from rucio.common.config import config_get, config_get_bool
|
|
26
25
|
|
|
27
26
|
if TYPE_CHECKING:
|
|
27
|
+
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
28
28
|
from logging import LogRecord, _SysExcInfoType
|
|
29
29
|
|
|
30
30
|
from _typeshed import OptExcInfo
|
|
@@ -76,7 +76,7 @@ def _json_serializable(obj: Any) -> Union[dict[Any, Any], str]:
|
|
|
76
76
|
return str(obj)
|
|
77
77
|
|
|
78
78
|
|
|
79
|
-
def _navigate_path(obj: Any, path: Sequence[str]) -> Optional[Any]:
|
|
79
|
+
def _navigate_path(obj: Any, path: 'Sequence[str]') -> Optional[Any]:
|
|
80
80
|
"""
|
|
81
81
|
Traverse the path in the given object either via attributes or via dict-like subscriptions.
|
|
82
82
|
Returns the found value; None if navigation fails
|
|
@@ -136,7 +136,7 @@ def _unflatten_dict(dictionary: dict[str, Any]) -> dict[str, Any]:
|
|
|
136
136
|
return ret
|
|
137
137
|
|
|
138
138
|
|
|
139
|
-
def _get_request_data(request_path: Sequence[str]) -> "Callable[[LogDataSource, LogRecord], Iterator[tuple[str, Optional[Any]]]]":
|
|
139
|
+
def _get_request_data(request_path: 'Sequence[str]') -> "Callable[[LogDataSource, LogRecord], Iterator[tuple[str, Optional[Any]]]]":
|
|
140
140
|
"""
|
|
141
141
|
Returns a function which, when called, will resolve the value
|
|
142
142
|
in the flask request object at request_path
|
|
@@ -146,7 +146,7 @@ def _get_request_data(request_path: Sequence[str]) -> "Callable[[LogDataSource,
|
|
|
146
146
|
# TODO: move to top of file once we got rid of/refactored rsemanager
|
|
147
147
|
from flask import has_request_context, request
|
|
148
148
|
|
|
149
|
-
def _request_data_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
|
|
149
|
+
def _request_data_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
|
|
150
150
|
value = None
|
|
151
151
|
if has_request_context() and request_path:
|
|
152
152
|
value = _navigate_path(request, request_path)
|
|
@@ -161,7 +161,7 @@ def _get_record_attribute(attribute: str) -> "Callable[[LogDataSource, LogRecord
|
|
|
161
161
|
the record passed in argument.
|
|
162
162
|
"""
|
|
163
163
|
|
|
164
|
-
def _record_attribute_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
|
|
164
|
+
def _record_attribute_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
|
|
165
165
|
value = None
|
|
166
166
|
try:
|
|
167
167
|
value = getattr(record, attribute)
|
|
@@ -172,7 +172,7 @@ def _get_record_attribute(attribute: str) -> "Callable[[LogDataSource, LogRecord
|
|
|
172
172
|
return _record_attribute_formatter
|
|
173
173
|
|
|
174
174
|
|
|
175
|
-
def _timestamp_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> Iterator[tuple[str, Optional[Any]]]:
|
|
175
|
+
def _timestamp_formatter(record_formatter: "LogDataSource", record: "LogRecord") -> 'Iterator[tuple[str, Optional[Any]]]':
|
|
176
176
|
"""
|
|
177
177
|
Format a timestamp
|
|
178
178
|
"""
|
|
@@ -214,7 +214,7 @@ class LogDataSource:
|
|
|
214
214
|
def __str__(self):
|
|
215
215
|
return self.__class__.__name__ + '(' + ', '.join(self.ecs_fields) + ')'
|
|
216
216
|
|
|
217
|
-
def format(self, record: "LogRecord") -> Optional[Iterator[tuple[str, Any]]]:
|
|
217
|
+
def format(self, record: "LogRecord") -> Optional['Iterator[tuple[str, Any]]']:
|
|
218
218
|
if not self._formatter:
|
|
219
219
|
return
|
|
220
220
|
for field_name, field_value in self._formatter(self, record):
|
|
@@ -241,7 +241,7 @@ class MessageLogDataSource(LogDataSource):
|
|
|
241
241
|
return exc_info
|
|
242
242
|
return None
|
|
243
243
|
|
|
244
|
-
def format(self, record: "LogRecord") -> Iterator[tuple[str, Optional[str]]]:
|
|
244
|
+
def format(self, record: "LogRecord") -> 'Iterator[tuple[str, Optional[str]]]':
|
|
245
245
|
exc_info = self._get_exc_info(record)
|
|
246
246
|
message = record.getMessage()
|
|
247
247
|
error_type, error_message, stack_trace = None, None, None
|
|
@@ -276,7 +276,7 @@ class ConstantStrDataSource(LogDataSource):
|
|
|
276
276
|
log_record = ECS_TO_LOG_RECORD_MAP.get(ecs_field, None)
|
|
277
277
|
self._str = _str
|
|
278
278
|
|
|
279
|
-
def _formatter(data_source: LogDataSource, record: "LogRecord") -> Iterator[tuple[str, str]]:
|
|
279
|
+
def _formatter(data_source: LogDataSource, record: "LogRecord") -> 'Iterator[tuple[str, str]]':
|
|
280
280
|
yield self.ecs_fields[0], self._str
|
|
281
281
|
|
|
282
282
|
super().__init__(ecs_fields=(ecs_field,), formatter=_formatter, dst_record_attr=log_record)
|
|
@@ -303,7 +303,7 @@ class RucioFormatter(logging.Formatter):
|
|
|
303
303
|
fmt: Optional[str] = None,
|
|
304
304
|
validate: Optional[bool] = None,
|
|
305
305
|
output_json: bool = False,
|
|
306
|
-
additional_fields: Optional[Mapping[ECS_FIELDS, str]] = None
|
|
306
|
+
additional_fields: Optional['Mapping[ECS_FIELDS, str]'] = None
|
|
307
307
|
):
|
|
308
308
|
_kwargs = {}
|
|
309
309
|
if validate is not None:
|
|
@@ -384,7 +384,7 @@ class RucioFormatter(logging.Formatter):
|
|
|
384
384
|
def rucio_log_formatter(process_name: Optional[str] = None) -> RucioFormatter:
|
|
385
385
|
config_logformat = config_get('common', 'logformat', raise_exception=False, default='%(asctime)s\t%(name)s\t%(process)d\t%(levelname)s\t%(message)s')
|
|
386
386
|
output_json = config_get_bool('common', 'logjson', default=False)
|
|
387
|
-
additional_fields = {}
|
|
387
|
+
additional_fields: 'Mapping[ECS_FIELDS, str]' = {}
|
|
388
388
|
if process_name:
|
|
389
389
|
additional_fields['process.name'] = process_name
|
|
390
390
|
return RucioFormatter(fmt=config_logformat, output_json=output_json, additional_fields=additional_fields)
|
|
@@ -406,7 +406,7 @@ def setup_logging(application: Optional["Flask"] = None, process_name: Optional[
|
|
|
406
406
|
application.logger.addHandler(stdouthandler)
|
|
407
407
|
|
|
408
408
|
|
|
409
|
-
def formatted_logger(innerfunc: Callable, formatstr: str = "%s") -> Callable:
|
|
409
|
+
def formatted_logger(innerfunc: 'Callable', formatstr: str = "%s") -> 'Callable':
|
|
410
410
|
"""
|
|
411
411
|
Decorates the passed function, formatting log input by
|
|
412
412
|
the passed formatstr. The format string must always include a %s.
|
|
@@ -415,6 +415,6 @@ def formatted_logger(innerfunc: Callable, formatstr: str = "%s") -> Callable:
|
|
|
415
415
|
:param formatstr: format string with %s as placeholder.
|
|
416
416
|
"""
|
|
417
417
|
@functools.wraps(innerfunc)
|
|
418
|
-
def log_format(level: int, msg: object, *args, **kwargs) -> Callable:
|
|
418
|
+
def log_format(level: int, msg: object, *args, **kwargs) -> 'Callable':
|
|
419
419
|
return innerfunc(level, formatstr % msg, *args, **kwargs)
|
|
420
420
|
return log_format
|