rucio-clients 37.7.0__py3-none-any.whl → 38.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/cli/bin_legacy/rucio.py +51 -107
- rucio/cli/bin_legacy/rucio_admin.py +26 -26
- rucio/cli/command.py +1 -0
- rucio/cli/did.py +2 -2
- rucio/cli/opendata.py +132 -0
- rucio/cli/replica.py +15 -5
- rucio/cli/rule.py +7 -2
- rucio/cli/scope.py +3 -2
- rucio/cli/utils.py +28 -4
- rucio/client/baseclient.py +9 -1
- rucio/client/client.py +2 -0
- rucio/client/diracclient.py +73 -12
- rucio/client/opendataclient.py +249 -0
- rucio/client/subscriptionclient.py +30 -0
- rucio/client/uploadclient.py +10 -13
- rucio/common/constants.py +4 -1
- rucio/common/exception.py +55 -0
- rucio/common/plugins.py +45 -8
- rucio/common/schema/generic.py +5 -3
- rucio/common/schema/generic_multi_vo.py +4 -2
- rucio/common/types.py +8 -7
- rucio/common/utils.py +176 -11
- rucio/rse/protocols/protocol.py +9 -5
- rucio/rse/translation.py +17 -6
- rucio/vcsversion.py +4 -4
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/data/etc/rucio.cfg.template +2 -2
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/data/requirements.client.txt +1 -1
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/scripts/rucio +2 -1
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/METADATA +2 -2
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/RECORD +39 -38
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/licenses/AUTHORS.rst +1 -0
- rucio/client/fileclient.py +0 -57
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-37.7.0.data → rucio_clients-38.0.0.data}/scripts/rucio-admin +0 -0
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/WHEEL +0 -0
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-37.7.0.dist-info → rucio_clients-38.0.0.dist-info}/top_level.txt +0 -0
|
@@ -180,6 +180,36 @@ class SubscriptionClient(BaseClient):
|
|
|
180
180
|
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
181
181
|
raise exc_cls(exc_msg)
|
|
182
182
|
|
|
183
|
+
def deactivate_subscription(
|
|
184
|
+
self,
|
|
185
|
+
name: str,
|
|
186
|
+
account: Optional[str] = None
|
|
187
|
+
) -> Literal[True]:
|
|
188
|
+
"""
|
|
189
|
+
Mark a subscription as inactive
|
|
190
|
+
|
|
191
|
+
Parameters
|
|
192
|
+
----------
|
|
193
|
+
name : Name of the subscription
|
|
194
|
+
account : Account identifier
|
|
195
|
+
|
|
196
|
+
Raises
|
|
197
|
+
------
|
|
198
|
+
NotFound
|
|
199
|
+
If subscription is not found
|
|
200
|
+
"""
|
|
201
|
+
if not account:
|
|
202
|
+
account = self.account
|
|
203
|
+
path = self.SUB_BASEURL + '/' + account + '/' + name # type: ignore
|
|
204
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
205
|
+
data = dumps({'options': {'state': 'I'}})
|
|
206
|
+
result = self._send_request(url, type_='PUT', data=data)
|
|
207
|
+
if result.status_code == codes.created: # pylint: disable=no-member
|
|
208
|
+
return True
|
|
209
|
+
else:
|
|
210
|
+
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
211
|
+
raise exc_cls(exc_msg)
|
|
212
|
+
|
|
183
213
|
def list_subscription_rules(
|
|
184
214
|
self,
|
|
185
215
|
account: str,
|
rucio/client/uploadclient.py
CHANGED
|
@@ -21,7 +21,8 @@ import os.path
|
|
|
21
21
|
import random
|
|
22
22
|
import socket
|
|
23
23
|
import time
|
|
24
|
-
from
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
|
|
25
26
|
|
|
26
27
|
from rucio import version
|
|
27
28
|
from rucio.client.client import Client
|
|
@@ -126,7 +127,7 @@ class UploadClient:
|
|
|
126
127
|
def upload(
|
|
127
128
|
self,
|
|
128
129
|
items: "Iterable[FileToUploadDict]",
|
|
129
|
-
summary_file_path: Optional[str] = None,
|
|
130
|
+
summary_file_path: Optional[Union[str, os.PathLike[str]]] = None,
|
|
130
131
|
traces_copy_out: Optional[list["TraceBaseDict"]] = None,
|
|
131
132
|
ignore_availability: bool = False,
|
|
132
133
|
activity: Optional[str] = None
|
|
@@ -418,7 +419,7 @@ class UploadClient:
|
|
|
418
419
|
rse_attributes = {}
|
|
419
420
|
try:
|
|
420
421
|
rse_attributes = self.client.list_rse_attributes(rse)
|
|
421
|
-
except:
|
|
422
|
+
except Exception:
|
|
422
423
|
logger(logging.WARNING, 'Attributes of the RSE: %s not available.' % rse)
|
|
423
424
|
if self.client_location and 'lan' in rse_settings['domain'] and RseAttr.SITE in rse_attributes:
|
|
424
425
|
if self.client_location['site'] == rse_attributes[RseAttr.SITE]:
|
|
@@ -612,7 +613,8 @@ class UploadClient:
|
|
|
612
613
|
if checksum_name in file:
|
|
613
614
|
final_summary[file_did_str][checksum_name] = file[checksum_name]
|
|
614
615
|
|
|
615
|
-
|
|
616
|
+
summary_path = Path(summary_file_path)
|
|
617
|
+
with summary_path.open('w') as summary_file:
|
|
616
618
|
json.dump(final_summary, summary_file, sort_keys=True, indent=1)
|
|
617
619
|
|
|
618
620
|
if num_succeeded == 0:
|
|
@@ -865,7 +867,7 @@ class UploadClient:
|
|
|
865
867
|
This method iterates over the provided items, each describing a local path and
|
|
866
868
|
associated upload parameters, checks that each item has a valid path and RSE, and
|
|
867
869
|
computes basic file details such as size and checksums. If the item is a directory
|
|
868
|
-
and `recursive` is set, the method calls `
|
|
870
|
+
and `recursive` is set, the method calls `_collect_files_recursive` to traverse subdirectories,
|
|
869
871
|
creating or attaching them as Rucio datasets or containers.
|
|
870
872
|
|
|
871
873
|
Parameters
|
|
@@ -933,7 +935,7 @@ class UploadClient:
|
|
|
933
935
|
logger(logging.WARNING,
|
|
934
936
|
'Skipping %s because it has no files in it. Subdirectories are not supported.' % dname)
|
|
935
937
|
elif os.path.isdir(path) and recursive:
|
|
936
|
-
files.extend(cast("list[FileToUploadWithCollectedInfoDict]", self.
|
|
938
|
+
files.extend(cast("list[FileToUploadWithCollectedInfoDict]", self._collect_files_recursive(item)))
|
|
937
939
|
elif os.path.isfile(path) and not recursive:
|
|
938
940
|
file = self._collect_file_info(path, item)
|
|
939
941
|
files.append(file)
|
|
@@ -1322,10 +1324,7 @@ class UploadClient:
|
|
|
1322
1324
|
if self.tracing:
|
|
1323
1325
|
send_trace(trace, self.client.trace_host, self.client.user_agent)
|
|
1324
1326
|
|
|
1325
|
-
def
|
|
1326
|
-
self,
|
|
1327
|
-
item: "FileToUploadDict"
|
|
1328
|
-
) -> list["FileToUploadWithCollectedAndDatasetInfoDict"]:
|
|
1327
|
+
def _collect_files_recursive(self, item: "FileToUploadDict") -> list["FileToUploadWithCollectedAndDatasetInfoDict"]:
|
|
1329
1328
|
"""
|
|
1330
1329
|
Recursively inspects a folder and creates corresponding Rucio datasets or containers.
|
|
1331
1330
|
|
|
@@ -1380,10 +1379,9 @@ class UploadClient:
|
|
|
1380
1379
|
if path and isinstance(path, str):
|
|
1381
1380
|
if path[-1] == '/':
|
|
1382
1381
|
path = path[0:-1]
|
|
1383
|
-
i = 0
|
|
1384
1382
|
path = os.path.abspath(path)
|
|
1385
1383
|
for root, dirs, fnames in os.walk(path):
|
|
1386
|
-
if len(dirs) > 0 and len(fnames) > 0
|
|
1384
|
+
if len(dirs) > 0 and len(fnames) > 0:
|
|
1387
1385
|
self.logger(logging.ERROR, 'A container can only have either collections or files, not both')
|
|
1388
1386
|
raise InputValidationError('Invalid input folder structure')
|
|
1389
1387
|
if len(fnames) > 0:
|
|
@@ -1404,7 +1402,6 @@ class UploadClient:
|
|
|
1404
1402
|
elif len(dirs) == 0 and len(fnames) == 0:
|
|
1405
1403
|
self.logger(logging.WARNING, 'The folder %s is empty, skipping' % root)
|
|
1406
1404
|
continue
|
|
1407
|
-
i += 1
|
|
1408
1405
|
# if everything went ok, replicate the folder structure in Rucio storage
|
|
1409
1406
|
for dataset in datasets:
|
|
1410
1407
|
try:
|
rucio/common/constants.py
CHANGED
|
@@ -215,4 +215,7 @@ RSE_ATTRS_BOOL = Literal[
|
|
|
215
215
|
]
|
|
216
216
|
|
|
217
217
|
SUPPORTED_SIGN_URL_SERVICES_LITERAL = Literal['gcs', 's3', 'swift']
|
|
218
|
-
SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))
|
|
218
|
+
SUPPORTED_SIGN_URL_SERVICES = list(get_args(SUPPORTED_SIGN_URL_SERVICES_LITERAL))
|
|
219
|
+
|
|
220
|
+
OPENDATA_DID_STATE_LITERAL = Literal['draft', 'public', 'suspended']
|
|
221
|
+
OPENDATA_DID_STATE_LITERAL_LIST = list(get_args(OPENDATA_DID_STATE_LITERAL))
|
rucio/common/exception.py
CHANGED
|
@@ -1206,3 +1206,58 @@ class ConnectionParameterNotFound(RucioException):
|
|
|
1206
1206
|
super(ConnectionParameterNotFound, self).__init__(*args)
|
|
1207
1207
|
self._message = f"Required connection parameter '{param}' is not provided."
|
|
1208
1208
|
self.error_code = 114
|
|
1209
|
+
|
|
1210
|
+
|
|
1211
|
+
class OpenDataError(RucioException):
|
|
1212
|
+
"""
|
|
1213
|
+
Error related to open data.
|
|
1214
|
+
"""
|
|
1215
|
+
|
|
1216
|
+
def __init__(self, *args):
|
|
1217
|
+
super(OpenDataError, self).__init__(*args)
|
|
1218
|
+
self._message = "Error related to open data."
|
|
1219
|
+
self.error_code = 115
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
class OpenDataDataIdentifierNotFound(OpenDataError):
|
|
1223
|
+
"""
|
|
1224
|
+
Throws when the data identifier is not in the open data catalog.
|
|
1225
|
+
"""
|
|
1226
|
+
|
|
1227
|
+
def __init__(self, *args):
|
|
1228
|
+
super(OpenDataDataIdentifierNotFound, self).__init__(*args)
|
|
1229
|
+
self._message = "Data identifier not found in the open data catalog."
|
|
1230
|
+
self.error_code = 116
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
class OpenDataDataIdentifierAlreadyExists(OpenDataError):
|
|
1234
|
+
"""
|
|
1235
|
+
Throws when the data identifier already exists in the open data catalog.
|
|
1236
|
+
"""
|
|
1237
|
+
|
|
1238
|
+
def __init__(self, *args):
|
|
1239
|
+
super(OpenDataDataIdentifierAlreadyExists, self).__init__(*args)
|
|
1240
|
+
self._message = "Data identifier already exists in the open data catalog."
|
|
1241
|
+
self.error_code = 117
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
class OpenDataInvalidState(OpenDataError):
|
|
1245
|
+
"""
|
|
1246
|
+
Throws when the open data entry is in an invalid state.
|
|
1247
|
+
"""
|
|
1248
|
+
|
|
1249
|
+
def __init__(self, *args):
|
|
1250
|
+
super(OpenDataInvalidState, self).__init__(*args)
|
|
1251
|
+
self._message = "Open data entry is in an invalid state."
|
|
1252
|
+
self.error_code = 118
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
class OpenDataInvalidStateUpdate(OpenDataError):
|
|
1256
|
+
"""
|
|
1257
|
+
Throws when a forbidden state update is attempted (e.g. from public to draft).
|
|
1258
|
+
"""
|
|
1259
|
+
|
|
1260
|
+
def __init__(self, *args):
|
|
1261
|
+
super(OpenDataInvalidStateUpdate, self).__init__(*args)
|
|
1262
|
+
self._message = "Invalid state update attempted on open data entry."
|
|
1263
|
+
self.error_code = 119
|
rucio/common/plugins.py
CHANGED
|
@@ -16,12 +16,13 @@ import importlib
|
|
|
16
16
|
import logging
|
|
17
17
|
import os
|
|
18
18
|
from configparser import NoOptionError, NoSectionError
|
|
19
|
-
from typing import TYPE_CHECKING, Any, TypeVar
|
|
19
|
+
from typing import TYPE_CHECKING, Any, Optional, TypeVar
|
|
20
20
|
|
|
21
21
|
from packaging.specifiers import SpecifierSet
|
|
22
22
|
|
|
23
23
|
from rucio.common import config
|
|
24
24
|
from rucio.common.client import get_client_vo
|
|
25
|
+
from rucio.common.constants import DEFAULT_VO
|
|
25
26
|
from rucio.common.exception import InvalidAlgorithmName, PolicyPackageIsNotVersioned, PolicyPackageVersionError
|
|
26
27
|
from rucio.version import current_version
|
|
27
28
|
|
|
@@ -31,6 +32,8 @@ if TYPE_CHECKING:
|
|
|
31
32
|
|
|
32
33
|
from rucio.common.types import LoggerFunction
|
|
33
34
|
|
|
35
|
+
LOGGER = logging.getLogger('policy')
|
|
36
|
+
|
|
34
37
|
PolicyPackageAlgorithmsT = TypeVar('PolicyPackageAlgorithmsT', bound='PolicyPackageAlgorithms')
|
|
35
38
|
|
|
36
39
|
|
|
@@ -74,12 +77,44 @@ class PolicyPackageAlgorithms:
|
|
|
74
77
|
"""
|
|
75
78
|
_ALGORITHMS: dict[str, dict[str, 'Callable[..., Any]']] = {}
|
|
76
79
|
_loaded_policy_modules = False
|
|
80
|
+
_default_algorithms: dict[str, 'Callable[..., Any]'] = {}
|
|
77
81
|
|
|
78
82
|
def __init__(self) -> None:
|
|
79
83
|
if not self._loaded_policy_modules:
|
|
80
84
|
self._register_all_policy_package_algorithms()
|
|
81
85
|
self._loaded_policy_modules = True
|
|
82
86
|
|
|
87
|
+
@classmethod
|
|
88
|
+
def _get_default_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, vo: str = "") -> Optional['Callable[..., Any]']:
|
|
89
|
+
"""
|
|
90
|
+
Gets the default algorithm of this type, if present in the policy package.
|
|
91
|
+
The default algorithm is the function named algorithm_type within the module named algorithm_type.
|
|
92
|
+
Returns None if no default algorithm present.
|
|
93
|
+
"""
|
|
94
|
+
# check if default algorithm for this VO is already cached
|
|
95
|
+
type_for_vo = vo + "_" + algorithm_type
|
|
96
|
+
if type_for_vo in cls._default_algorithms:
|
|
97
|
+
return cls._default_algorithms[type_for_vo]
|
|
98
|
+
|
|
99
|
+
default_algorithm = None
|
|
100
|
+
try:
|
|
101
|
+
if vo == DEFAULT_VO:
|
|
102
|
+
vo = ''
|
|
103
|
+
package = cls._get_policy_package_name(vo)
|
|
104
|
+
except (NoOptionError, NoSectionError):
|
|
105
|
+
return default_algorithm
|
|
106
|
+
|
|
107
|
+
module_name = package + "." + algorithm_type
|
|
108
|
+
try:
|
|
109
|
+
module = importlib.import_module(module_name)
|
|
110
|
+
|
|
111
|
+
if hasattr(module, algorithm_type):
|
|
112
|
+
default_algorithm = getattr(module, algorithm_type)
|
|
113
|
+
cls._default_algorithms[type_for_vo] = default_algorithm
|
|
114
|
+
except ImportError:
|
|
115
|
+
LOGGER.info('Policy algorithm module %s could not be loaded' % module_name)
|
|
116
|
+
return default_algorithm
|
|
117
|
+
|
|
83
118
|
@classmethod
|
|
84
119
|
def _get_one_algorithm(cls: type[PolicyPackageAlgorithmsT], algorithm_type: str, name: str) -> 'Callable[..., Any]':
|
|
85
120
|
"""
|
|
@@ -143,16 +178,18 @@ class PolicyPackageAlgorithms:
|
|
|
143
178
|
for vo in vos:
|
|
144
179
|
cls._try_importing_policy(vo['vo'])
|
|
145
180
|
|
|
181
|
+
@classmethod
|
|
182
|
+
def _get_policy_package_name(cls: type[PolicyPackageAlgorithmsT], vo: str = "") -> str:
|
|
183
|
+
env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
|
|
184
|
+
package = os.getenv(env_name, "")
|
|
185
|
+
if not package:
|
|
186
|
+
package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
|
|
187
|
+
return package
|
|
188
|
+
|
|
146
189
|
@classmethod
|
|
147
190
|
def _try_importing_policy(cls: type[PolicyPackageAlgorithmsT], vo: str = "") -> None:
|
|
148
191
|
try:
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
env_name = 'RUCIO_POLICY_PACKAGE' + ('' if not vo else '_' + vo.upper())
|
|
152
|
-
package = os.getenv(env_name, "")
|
|
153
|
-
if not package:
|
|
154
|
-
package = str(config.config_get('policy', 'package' + ('' if not vo else '-' + vo)))
|
|
155
|
-
|
|
192
|
+
package = cls._get_policy_package_name(vo)
|
|
156
193
|
module = importlib.import_module(package)
|
|
157
194
|
check_policy_module_version(module)
|
|
158
195
|
|
rucio/common/schema/generic.py
CHANGED
|
@@ -56,7 +56,7 @@ NAME_LENGTH = 250
|
|
|
56
56
|
NAME = {"description": "Data Identifier name",
|
|
57
57
|
"type": "string",
|
|
58
58
|
"maxLength": NAME_LENGTH,
|
|
59
|
-
"pattern": "^[A-Za-z0-9][A-Za-z0-9
|
|
59
|
+
"pattern": r"^[/A-Za-z0-9][/A-Za-z0-9\.\-_]*$"}
|
|
60
60
|
|
|
61
61
|
R_NAME = {"description": "Data Identifier name",
|
|
62
62
|
"type": "string",
|
|
@@ -94,7 +94,9 @@ DEFAULT_RSE_ATTRIBUTE = {"description": "Default RSE attribute",
|
|
|
94
94
|
|
|
95
95
|
REPLICA_STATE = {"description": "Replica state",
|
|
96
96
|
"type": "string",
|
|
97
|
-
"enum": [
|
|
97
|
+
"enum": [
|
|
98
|
+
"AVAILABLE", "UNAVAILABLE", "COPYING", "BEING_DELETED", "BAD", "SOURCE", "TEMPORARY_UNAVAILABLE",
|
|
99
|
+
"A", "U", "C", "B", "D", "S", "T"]}
|
|
98
100
|
|
|
99
101
|
DATE = {"description": "Date",
|
|
100
102
|
"type": "string",
|
|
@@ -367,7 +369,7 @@ ACCOUNT_ATTRIBUTE = {"description": "Account attribute",
|
|
|
367
369
|
"type": "string",
|
|
368
370
|
"pattern": r'^[a-zA-Z0-9-_\\/\\.]{1,30}$'}
|
|
369
371
|
|
|
370
|
-
SCOPE_NAME_REGEXP =
|
|
372
|
+
SCOPE_NAME_REGEXP = r"/([^/]+)/(.*)"
|
|
371
373
|
|
|
372
374
|
DISTANCE = {"description": "RSE distance",
|
|
373
375
|
"type": "object",
|
|
@@ -57,7 +57,7 @@ NAME_LENGTH = 250
|
|
|
57
57
|
NAME = {"description": "Data Identifier name",
|
|
58
58
|
"type": "string",
|
|
59
59
|
"maxLength": NAME_LENGTH,
|
|
60
|
-
"pattern": "^[A-Za-z0-9][A-Za-z0-9
|
|
60
|
+
"pattern": r"^[/A-Za-z0-9][/A-Za-z0-9\.\-_]*$"}
|
|
61
61
|
|
|
62
62
|
R_NAME = {"description": "Data Identifier name",
|
|
63
63
|
"type": "string",
|
|
@@ -95,7 +95,9 @@ DEFAULT_RSE_ATTRIBUTE = {"description": "Default RSE attribute",
|
|
|
95
95
|
|
|
96
96
|
REPLICA_STATE = {"description": "Replica state",
|
|
97
97
|
"type": "string",
|
|
98
|
-
"enum": [
|
|
98
|
+
"enum": [
|
|
99
|
+
"AVAILABLE", "UNAVAILABLE", "COPYING", "BEING_DELETED", "BAD", "SOURCE", "TEMPORARY_UNAVAILABLE",
|
|
100
|
+
"A", "U", "C", "B", "D", "S", "T"]}
|
|
99
101
|
|
|
100
102
|
DATE = {"description": "Date",
|
|
101
103
|
"type": "string",
|
rucio/common/types.py
CHANGED
|
@@ -19,10 +19,10 @@ from os import PathLike
|
|
|
19
19
|
from rucio.common.constants import DEFAULT_VO
|
|
20
20
|
|
|
21
21
|
if sys.version_info < (3, 11): # pragma: no cover
|
|
22
|
-
from typing_extensions import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict, TypeGuard, Union # noqa: UP035
|
|
22
|
+
from typing_extensions import TYPE_CHECKING, Any, Literal, NotRequired, Optional, Required, TypedDict, TypeGuard, Union # noqa: UP035
|
|
23
23
|
PathTypeAlias = Union[PathLike, str]
|
|
24
24
|
else:
|
|
25
|
-
from typing import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict, TypeGuard, Union
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Literal, NotRequired, Optional, Required, TypedDict, TypeGuard, Union
|
|
26
26
|
PathTypeAlias = PathLike
|
|
27
27
|
|
|
28
28
|
|
|
@@ -391,8 +391,8 @@ class TraceSchemaDict(TypedDict):
|
|
|
391
391
|
class FileToUploadDict(TypedDict):
|
|
392
392
|
path: PathTypeAlias
|
|
393
393
|
rse: str
|
|
394
|
-
did_scope: str
|
|
395
|
-
did_name: str
|
|
394
|
+
did_scope: NotRequired[str]
|
|
395
|
+
did_name: NotRequired[str]
|
|
396
396
|
dataset_scope: NotRequired[str]
|
|
397
397
|
dataset_name: NotRequired[str]
|
|
398
398
|
dataset_meta: NotRequired[str]
|
|
@@ -408,6 +408,8 @@ class FileToUploadDict(TypedDict):
|
|
|
408
408
|
|
|
409
409
|
|
|
410
410
|
class FileToUploadWithCollectedInfoDict(FileToUploadDict):
|
|
411
|
+
did_name: Required[str]
|
|
412
|
+
did_scope: Required[str]
|
|
411
413
|
basename: str
|
|
412
414
|
adler32: str
|
|
413
415
|
md5: str
|
|
@@ -417,12 +419,11 @@ class FileToUploadWithCollectedInfoDict(FileToUploadDict):
|
|
|
417
419
|
dirname: str
|
|
418
420
|
upload_result: dict
|
|
419
421
|
bytes: int
|
|
420
|
-
basename: str
|
|
421
422
|
|
|
422
423
|
|
|
423
424
|
class FileToUploadWithCollectedAndDatasetInfoDict(FileToUploadWithCollectedInfoDict):
|
|
424
|
-
dataset_scope: str
|
|
425
|
-
dataset_name: str
|
|
425
|
+
dataset_scope: Required[str]
|
|
426
|
+
dataset_name: Required[str]
|
|
426
427
|
|
|
427
428
|
|
|
428
429
|
class RequestGatewayDict(TypedDict):
|
rucio/common/utils.py
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import argparse
|
|
16
16
|
import base64
|
|
17
|
+
import copy
|
|
17
18
|
import datetime
|
|
18
19
|
import errno
|
|
19
20
|
import getpass
|
|
@@ -30,20 +31,22 @@ import subprocess
|
|
|
30
31
|
import tempfile
|
|
31
32
|
import threading
|
|
32
33
|
import time
|
|
34
|
+
import types
|
|
33
35
|
from collections import OrderedDict
|
|
34
36
|
from enum import Enum
|
|
35
|
-
from functools import cache, wraps
|
|
37
|
+
from functools import cache, update_wrapper, wraps
|
|
36
38
|
from io import StringIO
|
|
37
39
|
from itertools import zip_longest
|
|
38
|
-
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
|
40
|
+
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast
|
|
39
41
|
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
|
|
40
42
|
from uuid import uuid4 as uuid
|
|
41
43
|
from xml.etree import ElementTree
|
|
42
44
|
|
|
43
45
|
import requests
|
|
46
|
+
from typing_extensions import ParamSpec
|
|
44
47
|
|
|
45
48
|
from rucio.common.config import config_get, config_get_bool
|
|
46
|
-
from rucio.common.constants import BASE_SCHEME_MAP
|
|
49
|
+
from rucio.common.constants import BASE_SCHEME_MAP, DEFAULT_VO
|
|
47
50
|
from rucio.common.exception import DIDFilterSyntaxError, DuplicateCriteriaInDIDFilter, InputValidationError, InvalidType, MetalinkJsonParsingError, MissingModuleException, RucioException
|
|
48
51
|
from rucio.common.extra import import_extras
|
|
49
52
|
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
@@ -59,6 +62,7 @@ if EXTRA_MODULES['paramiko']:
|
|
|
59
62
|
|
|
60
63
|
if TYPE_CHECKING:
|
|
61
64
|
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
|
65
|
+
|
|
62
66
|
T = TypeVar('T')
|
|
63
67
|
HashableKT = TypeVar('HashableKT')
|
|
64
68
|
HashableVT = TypeVar('HashableVT')
|
|
@@ -390,17 +394,24 @@ class NonDeterministicPFNAlgorithms(PolicyPackageAlgorithms):
|
|
|
390
394
|
|
|
391
395
|
_algorithm_type = 'non_deterministic_pfn'
|
|
392
396
|
|
|
393
|
-
def __init__(self) -> None:
|
|
397
|
+
def __init__(self, vo: str = DEFAULT_VO) -> None:
|
|
394
398
|
"""
|
|
395
399
|
Initialises a non-deterministic PFN construction object
|
|
396
400
|
"""
|
|
397
401
|
super().__init__()
|
|
398
402
|
|
|
403
|
+
self.vo = vo
|
|
404
|
+
|
|
399
405
|
def construct_non_deterministic_pfn(self, dsn: str, scope: Optional[str], filename: str, naming_convention: str) -> str:
|
|
400
406
|
"""
|
|
401
407
|
Calls the correct algorithm to generate a non-deterministic PFN
|
|
402
408
|
"""
|
|
403
|
-
|
|
409
|
+
fn = None
|
|
410
|
+
if naming_convention == 'def':
|
|
411
|
+
fn = super()._get_default_algorithm(NonDeterministicPFNAlgorithms._algorithm_type, self.vo)
|
|
412
|
+
if fn is None:
|
|
413
|
+
fn = self.get_algorithm(naming_convention)
|
|
414
|
+
return fn(dsn, scope, filename)
|
|
404
415
|
|
|
405
416
|
@classmethod
|
|
406
417
|
def supports(cls: type[NonDeterministicPFNAlgorithmsT], naming_convention: str) -> bool:
|
|
@@ -510,7 +521,7 @@ class NonDeterministicPFNAlgorithms(PolicyPackageAlgorithms):
|
|
|
510
521
|
NonDeterministicPFNAlgorithms._module_init_()
|
|
511
522
|
|
|
512
523
|
|
|
513
|
-
def construct_non_deterministic_pfn(dsn: str, scope: Optional[str], filename: str, naming_convention: Optional[str] = None) -> str:
|
|
524
|
+
def construct_non_deterministic_pfn(dsn: str, scope: Optional[str], filename: str, naming_convention: Optional[str] = None, vo: str = DEFAULT_VO) -> str:
|
|
514
525
|
"""
|
|
515
526
|
Applies non-deterministic PFN convention to the given replica.
|
|
516
527
|
use the naming_convention to call the actual function which will do the job.
|
|
@@ -518,7 +529,7 @@ def construct_non_deterministic_pfn(dsn: str, scope: Optional[str], filename: st
|
|
|
518
529
|
which are not implemented inside this main rucio repository, so changing the
|
|
519
530
|
argument list must be done with caution.
|
|
520
531
|
"""
|
|
521
|
-
pfn_algorithms = NonDeterministicPFNAlgorithms()
|
|
532
|
+
pfn_algorithms = NonDeterministicPFNAlgorithms(vo)
|
|
522
533
|
if naming_convention is None or not NonDeterministicPFNAlgorithms.supports(naming_convention):
|
|
523
534
|
naming_convention = 'def'
|
|
524
535
|
return pfn_algorithms.construct_non_deterministic_pfn(dsn, scope, filename, naming_convention)
|
|
@@ -551,17 +562,24 @@ class ScopeExtractionAlgorithms(PolicyPackageAlgorithms):
|
|
|
551
562
|
|
|
552
563
|
_algorithm_type = 'scope'
|
|
553
564
|
|
|
554
|
-
def __init__(self) -> None:
|
|
565
|
+
def __init__(self, vo: str = DEFAULT_VO) -> None:
|
|
555
566
|
"""
|
|
556
567
|
Initialises scope extraction algorithms object
|
|
557
568
|
"""
|
|
558
569
|
super().__init__()
|
|
559
570
|
|
|
571
|
+
self.vo = vo
|
|
572
|
+
|
|
560
573
|
def extract_scope(self, did: str, scopes: Optional['Sequence[str]'], extract_scope_convention: str) -> 'Sequence[str]':
|
|
561
574
|
"""
|
|
562
575
|
Calls the correct algorithm for scope extraction
|
|
563
576
|
"""
|
|
564
|
-
|
|
577
|
+
fn = None
|
|
578
|
+
if extract_scope_convention == 'def':
|
|
579
|
+
fn = super()._get_default_algorithm(ScopeExtractionAlgorithms._algorithm_type, self.vo)
|
|
580
|
+
if fn is None:
|
|
581
|
+
fn = self.get_algorithm(extract_scope_convention)
|
|
582
|
+
return fn(did, scopes)
|
|
565
583
|
|
|
566
584
|
@classmethod
|
|
567
585
|
def supports(cls: type[ScopeExtractionAlgorithmsT], extract_scope_convention: str) -> bool:
|
|
@@ -642,9 +660,10 @@ ScopeExtractionAlgorithms._module_init_()
|
|
|
642
660
|
def extract_scope(
|
|
643
661
|
did: str,
|
|
644
662
|
scopes: Optional['Sequence[str]'] = None,
|
|
645
|
-
default_extract: str = 'def'
|
|
663
|
+
default_extract: str = 'def',
|
|
664
|
+
vo: str = DEFAULT_VO
|
|
646
665
|
) -> 'Sequence[str]':
|
|
647
|
-
scope_extraction_algorithms = ScopeExtractionAlgorithms()
|
|
666
|
+
scope_extraction_algorithms = ScopeExtractionAlgorithms(vo)
|
|
648
667
|
extract_scope_convention = config_get('common', 'extract_scope', False, None) or config_get('policy', 'extract_scope', False, None)
|
|
649
668
|
if extract_scope_convention is None or not ScopeExtractionAlgorithms.supports(extract_scope_convention):
|
|
650
669
|
extract_scope_convention = default_extract
|
|
@@ -1706,3 +1725,149 @@ def get_transfer_schemas() -> dict[str, list[str]]:
|
|
|
1706
1725
|
scheme_map['davs'].append('srm')
|
|
1707
1726
|
|
|
1708
1727
|
return scheme_map
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
def wlcg_token_discovery() -> Optional[str]:
|
|
1731
|
+
"""
|
|
1732
|
+
Discovers a WLCG bearer token from the environment, following the specified precedence.
|
|
1733
|
+
Specs: https://zenodo.org/records/3937438
|
|
1734
|
+
|
|
1735
|
+
:returns: The discovered token (string), or None if no valid token is found.
|
|
1736
|
+
"""
|
|
1737
|
+
user_id = os.geteuid()
|
|
1738
|
+
token = None
|
|
1739
|
+
|
|
1740
|
+
# 1. Check BEARER_TOKEN environment variable
|
|
1741
|
+
token = os.environ.get('BEARER_TOKEN')
|
|
1742
|
+
if token is not None:
|
|
1743
|
+
token = token.strip()
|
|
1744
|
+
if token:
|
|
1745
|
+
return token
|
|
1746
|
+
|
|
1747
|
+
# 2. Check BEARER_TOKEN_FILE environment variable
|
|
1748
|
+
token_file = os.environ.get('BEARER_TOKEN_FILE')
|
|
1749
|
+
if token_file:
|
|
1750
|
+
try:
|
|
1751
|
+
with open(token_file, 'r') as f:
|
|
1752
|
+
token = f.read().strip()
|
|
1753
|
+
if token:
|
|
1754
|
+
return token
|
|
1755
|
+
except FileNotFoundError:
|
|
1756
|
+
pass
|
|
1757
|
+
except Exception:
|
|
1758
|
+
return None
|
|
1759
|
+
|
|
1760
|
+
# 3. Check $XDG_RUNTIME_DIR/bt_u$ID
|
|
1761
|
+
xdg_runtime_dir = os.environ.get('XDG_RUNTIME_DIR')
|
|
1762
|
+
if xdg_runtime_dir:
|
|
1763
|
+
token_path = os.path.join(xdg_runtime_dir, f'bt_u{user_id}')
|
|
1764
|
+
try:
|
|
1765
|
+
with open(token_path, 'r') as f:
|
|
1766
|
+
token = f.read().strip()
|
|
1767
|
+
if token:
|
|
1768
|
+
return token
|
|
1769
|
+
except FileNotFoundError:
|
|
1770
|
+
pass
|
|
1771
|
+
except Exception:
|
|
1772
|
+
return None
|
|
1773
|
+
|
|
1774
|
+
# 4. Check /tmp/bt_u$ID
|
|
1775
|
+
token_path = f'/tmp/bt_u{user_id}'
|
|
1776
|
+
try:
|
|
1777
|
+
with open(token_path, 'r') as f:
|
|
1778
|
+
token = f.read().strip()
|
|
1779
|
+
if token:
|
|
1780
|
+
return token
|
|
1781
|
+
except FileNotFoundError:
|
|
1782
|
+
pass
|
|
1783
|
+
except Exception:
|
|
1784
|
+
return None
|
|
1785
|
+
|
|
1786
|
+
# No valid token found
|
|
1787
|
+
return None
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
P = ParamSpec('P')
|
|
1791
|
+
R = TypeVar('R')
|
|
1792
|
+
|
|
1793
|
+
|
|
1794
|
+
def clone_function(
|
|
1795
|
+
func: 'Callable[P, R]',
|
|
1796
|
+
*,
|
|
1797
|
+
keep_wrapped: bool = False
|
|
1798
|
+
) -> 'Callable[P, R]':
|
|
1799
|
+
"""
|
|
1800
|
+
Create and return an **independent** copy of *func*.
|
|
1801
|
+
|
|
1802
|
+
The copy shares the original code object and global namespace but has
|
|
1803
|
+
its **own** identity, making it safe to mutate attributes such as
|
|
1804
|
+
``__doc__``, ``__name__`` or custom flags without affecting the source
|
|
1805
|
+
function. Closure cells, default arguments and keyword‑only defaults
|
|
1806
|
+
are preserved.
|
|
1807
|
+
|
|
1808
|
+
Parameters
|
|
1809
|
+
----------
|
|
1810
|
+
func
|
|
1811
|
+
The function to duplicate.
|
|
1812
|
+
keep_wrapped
|
|
1813
|
+
If *True* retains the ``__wrapped__`` pointer that ``update_wrapper`` adds
|
|
1814
|
+
(useful when we *do* want wrapper semantics). The default *False* removes
|
|
1815
|
+
it so that introspection treats the clone as a stand‑alone function.
|
|
1816
|
+
|
|
1817
|
+
Returns
|
|
1818
|
+
-------
|
|
1819
|
+
Callable[P, R]
|
|
1820
|
+
A new callable that behaves exactly like *func*. At runtime the
|
|
1821
|
+
object is a concrete ``types.FunctionType`` instance, but its static
|
|
1822
|
+
type mirrors the original callable’s *parameter list* and *return type*.
|
|
1823
|
+
|
|
1824
|
+
Examples
|
|
1825
|
+
--------
|
|
1826
|
+
>>> def greet(name: str) -> str:
|
|
1827
|
+
... \"\"\"Return a greeting.\"\"\"
|
|
1828
|
+
... return f"Hello {name}"
|
|
1829
|
+
...
|
|
1830
|
+
>>> new_greet = clone_function(greet)
|
|
1831
|
+
>>> new_greet.__doc__ = "An altered docstring."
|
|
1832
|
+
>>> greet.__doc__
|
|
1833
|
+
'Return a greeting.'
|
|
1834
|
+
>>> new_greet("world")
|
|
1835
|
+
'Hello world'
|
|
1836
|
+
"""
|
|
1837
|
+
orig = cast('types.FunctionType', func)
|
|
1838
|
+
|
|
1839
|
+
# 1. Re‑create the bare function object.
|
|
1840
|
+
new = types.FunctionType(
|
|
1841
|
+
orig.__code__,
|
|
1842
|
+
orig.__globals__,
|
|
1843
|
+
name=orig.__name__,
|
|
1844
|
+
argdefs=orig.__defaults__,
|
|
1845
|
+
closure=orig.__closure__,
|
|
1846
|
+
)
|
|
1847
|
+
|
|
1848
|
+
# 2. Copy metadata such as ``__name__``, ``__qualname__``, ``__module__`` and ``__doc__``.
|
|
1849
|
+
update_wrapper(
|
|
1850
|
+
new,
|
|
1851
|
+
orig,
|
|
1852
|
+
assigned=(
|
|
1853
|
+
"__module__",
|
|
1854
|
+
"__name__",
|
|
1855
|
+
"__qualname__",
|
|
1856
|
+
"__doc__",
|
|
1857
|
+
"__annotations__",
|
|
1858
|
+
),
|
|
1859
|
+
updated=(),
|
|
1860
|
+
)
|
|
1861
|
+
|
|
1862
|
+
# 3. Shallow‑copy the attribute dict so later mutations are independent.
|
|
1863
|
+
new.__dict__.update(copy.copy(orig.__dict__))
|
|
1864
|
+
|
|
1865
|
+
# 4. Copy the (kw‑only) default values if present.
|
|
1866
|
+
if orig.__kwdefaults__:
|
|
1867
|
+
new.__kwdefaults__ = orig.__kwdefaults__.copy()
|
|
1868
|
+
|
|
1869
|
+
# 5. Detach from the original wrapper chain unless explicitly requested.
|
|
1870
|
+
if not keep_wrapped and hasattr(new, "__wrapped__"):
|
|
1871
|
+
delattr(new, "__wrapped__")
|
|
1872
|
+
|
|
1873
|
+
return cast('Callable[P, R]', new)
|