rucio-clients 37.5.0__py3-none-any.whl → 37.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/cli/bin_legacy/rucio.py +41 -22
- rucio/cli/bin_legacy/rucio_admin.py +1 -1
- rucio/cli/did.py +2 -2
- rucio/cli/rse.py +2 -3
- rucio/cli/rule.py +9 -5
- rucio/cli/subscription.py +1 -1
- rucio/client/baseclient.py +9 -4
- rucio/client/didclient.py +16 -16
- rucio/client/downloadclient.py +16 -15
- rucio/client/exportclient.py +45 -4
- rucio/client/lockclient.py +3 -3
- rucio/client/pingclient.py +35 -4
- rucio/client/replicaclient.py +2 -2
- rucio/client/touchclient.py +3 -2
- rucio/client/uploadclient.py +728 -183
- rucio/common/cache.py +1 -2
- rucio/common/client.py +4 -30
- rucio/common/config.py +27 -3
- rucio/common/constants.py +5 -1
- rucio/common/didtype.py +2 -2
- rucio/common/pcache.py +20 -25
- rucio/common/plugins.py +12 -19
- rucio/common/policy.py +3 -2
- rucio/common/schema/__init__.py +11 -8
- rucio/common/types.py +7 -5
- rucio/common/utils.py +1 -1
- rucio/rse/__init__.py +7 -6
- rucio/rse/protocols/ngarc.py +2 -2
- rucio/rse/protocols/srm.py +1 -1
- rucio/rse/protocols/webdav.py +8 -1
- rucio/rse/rsemanager.py +5 -4
- rucio/rse/translation.py +2 -2
- rucio/vcsversion.py +3 -3
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/METADATA +1 -1
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/RECORD +46 -46
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/data/etc/rucio.cfg.template +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/data/requirements.client.txt +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/scripts/rucio +0 -0
- {rucio_clients-37.5.0.data → rucio_clients-37.7.0.data}/scripts/rucio-admin +0 -0
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/WHEEL +0 -0
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-37.5.0.dist-info → rucio_clients-37.7.0.dist-info}/top_level.txt +0 -0
rucio/cli/bin_legacy/rucio.py
CHANGED
|
@@ -26,7 +26,7 @@ import uuid
|
|
|
26
26
|
from copy import deepcopy
|
|
27
27
|
from datetime import datetime
|
|
28
28
|
from logging import DEBUG
|
|
29
|
-
from typing import Optional
|
|
29
|
+
from typing import TYPE_CHECKING, Optional
|
|
30
30
|
|
|
31
31
|
from rich.console import Console
|
|
32
32
|
from rich.padding import Padding
|
|
@@ -58,6 +58,9 @@ from rucio.common.extra import import_extras
|
|
|
58
58
|
from rucio.common.test_rucio_server import TestRucioServer
|
|
59
59
|
from rucio.common.utils import Color, StoreAndDeprecateWarningAction, chunks, extract_scope, parse_did_filter_from_string, parse_did_filter_from_string_fe, setup_logger, sizefmt
|
|
60
60
|
|
|
61
|
+
if TYPE_CHECKING:
|
|
62
|
+
from rucio.common.types import FileToUploadDict
|
|
63
|
+
|
|
61
64
|
EXTRA_MODULES = import_extras(['argcomplete'])
|
|
62
65
|
|
|
63
66
|
if EXTRA_MODULES['argcomplete']:
|
|
@@ -947,29 +950,44 @@ def upload(args, client, logger, console, spinner):
|
|
|
947
950
|
elif len(did) == 2:
|
|
948
951
|
logger.warning('Ignoring input {} because dataset DID is already set {}:{}'.format(arg, dsscope, dsname))
|
|
949
952
|
|
|
950
|
-
items = []
|
|
953
|
+
items: list[FileToUploadDict] = []
|
|
951
954
|
for arg in args.args:
|
|
952
955
|
if arg.count(':') > 0:
|
|
953
956
|
continue
|
|
957
|
+
if args.pfn and args.impl:
|
|
958
|
+
logger.warning('Ignoring --impl option because --pfn option given')
|
|
959
|
+
args.impl = None
|
|
960
|
+
|
|
961
|
+
item: FileToUploadDict = {'path': arg, 'rse': args.rse}
|
|
962
|
+
|
|
963
|
+
if args.scope:
|
|
964
|
+
item['did_scope'] = args.scope
|
|
965
|
+
if args.name:
|
|
966
|
+
item['did_name'] = args.name
|
|
967
|
+
if dsscope:
|
|
968
|
+
item['dataset_scope'] = dsscope
|
|
969
|
+
if dsname:
|
|
970
|
+
item['dataset_name'] = dsname
|
|
971
|
+
if args.impl:
|
|
972
|
+
item['impl'] = args.impl
|
|
973
|
+
if args.protocol:
|
|
974
|
+
item['force_scheme'] = args.protocol
|
|
954
975
|
if args.pfn:
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
'transfer_timeout': args.transfer_timeout,
|
|
971
|
-
'guid': args.guid,
|
|
972
|
-
'recursive': args.recursive})
|
|
976
|
+
item['pfn'] = args.pfn
|
|
977
|
+
if args.no_register:
|
|
978
|
+
item['no_register'] = True
|
|
979
|
+
if args.register_after_upload:
|
|
980
|
+
item['register_after_upload'] = True
|
|
981
|
+
if args.lifetime is not None:
|
|
982
|
+
item['lifetime'] = int(args.lifetime)
|
|
983
|
+
if args.transfer_timeout is not None:
|
|
984
|
+
item['transfer_timeout'] = int(args.transfer_timeout)
|
|
985
|
+
if args.guid:
|
|
986
|
+
item['guid'] = args.guid
|
|
987
|
+
if args.recursive:
|
|
988
|
+
item['recursive'] = True
|
|
989
|
+
|
|
990
|
+
items.append(item)
|
|
973
991
|
|
|
974
992
|
if len(items) < 1:
|
|
975
993
|
raise InputValidationError('No files could be extracted from the given arguments')
|
|
@@ -978,6 +996,7 @@ def upload(args, client, logger, console, spinner):
|
|
|
978
996
|
logger.error("A single GUID was specified on the command line, but there are multiple files to upload.")
|
|
979
997
|
logger.error("If GUID auto-detection is not used, only one file may be uploaded at a time")
|
|
980
998
|
raise InputValidationError('Invalid input argument composition')
|
|
999
|
+
|
|
981
1000
|
if len(items) > 1 and args.name:
|
|
982
1001
|
logger.error("A single LFN was specified on the command line, but there are multiple files to upload.")
|
|
983
1002
|
logger.error("If LFN auto-detection is not used, only one file may be uploaded at a time")
|
|
@@ -991,7 +1010,7 @@ def upload(args, client, logger, console, spinner):
|
|
|
991
1010
|
from rucio.client.uploadclient import UploadClient
|
|
992
1011
|
upload_client = UploadClient(client, logger=logger)
|
|
993
1012
|
summary_file_path = 'rucio_upload.json' if args.summary else None
|
|
994
|
-
upload_client.upload(items, summary_file_path)
|
|
1013
|
+
upload_client.upload(items=items, summary_file_path=summary_file_path)
|
|
995
1014
|
return SUCCESS
|
|
996
1015
|
|
|
997
1016
|
|
|
@@ -1266,7 +1285,7 @@ def add_rule(args, client, logger, console, spinner):
|
|
|
1266
1285
|
"""
|
|
1267
1286
|
%(prog)s add-rule <did> <copies> <rse-expression> [options]
|
|
1268
1287
|
|
|
1269
|
-
Add a rule to a
|
|
1288
|
+
Add a rule to a DID.
|
|
1270
1289
|
"""
|
|
1271
1290
|
|
|
1272
1291
|
dids = []
|
|
@@ -752,7 +752,7 @@ def add_scope(args, client, logger, console, spinner):
|
|
|
752
752
|
|
|
753
753
|
"""
|
|
754
754
|
client.add_scope(account=args.account, scope=args.scope)
|
|
755
|
-
print('Added new scope to account:
|
|
755
|
+
print(f'Added new scope to {args.account}: {args.scope}')
|
|
756
756
|
return SUCCESS
|
|
757
757
|
|
|
758
758
|
|
rucio/cli/did.py
CHANGED
|
@@ -130,7 +130,7 @@ def content_history(ctx, dids):
|
|
|
130
130
|
@click.argument("dids", nargs=-1)
|
|
131
131
|
@click.pass_context
|
|
132
132
|
def content_add_(ctx, to_did, from_file, dids):
|
|
133
|
-
"""Attach a list [dids] of
|
|
133
|
+
"""Attach a list [dids] of data identifiers (file or collection-type) to another data identifier (collection-type)"""
|
|
134
134
|
args = Arguments({"no_pager": ctx.obj.no_pager, "dids": dids, "todid": to_did, "fromfile": from_file})
|
|
135
135
|
attach(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
136
136
|
|
|
@@ -140,7 +140,7 @@ def content_add_(ctx, to_did, from_file, dids):
|
|
|
140
140
|
@click.argument("dids", nargs=-1)
|
|
141
141
|
@click.pass_context
|
|
142
142
|
def content_remove(ctx, dids, from_did):
|
|
143
|
-
"""Detach [dids], a list of DIDs (file or collection-type) from
|
|
143
|
+
"""Detach [dids], a list of DIDs (file or collection-type) from another Data Identifier (collection type)"""
|
|
144
144
|
args = Arguments({"no_pager": ctx.obj.no_pager, "dids": dids, "fromdid": from_did})
|
|
145
145
|
detach(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
146
146
|
|
rucio/cli/rse.py
CHANGED
|
@@ -55,11 +55,10 @@ def list_(ctx, rses, csv):
|
|
|
55
55
|
|
|
56
56
|
@rse.command("show")
|
|
57
57
|
@click.argument("rse-name")
|
|
58
|
-
@click.option("--csv", is_flag=True, default=False, help="Output list of RSE property key and values as a csv")
|
|
59
58
|
@click.pass_context
|
|
60
|
-
def show(ctx, rse_name
|
|
59
|
+
def show(ctx, rse_name):
|
|
61
60
|
"""Usage, protocols, settings, and attributes for a given RSE"""
|
|
62
|
-
info_rse(Arguments({"no_pager": ctx.obj.no_pager, "rse": rse_name
|
|
61
|
+
info_rse(Arguments({"no_pager": ctx.obj.no_pager, "rse": rse_name}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
63
62
|
|
|
64
63
|
|
|
65
64
|
@rse.command("add")
|
rucio/cli/rule.py
CHANGED
|
@@ -116,13 +116,17 @@ def move(ctx, rule_id, rses, activity, source_rses):
|
|
|
116
116
|
@click.option("--comment", help="Comment about the replication rule")
|
|
117
117
|
@click.option("--account", help="The account owning the rule")
|
|
118
118
|
@click.option("--stuck", is_flag=True, default=False, help="Set state to STUCK.")
|
|
119
|
+
@click.option('--suspend', is_flag=True, default=None, help='Set state to SUSPENDED.')
|
|
119
120
|
@click.option("--activity", help="Activity of the rule.")
|
|
120
121
|
@click.option("--cancel-requests", is_flag=True, default=False, help="Cancel requests when setting rules to stuck.")
|
|
121
122
|
@click.option("--priority", help="Priority of the requests of the rule.")
|
|
122
123
|
@click.option("--child-rule-id", help='Child rule id of the rule. Use "None" to remove an existing parent/child relationship.')
|
|
123
124
|
@click.option("--boost-rule", is_flag=True, default=False, help="Quickens the transition of a rule from STUCK to REPLICATING.")
|
|
124
125
|
@click.pass_context
|
|
125
|
-
def update(
|
|
126
|
+
def update(
|
|
127
|
+
ctx, rule_id: str, lifetime: str, locked: bool, source_rses: str, activity: str, comment: str,
|
|
128
|
+
account: str, stuck: bool, suspend: bool, cancel_requests: bool, priority: str, child_rule_id: str, boost_rule: bool
|
|
129
|
+
):
|
|
126
130
|
"""Update an existing rule"""
|
|
127
131
|
args = Arguments(
|
|
128
132
|
{
|
|
@@ -134,6 +138,7 @@ def update(ctx, rule_id, lifetime, locked, source_rses, activity, comment, accou
|
|
|
134
138
|
"rule_account": account,
|
|
135
139
|
"source_replica_expression": source_rses,
|
|
136
140
|
"state_stuck": stuck,
|
|
141
|
+
"state_suspended": suspend,
|
|
137
142
|
"cancel_requests": cancel_requests,
|
|
138
143
|
"priority": priority,
|
|
139
144
|
"child_rule_id": child_rule_id,
|
|
@@ -144,15 +149,14 @@ def update(ctx, rule_id, lifetime, locked, source_rses, activity, comment, accou
|
|
|
144
149
|
|
|
145
150
|
|
|
146
151
|
@rule.command("list")
|
|
147
|
-
@click.
|
|
148
|
-
@click.option("--id", "rule_id", help="List by rule id", hidden=True) # TODO: Remove. This doesn't work and does the same thing as show
|
|
152
|
+
@click.argument("did")
|
|
149
153
|
@click.option("--traverse", is_flag=True, default=False, help="Traverse the did tree and search for rules affecting this did")
|
|
150
154
|
@click.option("--csv", is_flag=True, default=False, help="Comma Separated Value output")
|
|
151
155
|
@click.option("--file", help="Filter by file")
|
|
152
156
|
@click.option("--account", help="Filter by account")
|
|
153
157
|
@click.option("--subscription", help="Filter by subscription name")
|
|
154
158
|
@click.pass_context
|
|
155
|
-
def list_(ctx, did,
|
|
159
|
+
def list_(ctx, did, traverse, csv, file, account, subscription):
|
|
156
160
|
"""List all rules impacting a given DID"""
|
|
157
|
-
args = Arguments({"no_pager": ctx.obj.no_pager, "did": did, "rule_id":
|
|
161
|
+
args = Arguments({"no_pager": ctx.obj.no_pager, "did": did, "rule_id": None, "traverse": traverse, "csv": csv, "file": file, "subscription": (account if account is not None else ctx.obj.client.account, subscription)})
|
|
158
162
|
list_rules(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
rucio/cli/subscription.py
CHANGED
|
@@ -68,6 +68,6 @@ def add_(ctx, subscription_name, did_filter, rule, comment, lifetime, account, p
|
|
|
68
68
|
@click.pass_context
|
|
69
69
|
def touch(ctx, dids):
|
|
70
70
|
"""Reevaluate list of DIDs against all active subscriptions"""
|
|
71
|
-
# TODO make reeval accept
|
|
71
|
+
# TODO make reeval accept DIDs as a list
|
|
72
72
|
dids = ",".join(dids)
|
|
73
73
|
reevaluate_did_for_subscription(Arguments({"no_pager": ctx.obj.no_pager, "dids": dids}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
rucio/client/baseclient.py
CHANGED
|
@@ -38,6 +38,7 @@ from requests.status_codes import codes
|
|
|
38
38
|
from rucio import version
|
|
39
39
|
from rucio.common import exception
|
|
40
40
|
from rucio.common.config import config_get, config_get_bool, config_get_int, config_has_section
|
|
41
|
+
from rucio.common.constants import DEFAULT_VO
|
|
41
42
|
from rucio.common.exception import CannotAuthenticate, ClientProtocolNotFound, ClientProtocolNotSupported, ConfigNotFound, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException
|
|
42
43
|
from rucio.common.extra import import_extras
|
|
43
44
|
from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign
|
|
@@ -225,10 +226,10 @@ class BaseClient:
|
|
|
225
226
|
self.vo = config_get('client', 'vo')
|
|
226
227
|
except (NoOptionError, NoSectionError):
|
|
227
228
|
self.logger.debug('No VO found. Using default VO.')
|
|
228
|
-
self.vo =
|
|
229
|
+
self.vo = DEFAULT_VO
|
|
229
230
|
except ConfigNotFound:
|
|
230
231
|
self.logger.debug('No configuration found. Using default VO.')
|
|
231
|
-
self.vo =
|
|
232
|
+
self.vo = DEFAULT_VO
|
|
232
233
|
|
|
233
234
|
self.auth_token_file_path, self.token_exp_epoch_file, self.token_file, self.token_path = self._get_auth_tokens()
|
|
234
235
|
self.__authenticate()
|
|
@@ -251,7 +252,7 @@ class BaseClient:
|
|
|
251
252
|
|
|
252
253
|
else:
|
|
253
254
|
token_path = self.TOKEN_PATH_PREFIX + getpass.getuser()
|
|
254
|
-
if self.vo !=
|
|
255
|
+
if self.vo != DEFAULT_VO:
|
|
255
256
|
token_path += '@%s' % self.vo
|
|
256
257
|
|
|
257
258
|
token_file = token_path + '/' + self.TOKEN_PREFIX + token_filename_suffix
|
|
@@ -967,7 +968,11 @@ class BaseClient:
|
|
|
967
968
|
if not os.path.isdir(self.token_path):
|
|
968
969
|
try:
|
|
969
970
|
self.logger.debug('rucio token folder \'%s\' not found. Create it.' % self.token_path)
|
|
970
|
-
|
|
971
|
+
try:
|
|
972
|
+
makedirs(self.token_path, 0o700)
|
|
973
|
+
except FileExistsError:
|
|
974
|
+
msg = f'Token directory already exists at {self.token_path} - skipping'
|
|
975
|
+
self.logger.debug(msg)
|
|
971
976
|
except Exception:
|
|
972
977
|
raise
|
|
973
978
|
|
rucio/client/didclient.py
CHANGED
|
@@ -55,7 +55,7 @@ class DIDClient(BaseClient):
|
|
|
55
55
|
like <key>.<operation>, e.g. key1 >= value1 is equivalent to {'key1.gte': value}, where <operation> belongs to one
|
|
56
56
|
of the set {'lte', 'gte', 'gt', 'lt', 'ne' or ''}. Equivalence doesn't require an operator.
|
|
57
57
|
did_type :
|
|
58
|
-
The type of the
|
|
58
|
+
The type of the DID: 'all'(container, dataset or file)|'collection'(dataset or container)|'dataset'|'container'|'file'
|
|
59
59
|
long :
|
|
60
60
|
Long format option to display more information for each DID.
|
|
61
61
|
recursive :
|
|
@@ -325,14 +325,14 @@ class DIDClient(BaseClient):
|
|
|
325
325
|
ignore_duplicate: bool = False
|
|
326
326
|
) -> bool:
|
|
327
327
|
"""
|
|
328
|
-
Add
|
|
328
|
+
Add DIDs to DIDs.
|
|
329
329
|
|
|
330
330
|
Parameters
|
|
331
331
|
----------
|
|
332
332
|
attachments :
|
|
333
333
|
The attachments.
|
|
334
|
-
An attachment contains: "scope", "name", "
|
|
335
|
-
|
|
334
|
+
An attachment contains: "scope", "name", "DIDs".
|
|
335
|
+
DIDs is: [{'scope': scope, 'name': name}, ...]
|
|
336
336
|
ignore_duplicate :
|
|
337
337
|
If True, ignore duplicate entries.
|
|
338
338
|
"""
|
|
@@ -358,8 +358,8 @@ class DIDClient(BaseClient):
|
|
|
358
358
|
----------
|
|
359
359
|
attachments :
|
|
360
360
|
The attachments.
|
|
361
|
-
An attachment contains: "scope", "name", "
|
|
362
|
-
|
|
361
|
+
An attachment contains: "scope", "name", "DIDs".
|
|
362
|
+
DIDs is: [{'scope': scope, 'name': name}, ...]
|
|
363
363
|
ignore_duplicate :
|
|
364
364
|
If True, ignore duplicate entries.
|
|
365
365
|
"""
|
|
@@ -377,8 +377,8 @@ class DIDClient(BaseClient):
|
|
|
377
377
|
----------
|
|
378
378
|
attachments :
|
|
379
379
|
The attachments.
|
|
380
|
-
An attachment contains: "scope", "name", "
|
|
381
|
-
|
|
380
|
+
An attachment contains: "scope", "name", "DIDs".
|
|
381
|
+
DIDs is: [{'scope': scope, 'name': name}, ...]
|
|
382
382
|
"""
|
|
383
383
|
return self.attach_dids_to_dids(attachments=attachments)
|
|
384
384
|
|
|
@@ -393,8 +393,8 @@ class DIDClient(BaseClient):
|
|
|
393
393
|
----------
|
|
394
394
|
attachments :
|
|
395
395
|
The attachments.
|
|
396
|
-
An attachment contains: "scope", "name", "
|
|
397
|
-
|
|
396
|
+
An attachment contains: "scope", "name", "DIDs".
|
|
397
|
+
DIDs is: [{'scope': scope, 'name': name}, ...]
|
|
398
398
|
"""
|
|
399
399
|
return self.attach_dids_to_dids(attachments=attachments)
|
|
400
400
|
|
|
@@ -661,7 +661,7 @@ class DIDClient(BaseClient):
|
|
|
661
661
|
Parameters
|
|
662
662
|
----------
|
|
663
663
|
dids :
|
|
664
|
-
A list of
|
|
664
|
+
A list of DIDs.
|
|
665
665
|
inherit :
|
|
666
666
|
A boolean. If set to true, the metadata of the parent are concatenated.
|
|
667
667
|
plugin :
|
|
@@ -752,7 +752,7 @@ class DIDClient(BaseClient):
|
|
|
752
752
|
Parameters
|
|
753
753
|
----------
|
|
754
754
|
dids :
|
|
755
|
-
A list of
|
|
755
|
+
A list of DIDs including metadata, i.e.
|
|
756
756
|
[{'scope': scope1, 'name': name1, 'meta': {key1: value1, key2: value2}}, ...].
|
|
757
757
|
recursive :
|
|
758
758
|
Option to propagate the metadata update to content.
|
|
@@ -902,7 +902,7 @@ class DIDClient(BaseClient):
|
|
|
902
902
|
Returns
|
|
903
903
|
-------
|
|
904
904
|
|
|
905
|
-
A
|
|
905
|
+
A DID
|
|
906
906
|
"""
|
|
907
907
|
|
|
908
908
|
path = '/'.join([self.DIDS_BASEURL, guid, 'guid'])
|
|
@@ -954,7 +954,7 @@ class DIDClient(BaseClient):
|
|
|
954
954
|
name: str
|
|
955
955
|
) -> "Iterator[dict[str, Any]]":
|
|
956
956
|
"""
|
|
957
|
-
List parent dataset/containers of a
|
|
957
|
+
List parent dataset/containers of a DID.
|
|
958
958
|
|
|
959
959
|
Parameters
|
|
960
960
|
----------
|
|
@@ -1016,12 +1016,12 @@ class DIDClient(BaseClient):
|
|
|
1016
1016
|
|
|
1017
1017
|
def resurrect(self, dids: "Sequence[Mapping[str, Any]]") -> bool:
|
|
1018
1018
|
"""
|
|
1019
|
-
Resurrect a list of
|
|
1019
|
+
Resurrect a list of DIDs.
|
|
1020
1020
|
|
|
1021
1021
|
Parameters
|
|
1022
1022
|
----------
|
|
1023
1023
|
dids :
|
|
1024
|
-
A list of
|
|
1024
|
+
A list of DIDs [{'scope': scope, 'name': name}, ...]
|
|
1025
1025
|
"""
|
|
1026
1026
|
path = '/'.join([self.DIDS_BASEURL, 'resurrect'])
|
|
1027
1027
|
url = build_url(choice(self.list_hosts), path=path)
|
rucio/client/downloadclient.py
CHANGED
|
@@ -32,6 +32,7 @@ from rucio.client.client import Client
|
|
|
32
32
|
from rucio.common.checksum import CHECKSUM_ALGO_DICT, GLOBALLY_SUPPORTED_CHECKSUMS, PREFERRED_CHECKSUM, adler32
|
|
33
33
|
from rucio.common.client import detect_client_location
|
|
34
34
|
from rucio.common.config import config_get
|
|
35
|
+
from rucio.common.constants import DEFAULT_VO
|
|
35
36
|
from rucio.common.didtype import DID
|
|
36
37
|
from rucio.common.exception import InputValidationError, NoFilesDownloaded, NotAllFilesDownloaded, RucioException
|
|
37
38
|
from rucio.common.pcache import Pcache
|
|
@@ -213,7 +214,7 @@ class DownloadClient:
|
|
|
213
214
|
self.trace_tpl['hostname'] = self.client_location['fqdn']
|
|
214
215
|
self.trace_tpl['localSite'] = self.client_location['site']
|
|
215
216
|
self.trace_tpl['account'] = self.client.account
|
|
216
|
-
if self.client.vo !=
|
|
217
|
+
if self.client.vo != DEFAULT_VO:
|
|
217
218
|
self.trace_tpl['vo'] = self.client.vo
|
|
218
219
|
self.trace_tpl['eventType'] = 'download'
|
|
219
220
|
self.trace_tpl['eventVersion'] = 'api_%s' % version.RUCIO_VERSION[0]
|
|
@@ -285,7 +286,7 @@ class DownloadClient:
|
|
|
285
286
|
-------
|
|
286
287
|
|
|
287
288
|
A list of dictionaries with an entry for each file, containing the input options,
|
|
288
|
-
the
|
|
289
|
+
the DID, and the clientState. clientState can be one of the following:
|
|
289
290
|
ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
|
|
290
291
|
|
|
291
292
|
Raises
|
|
@@ -422,7 +423,7 @@ class DownloadClient:
|
|
|
422
423
|
-------
|
|
423
424
|
|
|
424
425
|
A list of dictionaries with an entry for each file, containing the input options,
|
|
425
|
-
the
|
|
426
|
+
the DID, and the clientState.
|
|
426
427
|
|
|
427
428
|
Raises
|
|
428
429
|
------
|
|
@@ -499,7 +500,7 @@ class DownloadClient:
|
|
|
499
500
|
-------
|
|
500
501
|
|
|
501
502
|
A list of dictionaries with an entry for each file, containing the input options,
|
|
502
|
-
the
|
|
503
|
+
the DID, and the clientState.
|
|
503
504
|
|
|
504
505
|
Raises
|
|
505
506
|
------
|
|
@@ -1021,7 +1022,7 @@ class DownloadClient:
|
|
|
1021
1022
|
-------
|
|
1022
1023
|
|
|
1023
1024
|
A list of dictionaries with an entry for each file, containing the input options,
|
|
1024
|
-
the
|
|
1025
|
+
the DID, and the clientState.
|
|
1025
1026
|
|
|
1026
1027
|
|
|
1027
1028
|
Raises
|
|
@@ -1173,7 +1174,7 @@ class DownloadClient:
|
|
|
1173
1174
|
-------
|
|
1174
1175
|
|
|
1175
1176
|
A list of dictionaries with an entry for each file, containing the input options,
|
|
1176
|
-
the
|
|
1177
|
+
the DID, and the clientState.
|
|
1177
1178
|
"""
|
|
1178
1179
|
trace_custom_fields = trace_custom_fields or {}
|
|
1179
1180
|
logger = self.logger
|
|
@@ -1322,7 +1323,7 @@ class DownloadClient:
|
|
|
1322
1323
|
|
|
1323
1324
|
def _resolve_one_item_dids(self, item: dict[str, Any]) -> "Iterator[dict[str, Any]]":
|
|
1324
1325
|
"""
|
|
1325
|
-
Resolve scopes or wildcard DIDs to lists of full
|
|
1326
|
+
Resolve scopes or wildcard DIDs to lists of full DID names:
|
|
1326
1327
|
|
|
1327
1328
|
Parameters
|
|
1328
1329
|
----------
|
|
@@ -1456,10 +1457,10 @@ class DownloadClient:
|
|
|
1456
1457
|
if not found_compatible_group:
|
|
1457
1458
|
item_groups.append([item])
|
|
1458
1459
|
|
|
1459
|
-
# List replicas for
|
|
1460
|
+
# List replicas for DIDs
|
|
1460
1461
|
merged_items_with_sources = []
|
|
1461
1462
|
for item_group in item_groups:
|
|
1462
|
-
# Take configuration from the first item in the group; but
|
|
1463
|
+
# Take configuration from the first item in the group; but DIDs from all items
|
|
1463
1464
|
item = item_group[0]
|
|
1464
1465
|
input_dids = {DID(did): did
|
|
1465
1466
|
for item in item_group
|
|
@@ -1512,15 +1513,15 @@ class DownloadClient:
|
|
|
1512
1513
|
logger(logging.DEBUG, 'num resolved files: %s' % len(file_items))
|
|
1513
1514
|
|
|
1514
1515
|
if not nrandom or nrandom != len(file_items):
|
|
1515
|
-
# If list_replicas didn't resolve any file DIDs for any input
|
|
1516
|
+
# If list_replicas didn't resolve any file DIDs for any input DID, we pass through the input DID.
|
|
1516
1517
|
# This is done to keep compatibility with later code which generates "FILE_NOT_FOUND" traces
|
|
1517
1518
|
# and output items.
|
|
1518
1519
|
# In the special case of nrandom, when serverside filtering is applied, it's "normal" for some input
|
|
1519
|
-
#
|
|
1520
|
+
# DIDs to be ignored as long as we got exactly nrandom file_items from the server.
|
|
1520
1521
|
for input_did in input_dids:
|
|
1521
1522
|
if not any([input_did == f['did'] or str(input_did) in f['parent_dids'] for f in file_items]):
|
|
1522
1523
|
logger(logging.ERROR, 'DID does not exist: %s' % input_did)
|
|
1523
|
-
# TODO: store
|
|
1524
|
+
# TODO: store DID directly as DIDType object
|
|
1524
1525
|
file_items.append({'did': str(input_did), 'adler32': None, 'md5': None, 'sources': [], 'parent_dids': set(), 'impl': impl or None})
|
|
1525
1526
|
|
|
1526
1527
|
# filtering out tape sources
|
|
@@ -1534,7 +1535,7 @@ class DownloadClient:
|
|
|
1534
1535
|
logger(logging.WARNING, 'The requested DID {} only has replicas on tape. Direct download from tape is prohibited. '
|
|
1535
1536
|
'Please request a transfer to a non-tape endpoint.'.format(file_item['did']))
|
|
1536
1537
|
|
|
1537
|
-
# Match the file
|
|
1538
|
+
# Match the file DID back to the DIDs which were provided to list_replicas.
|
|
1538
1539
|
# Later, this will allow to match the file back to input_items via did_to_input_items
|
|
1539
1540
|
for file_item in file_items:
|
|
1540
1541
|
file_did = DID(file_item['did'])
|
|
@@ -1628,7 +1629,7 @@ class DownloadClient:
|
|
|
1628
1629
|
|
|
1629
1630
|
all_dest_file_paths = set()
|
|
1630
1631
|
|
|
1631
|
-
# get replicas for every file of the given
|
|
1632
|
+
# get replicas for every file of the given DIDs
|
|
1632
1633
|
for file_item in file_items:
|
|
1633
1634
|
file_did = DID(file_item['did'])
|
|
1634
1635
|
input_items = list(itertools.chain.from_iterable(did_to_input_items.get(did, []) for did in file_item['input_dids']))
|
|
@@ -1676,7 +1677,7 @@ class DownloadClient:
|
|
|
1676
1677
|
file_item['dest_file_paths'] = list(dest_file_paths)
|
|
1677
1678
|
file_item['temp_file_path'] = '%s.part' % file_item['dest_file_paths'][0]
|
|
1678
1679
|
|
|
1679
|
-
# the file
|
|
1680
|
+
# the file DID str is not a unique key for this dict because multiple calls of list_replicas
|
|
1680
1681
|
# could result in the same DID multiple times. So we're using the id of the dictionary objects
|
|
1681
1682
|
fiid = id(file_item)
|
|
1682
1683
|
fiid_to_file_item[fiid] = file_item
|
rucio/client/exportclient.py
CHANGED
|
@@ -27,16 +27,57 @@ class ExportClient(BaseClient):
|
|
|
27
27
|
|
|
28
28
|
def export_data(self, distance: bool = True) -> dict[str, Any]:
|
|
29
29
|
"""
|
|
30
|
-
|
|
30
|
+
Retrieve a detailed snapshot of the current RSE configuration.
|
|
31
|
+
|
|
32
|
+
The exported information includes all registered RSEs with their settings and
|
|
33
|
+
attributes. When `distance` is `True`, the RSE distance matrix is included as well.
|
|
34
|
+
The snapshot is intended for use cases such as configuration back‑ups, migrations
|
|
35
|
+
between instances, and monitoring (e.g. generating monitoring dashboards).
|
|
36
|
+
|
|
31
37
|
Parameters
|
|
32
38
|
----------
|
|
33
|
-
distance
|
|
34
|
-
|
|
39
|
+
distance
|
|
40
|
+
If *True* (default), the server also returns the inter‑RSE distance matrix in
|
|
41
|
+
the payload.
|
|
42
|
+
|
|
43
|
+
_**Note:**_ Omitting the distance information can significantly reduce the
|
|
44
|
+
response size and improve transfer times.
|
|
35
45
|
|
|
36
46
|
Returns
|
|
37
47
|
-------
|
|
48
|
+
dict[str, Any]
|
|
49
|
+
A nested dictionary that mirrors the server‑side JSON structure.
|
|
50
|
+
The top‑level keys are:
|
|
51
|
+
|
|
52
|
+
**`rses`**:
|
|
53
|
+
Per‑RSE settings (name, deterministic flag, QoS class, supported protocol, etc.).
|
|
54
|
+
|
|
55
|
+
**`distances`**:
|
|
56
|
+
Pairwise RSE‑to‑RSE distance values (only present when `distance=True`).
|
|
57
|
+
|
|
58
|
+
Raises
|
|
59
|
+
------
|
|
60
|
+
RucioException
|
|
61
|
+
Raised if the HTTP status code is not *200 OK*.
|
|
62
|
+
|
|
63
|
+
Examples
|
|
64
|
+
--------
|
|
65
|
+
??? Example
|
|
66
|
+
|
|
67
|
+
Retrieve a full export of all configured RSEs, including their attributes and
|
|
68
|
+
inter-RSE distances:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from rucio.client.exportclient import ExportClient
|
|
72
|
+
|
|
73
|
+
export_client = ExportClient()
|
|
38
74
|
|
|
39
|
-
|
|
75
|
+
try:
|
|
76
|
+
rse_data = export_client.export_data() # distance=True by default
|
|
77
|
+
print(f"Full RSE properties: {rse_data}")
|
|
78
|
+
except Exception as err:
|
|
79
|
+
print(f"Action failed: {err}")
|
|
80
|
+
```
|
|
40
81
|
"""
|
|
41
82
|
payload = {'distance': distance}
|
|
42
83
|
path = '/'.join([self.EXPORT_BASEURL])
|
rucio/client/lockclient.py
CHANGED
|
@@ -41,9 +41,9 @@ class LockClient(BaseClient):
|
|
|
41
41
|
Parameters
|
|
42
42
|
----------
|
|
43
43
|
scope :
|
|
44
|
-
The scope of the
|
|
44
|
+
The scope of the DID of the locks to list.
|
|
45
45
|
name :
|
|
46
|
-
The name of the
|
|
46
|
+
The name of the DID of the locks to list.
|
|
47
47
|
|
|
48
48
|
"""
|
|
49
49
|
|
|
@@ -80,7 +80,7 @@ class LockClient(BaseClient):
|
|
|
80
80
|
list of dictionaries with lock info
|
|
81
81
|
"""
|
|
82
82
|
|
|
83
|
-
# convert
|
|
83
|
+
# convert DID list to list of dictionaries
|
|
84
84
|
if not all(did.get("type", "dataset") in ("dataset", "container") for did in dids):
|
|
85
85
|
raise ValueError("DID type can be either 'container' or 'dataset'")
|
|
86
86
|
|
rucio/client/pingclient.py
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from json import loads
|
|
16
|
+
from typing import Any
|
|
16
17
|
|
|
17
18
|
from requests.status_codes import codes
|
|
18
19
|
|
|
@@ -21,16 +22,46 @@ from rucio.common.utils import build_url
|
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
class PingClient(BaseClient):
|
|
24
|
-
|
|
25
25
|
"""Ping client class"""
|
|
26
26
|
|
|
27
|
-
def ping(self):
|
|
27
|
+
def ping(self) -> dict[str, Any]:
|
|
28
28
|
"""
|
|
29
|
-
|
|
29
|
+
This is a light‑weight “are you alive?” call (*ping* request) to the configured Rucio.
|
|
30
|
+
|
|
31
|
+
A quick way to verify (without any required authentication):
|
|
32
|
+
|
|
33
|
+
- Network connectivity between the client and the server.
|
|
34
|
+
|
|
35
|
+
- Whether the server process is running and able to respond.
|
|
36
|
+
|
|
37
|
+
- The server’s build / version.
|
|
30
38
|
|
|
31
39
|
Returns
|
|
40
|
+
-------
|
|
41
|
+
dict[str, Any]
|
|
42
|
+
A dictionary with a single key: the server version (e.g. {'version': '37.0.0'})
|
|
43
|
+
|
|
44
|
+
Raises
|
|
45
|
+
------
|
|
46
|
+
rucio.common.exception.RucioException
|
|
47
|
+
If the HTTP status code is not *200 OK*.
|
|
48
|
+
|
|
49
|
+
Examples
|
|
32
50
|
--------
|
|
33
|
-
|
|
51
|
+
??? Example
|
|
52
|
+
|
|
53
|
+
Basic connectivity check:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from rucio.client.pingclient import PingClient
|
|
57
|
+
ping_client = PingClient()
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
info = ping_client.ping()
|
|
61
|
+
print(f"Connected to Rucio {info['version']}")
|
|
62
|
+
except Exception as err:
|
|
63
|
+
print(f"Ping failed: {err}")
|
|
64
|
+
```
|
|
34
65
|
"""
|
|
35
66
|
|
|
36
67
|
headers = None
|
rucio/client/replicaclient.py
CHANGED
|
@@ -416,7 +416,7 @@ class ReplicaClient(BaseClient):
|
|
|
416
416
|
|
|
417
417
|
def list_dataset_replicas(self, scope, name, deep=False):
|
|
418
418
|
"""
|
|
419
|
-
List dataset replicas for a
|
|
419
|
+
List dataset replicas for a DID (scope:name).
|
|
420
420
|
|
|
421
421
|
Parameters
|
|
422
422
|
----------
|
|
@@ -446,7 +446,7 @@ class ReplicaClient(BaseClient):
|
|
|
446
446
|
|
|
447
447
|
def list_dataset_replicas_bulk(self, dids):
|
|
448
448
|
"""
|
|
449
|
-
List dataset replicas for a
|
|
449
|
+
List dataset replicas for a DID (scope:name).
|
|
450
450
|
|
|
451
451
|
Parameters
|
|
452
452
|
----------
|