rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio_clients-35.8.0.data/data/requirements.client.txt +15 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py +2 -5
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio +87 -85
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/METADATA +13 -13
- rucio_clients-35.8.0.dist-info/RECORD +88 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/WHEEL +1 -1
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio_clients-32.8.6.data/data/requirements.txt +0 -55
- rucio_clients-32.8.6.dist-info/RECORD +0 -88
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.template +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -20,7 +19,6 @@ import itertools
|
|
|
20
19
|
import logging
|
|
21
20
|
import math
|
|
22
21
|
import os
|
|
23
|
-
import re
|
|
24
22
|
import signal
|
|
25
23
|
import subprocess
|
|
26
24
|
import sys
|
|
@@ -28,28 +26,41 @@ import time
|
|
|
28
26
|
import traceback
|
|
29
27
|
import unittest
|
|
30
28
|
import uuid
|
|
29
|
+
from configparser import NoOptionError, NoSectionError
|
|
31
30
|
from copy import deepcopy
|
|
32
31
|
from datetime import datetime
|
|
33
32
|
from functools import wraps
|
|
33
|
+
from typing import Optional
|
|
34
34
|
|
|
35
|
-
from configparser import NoOptionError, NoSectionError
|
|
36
35
|
from tabulate import tabulate
|
|
37
36
|
|
|
38
37
|
# rucio module has the same name as this executable module, so this rule fails. pylint: disable=no-name-in-module
|
|
39
38
|
from rucio import version
|
|
40
39
|
from rucio.client import Client
|
|
41
40
|
from rucio.common.config import config_get, config_get_float
|
|
42
|
-
from rucio.common.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
from rucio.common.constants import ReplicaState
|
|
42
|
+
from rucio.common.exception import (
|
|
43
|
+
AccessDenied,
|
|
44
|
+
CannotAuthenticate,
|
|
45
|
+
DataIdentifierAlreadyExists,
|
|
46
|
+
DataIdentifierNotFound,
|
|
47
|
+
DIDFilterSyntaxError,
|
|
48
|
+
DuplicateContent,
|
|
49
|
+
DuplicateCriteriaInDIDFilter,
|
|
50
|
+
DuplicateRule,
|
|
51
|
+
InputValidationError,
|
|
52
|
+
InvalidObject,
|
|
53
|
+
InvalidRSEExpression,
|
|
54
|
+
InvalidType,
|
|
55
|
+
MissingDependency,
|
|
56
|
+
RSENotFound,
|
|
57
|
+
RucioException,
|
|
58
|
+
RuleNotFound,
|
|
59
|
+
UnsupportedOperation,
|
|
60
|
+
)
|
|
47
61
|
from rucio.common.extra import import_extras
|
|
48
62
|
from rucio.common.test_rucio_server import TestRucioServer
|
|
49
|
-
from rucio.common.utils import
|
|
50
|
-
parse_did_filter_from_string_fe, extract_scope, setup_logger, StoreAndDeprecateWarningAction
|
|
51
|
-
from rucio.common.constants import ReplicaState
|
|
52
|
-
from rucio.rse.protocols.protocol import RSEProtocol
|
|
63
|
+
from rucio.common.utils import Color, StoreAndDeprecateWarningAction, chunks, detect_client_location, extract_scope, parse_did_filter_from_string, parse_did_filter_from_string_fe, setup_logger, sizefmt
|
|
53
64
|
|
|
54
65
|
EXTRA_MODULES = import_extras(['argcomplete'])
|
|
55
66
|
|
|
@@ -156,7 +167,7 @@ def exception_handler(function):
|
|
|
156
167
|
used_account = '%s (from rucio.cfg)' % config_get('client', 'account')
|
|
157
168
|
except:
|
|
158
169
|
pass
|
|
159
|
-
try: # are we
|
|
170
|
+
try: # are we overridden by the environment?
|
|
160
171
|
used_account = '%s (from RUCIO_ACCOUNT)' % os.environ['RUCIO_ACCOUNT']
|
|
161
172
|
except:
|
|
162
173
|
pass
|
|
@@ -224,6 +235,8 @@ def get_client(args):
|
|
|
224
235
|
'oidc_issuer': args.oidc_issuer,
|
|
225
236
|
'oidc_username': args.oidc_username,
|
|
226
237
|
'oidc_password': args.oidc_password}
|
|
238
|
+
elif auth_type == "x509":
|
|
239
|
+
creds = {'client_cert': args.certificate, "client_key": args.client_key}
|
|
227
240
|
else:
|
|
228
241
|
creds = None
|
|
229
242
|
|
|
@@ -399,7 +412,7 @@ def list_file_replicas(args):
|
|
|
399
412
|
pfn_dir, dst_dir = args.link.split(':')
|
|
400
413
|
if args.rses:
|
|
401
414
|
for replica, rse in itertools.product(replicas, rses):
|
|
402
|
-
if
|
|
415
|
+
if replica['rses'].get(rse):
|
|
403
416
|
for pfn in replica['rses'][rse]:
|
|
404
417
|
os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
|
|
405
418
|
else:
|
|
@@ -410,16 +423,17 @@ def list_file_replicas(args):
|
|
|
410
423
|
os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
|
|
411
424
|
elif args.pfns:
|
|
412
425
|
if args.rses:
|
|
413
|
-
for replica
|
|
414
|
-
|
|
415
|
-
|
|
426
|
+
for replica in replicas:
|
|
427
|
+
for pfn in replica['pfns']:
|
|
428
|
+
rse = replica['pfns'][pfn]['rse']
|
|
429
|
+
if replica['rses'].get(rse):
|
|
416
430
|
print(pfn)
|
|
417
431
|
else:
|
|
418
432
|
for replica in replicas:
|
|
419
|
-
for
|
|
433
|
+
for pfn in replica['pfns']:
|
|
434
|
+
rse = replica['pfns'][pfn]['rse']
|
|
420
435
|
if replica['rses'][rse]:
|
|
421
|
-
|
|
422
|
-
print(pfn)
|
|
436
|
+
print(pfn)
|
|
423
437
|
else:
|
|
424
438
|
if args.all_states:
|
|
425
439
|
header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', '(STATE) RSE: REPLICA']
|
|
@@ -427,18 +441,18 @@ def list_file_replicas(args):
|
|
|
427
441
|
header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']
|
|
428
442
|
for replica in replicas:
|
|
429
443
|
if 'bytes' in replica:
|
|
430
|
-
for
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
444
|
+
for pfn in replica['pfns']:
|
|
445
|
+
rse = replica['pfns'][pfn]['rse']
|
|
446
|
+
if args.all_states:
|
|
447
|
+
rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
|
|
448
|
+
else:
|
|
449
|
+
rse_string = '{0}: {1}'.format(rse, pfn)
|
|
450
|
+
if args.rses:
|
|
451
|
+
for selected_rse in rses:
|
|
452
|
+
if rse == selected_rse:
|
|
453
|
+
table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
|
|
454
|
+
else:
|
|
455
|
+
table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
|
|
442
456
|
print(tabulate(table, tablefmt=tablefmt, headers=header, disable_numparse=True))
|
|
443
457
|
return SUCCESS
|
|
444
458
|
|
|
@@ -490,7 +504,7 @@ def attach(args):
|
|
|
490
504
|
try:
|
|
491
505
|
f = open(dids[0], 'r')
|
|
492
506
|
dids = [did.rstrip() for did in f.readlines()]
|
|
493
|
-
except
|
|
507
|
+
except OSError:
|
|
494
508
|
logger.error("Can't open file '" + dids[0] + "'.")
|
|
495
509
|
return FAILURE
|
|
496
510
|
|
|
@@ -868,44 +882,6 @@ def erase(args):
|
|
|
868
882
|
return SUCCESS
|
|
869
883
|
|
|
870
884
|
|
|
871
|
-
@exception_handler
|
|
872
|
-
def list_impls(args):
|
|
873
|
-
"""
|
|
874
|
-
%(prog)s list-impls
|
|
875
|
-
|
|
876
|
-
List protocol implementations.
|
|
877
|
-
"""
|
|
878
|
-
|
|
879
|
-
PROTOCOL_DIRECTORY = '/opt/rucio/lib/rucio/rse/protocols'
|
|
880
|
-
|
|
881
|
-
for filename in os.listdir(PROTOCOL_DIRECTORY):
|
|
882
|
-
if os.path.isdir(os.path.join(PROTOCOL_DIRECTORY, filename)) or filename in ["__init__.py", "cache.py", "http_cache.py", "dummy.py", "protocol.py"]:
|
|
883
|
-
continue
|
|
884
|
-
else:
|
|
885
|
-
filename = re.sub(r".py", r"", filename)
|
|
886
|
-
__import__("rucio.rse.protocols", fromlist=[filename])
|
|
887
|
-
|
|
888
|
-
impls = []
|
|
889
|
-
|
|
890
|
-
def get_subclasses(cls):
|
|
891
|
-
for subclass in cls.__subclasses__():
|
|
892
|
-
if (str(subclass)[28:-2]).endswith('Default'):
|
|
893
|
-
class_name = str(subclass)[28:-10]
|
|
894
|
-
else:
|
|
895
|
-
class_name = str(subclass)[28:-2]
|
|
896
|
-
impls.append([class_name, subclass.__doc__])
|
|
897
|
-
get_subclasses(subclass)
|
|
898
|
-
|
|
899
|
-
get_subclasses(RSEProtocol)
|
|
900
|
-
impls = sorted(impls)
|
|
901
|
-
table = []
|
|
902
|
-
for impl in impls:
|
|
903
|
-
table.append([impl[0], impl[1]])
|
|
904
|
-
|
|
905
|
-
print(tabulate(table, tablefmt=tablefmt, headers=['impl', 'DESCRIPTION']))
|
|
906
|
-
return SUCCESS
|
|
907
|
-
|
|
908
|
-
|
|
909
885
|
@exception_handler
|
|
910
886
|
def upload(args):
|
|
911
887
|
"""
|
|
@@ -1105,6 +1081,24 @@ def download(args):
|
|
|
1105
1081
|
logger.debug(args.dids)
|
|
1106
1082
|
item_defaults['pfn'] = args.pfn
|
|
1107
1083
|
item_defaults['did'] = did_str
|
|
1084
|
+
if args.rses is None:
|
|
1085
|
+
logger.warning("No RSE was given, selecting one.")
|
|
1086
|
+
client = get_client(args)
|
|
1087
|
+
replicas = client.list_replicas(
|
|
1088
|
+
[{"scope": did_str.split(':')[0], "name": did_str.split(':')[-1]}],
|
|
1089
|
+
schemes=args.protocol,
|
|
1090
|
+
ignore_availability=False,
|
|
1091
|
+
client_location=detect_client_location(),
|
|
1092
|
+
resolve_archives=not args.no_resolve_archives
|
|
1093
|
+
)
|
|
1094
|
+
|
|
1095
|
+
download_rse = _get_rse_for_pfn(replicas, args.pfn)
|
|
1096
|
+
if download_rse is None:
|
|
1097
|
+
logger.error("Could not find RSE for pfn %s", args.pfn)
|
|
1098
|
+
return FAILURE
|
|
1099
|
+
else:
|
|
1100
|
+
item_defaults['rse'] = download_rse
|
|
1101
|
+
|
|
1108
1102
|
result = download_client.download_pfns([item_defaults], 1, trace_pattern, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
|
|
1109
1103
|
|
|
1110
1104
|
if not result:
|
|
@@ -1146,6 +1140,21 @@ def download(args):
|
|
|
1146
1140
|
return SUCCESS
|
|
1147
1141
|
|
|
1148
1142
|
|
|
1143
|
+
def _get_rse_for_pfn(replicas, pfn) -> Optional[str]:
|
|
1144
|
+
# Check each rse in the replica list for the pfn. If no pfn is found, returns None.
|
|
1145
|
+
# If it is found, stop the generator and return the item.
|
|
1146
|
+
for replica in replicas:
|
|
1147
|
+
try:
|
|
1148
|
+
download_rse = next(
|
|
1149
|
+
rse for rse in replica['rses']
|
|
1150
|
+
if pfn in replica['rses'][rse]
|
|
1151
|
+
)
|
|
1152
|
+
except StopIteration:
|
|
1153
|
+
continue
|
|
1154
|
+
else:
|
|
1155
|
+
return download_rse
|
|
1156
|
+
|
|
1157
|
+
|
|
1149
1158
|
@exception_handler
|
|
1150
1159
|
def get_metadata(args):
|
|
1151
1160
|
"""
|
|
@@ -1179,7 +1188,7 @@ def set_metadata(args):
|
|
|
1179
1188
|
client = get_client(args)
|
|
1180
1189
|
value = args.value
|
|
1181
1190
|
if args.key == 'lifetime':
|
|
1182
|
-
value = float(args.value)
|
|
1191
|
+
value = None if args.value.lower() == 'none' else float(args.value)
|
|
1183
1192
|
scope, name = get_scope(args.did, client)
|
|
1184
1193
|
client.set_metadata(scope=scope, name=name, key=args.key, value=value)
|
|
1185
1194
|
return SUCCESS
|
|
@@ -1374,9 +1383,6 @@ def info_rule(args):
|
|
|
1374
1383
|
|
|
1375
1384
|
Retrieve information about a rule.
|
|
1376
1385
|
"""
|
|
1377
|
-
if args.estimate_ttc:
|
|
1378
|
-
logger.error('"--estimate-ttc" is deprecated!')
|
|
1379
|
-
return FAILURE
|
|
1380
1386
|
client = get_client(args)
|
|
1381
1387
|
if args.examine:
|
|
1382
1388
|
analysis = client.examine_replication_rule(rule_id=args.rule_id)
|
|
@@ -1581,7 +1587,7 @@ def list_rse_attributes(args):
|
|
|
1581
1587
|
"""
|
|
1582
1588
|
client = get_client(args)
|
|
1583
1589
|
attributes = client.list_rse_attributes(rse=args.rse)
|
|
1584
|
-
table = [(k + ':', str(v)) for (k, v) in sorted(attributes.items())] # columns
|
|
1590
|
+
table = [(k + ':', str(v)) for (k, v) in sorted(attributes.items())] # columns have mixed datatypes
|
|
1585
1591
|
print(tabulate(table, tablefmt='plain', disable_numparse=True)) # disabling number parsing
|
|
1586
1592
|
return SUCCESS
|
|
1587
1593
|
|
|
@@ -1885,7 +1891,8 @@ def get_parser():
|
|
|
1885
1891
|
+ 'to the keys configured in the /etc/idpsecrets.json auth server configuration file.') # NOQA: W503
|
|
1886
1892
|
|
|
1887
1893
|
# Options for the x509 auth_strategy
|
|
1888
|
-
oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file.')
|
|
1894
|
+
oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file for x509 Authentication.')
|
|
1895
|
+
oparser.add_argument('--client_key', dest='client_key', default=None, help='Client key for x509 Authentication.')
|
|
1889
1896
|
oparser.add_argument('--ca-certificate', dest='ca_certificate', default=None, help='CA certificate to verify peer against (SSL).')
|
|
1890
1897
|
|
|
1891
1898
|
# Ping command
|
|
@@ -1957,7 +1964,7 @@ To list the missing replica of a dataset of a given RSE-expression::
|
|
|
1957
1964
|
list_file_replicas_parser.add_argument('--no-resolve-archives', dest='no_resolve_archives', default=False, action='store_true', help='Do not resolve archives which may contain the files.', required=False)
|
|
1958
1965
|
list_file_replicas_parser.add_argument('--sort', dest='sort', default=None, action='store', help='Replica sort algorithm. Available options: geoip (default), random', required=False)
|
|
1959
1966
|
list_file_replicas_parser.add_argument('--rses', dest='rses', default=None, action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
|
|
1960
|
-
can be found in ' + Color.BOLD + '
|
|
1967
|
+
can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
|
|
1961
1968
|
|
|
1962
1969
|
# The list-dataset-replicas command
|
|
1963
1970
|
list_dataset_replicas_parser = subparsers.add_parser('list-dataset-replicas', help='List the dataset replicas.',
|
|
@@ -2265,10 +2272,6 @@ You can filter by key/value, e.g.::
|
|
|
2265
2272
|
list_content_history_parser.set_defaults(function=list_content_history)
|
|
2266
2273
|
list_content_history_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
|
|
2267
2274
|
|
|
2268
|
-
# The list-impls command
|
|
2269
|
-
list_impls_parser = subparsers.add_parser('list-impls', help='List all supported protocol implementations.')
|
|
2270
|
-
list_impls_parser.set_defaults(function=list_impls)
|
|
2271
|
-
|
|
2272
2275
|
# The upload subparser
|
|
2273
2276
|
upload_parser = subparsers.add_parser('upload', help='Upload method.')
|
|
2274
2277
|
upload_parser.set_defaults(function=upload)
|
|
@@ -2396,7 +2399,6 @@ You can filter by key/value, e.g.::
|
|
|
2396
2399
|
info_rule_parser.set_defaults(function=info_rule)
|
|
2397
2400
|
info_rule_parser.add_argument(dest='rule_id', action='store', help='The rule ID')
|
|
2398
2401
|
info_rule_parser.add_argument('--examine', dest='examine', action='store_true', help='Detailed analysis of transfer errors')
|
|
2399
|
-
info_rule_parser.add_argument('--estimate-ttc', dest='estimate_ttc', action='store_true', help='Deprecated, specifying this will cause the program to fail.')
|
|
2400
2402
|
|
|
2401
2403
|
# The list_rules command
|
|
2402
2404
|
list_rules_parser = subparsers.add_parser('list-rules', help='List replication rules.', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
|
|
@@ -2464,13 +2466,13 @@ You can filter by account::
|
|
|
2464
2466
|
list_rses_parser = subparsers.add_parser('list-rses', help='Show the list of all the registered Rucio Storage Elements (RSEs).')
|
|
2465
2467
|
list_rses_parser.set_defaults(function=list_rses)
|
|
2466
2468
|
list_rses_parser.add_argument('--rses', dest='rses', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
|
|
2467
|
-
can be found in ' + Color.BOLD + '
|
|
2469
|
+
can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
|
|
2468
2470
|
|
|
2469
2471
|
# The list-suspicious-replicas command
|
|
2470
2472
|
list_suspicious_replicas_parser = subparsers.add_parser('list-suspicious-replicas', help='Show the list of all replicas marked "suspicious".')
|
|
2471
2473
|
list_suspicious_replicas_parser.set_defaults(function=list_suspicious_replicas)
|
|
2472
2474
|
list_suspicious_replicas_parser.add_argument('--expression', dest='rse_expression', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
|
|
2473
|
-
can be found in ' + Color.BOLD + '
|
|
2475
|
+
can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
|
|
2474
2476
|
list_suspicious_replicas_parser.add_argument('--younger_than', '--younger-than', new_option_string='--younger-than', dest='younger_than', action=StoreAndDeprecateWarningAction, help='List files that have been marked suspicious since the date "younger_than", e.g. 2021-11-29T00:00:00.') # NOQA: E501
|
|
2475
2477
|
list_suspicious_replicas_parser.add_argument('--nattempts', dest='nattempts', action='store', help='Minimum number of failed attempts to access a suspicious file.')
|
|
2476
2478
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -25,25 +24,37 @@ import signal
|
|
|
25
24
|
import sys
|
|
26
25
|
import time
|
|
27
26
|
import traceback
|
|
28
|
-
from
|
|
27
|
+
from configparser import NoOptionError, NoSectionError
|
|
29
28
|
from functools import wraps
|
|
29
|
+
from textwrap import dedent
|
|
30
30
|
|
|
31
|
-
from configparser import NoOptionError, NoSectionError
|
|
32
31
|
from tabulate import tabulate
|
|
33
32
|
|
|
34
33
|
from rucio import version
|
|
35
34
|
from rucio.client import Client
|
|
36
35
|
from rucio.common.config import config_get
|
|
37
|
-
from rucio.common.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
from rucio.common.constants import RseAttr
|
|
37
|
+
from rucio.common.exception import (
|
|
38
|
+
AccessDenied,
|
|
39
|
+
AccountNotFound,
|
|
40
|
+
CannotAuthenticate,
|
|
41
|
+
ConfigNotFound,
|
|
42
|
+
DataIdentifierAlreadyExists,
|
|
43
|
+
DataIdentifierNotFound,
|
|
44
|
+
Duplicate,
|
|
45
|
+
DuplicateContent,
|
|
46
|
+
InputValidationError,
|
|
47
|
+
InvalidObject,
|
|
48
|
+
InvalidRSEExpression,
|
|
49
|
+
ReplicaIsLocked,
|
|
50
|
+
ReplicaNotFound,
|
|
51
|
+
RSENotFound,
|
|
52
|
+
RSEOperationNotSupported,
|
|
53
|
+
RuleNotFound,
|
|
54
|
+
ScopeNotFound,
|
|
55
|
+
)
|
|
43
56
|
from rucio.common.extra import import_extras
|
|
44
|
-
from rucio.common.utils import
|
|
45
|
-
render_json, parse_response, extract_scope, clean_surls,
|
|
46
|
-
StoreAndDeprecateWarningAction)
|
|
57
|
+
from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_pfns, construct_non_deterministic_pfn, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
|
|
47
58
|
from rucio.rse import rsemanager as rsemgr
|
|
48
59
|
|
|
49
60
|
EXTRA_MODULES = import_extras(['argcomplete'])
|
|
@@ -375,7 +386,7 @@ def set_limits(args):
|
|
|
375
386
|
try:
|
|
376
387
|
byte_limit = int(limit_input)
|
|
377
388
|
except ValueError:
|
|
378
|
-
logger.error('The limit could not be set. Either you misspelled infinity or your input could not be converted to integer or you used a wrong pattern. Please use a format like 10GB with B,KB,MB,GB,TB,PB as units (not case
|
|
389
|
+
logger.error('The limit could not be set. Either you misspelled infinity or your input could not be converted to integer or you used a wrong pattern. Please use a format like 10GB with B,KB,MB,GB,TB,PB as units (not case sensitive)')
|
|
379
390
|
return FAILURE
|
|
380
391
|
|
|
381
392
|
client.set_account_limit(account=args.account, rse=args.rse, bytes_=byte_limit, locality=locality)
|
|
@@ -1137,21 +1148,21 @@ def declare_bad_file_replicas(args):
|
|
|
1137
1148
|
list_bad_pfns = []
|
|
1138
1149
|
cnt += 1
|
|
1139
1150
|
previous_pattern = None
|
|
1140
|
-
for
|
|
1151
|
+
for pfn in clean_pfns(chunk):
|
|
1141
1152
|
unknown = True
|
|
1142
1153
|
if previous_pattern:
|
|
1143
|
-
if previous_pattern in
|
|
1144
|
-
list_bad_pfns.append(
|
|
1154
|
+
if previous_pattern in pfn:
|
|
1155
|
+
list_bad_pfns.append(pfn)
|
|
1145
1156
|
unknown = False
|
|
1146
1157
|
continue
|
|
1147
1158
|
for pattern in prot_dict:
|
|
1148
|
-
if pattern in
|
|
1159
|
+
if pattern in pfn:
|
|
1149
1160
|
previous_pattern = prot_dict[pattern]
|
|
1150
|
-
list_bad_pfns.append(
|
|
1161
|
+
list_bad_pfns.append(pfn)
|
|
1151
1162
|
unknown = False
|
|
1152
1163
|
break
|
|
1153
1164
|
if unknown:
|
|
1154
|
-
print('Cannot find any RSE associated to %s' %
|
|
1165
|
+
print('Cannot find any RSE associated to %s' % pfn)
|
|
1155
1166
|
client.add_bad_pfns(pfns=list_bad_pfns, reason=args.reason, state='BAD', expires_at=None)
|
|
1156
1167
|
ndeclared = len(list_bad_pfns)
|
|
1157
1168
|
tot_file_declared += ndeclared
|
|
@@ -1185,7 +1196,9 @@ def declare_temporary_unavailable_replicas(args):
|
|
|
1185
1196
|
else:
|
|
1186
1197
|
bad_files = args.listbadfiles
|
|
1187
1198
|
|
|
1188
|
-
|
|
1199
|
+
if args.duration is None:
|
|
1200
|
+
raise ValueError("Duration should have been set, something went wrong!")
|
|
1201
|
+
|
|
1189
1202
|
expiration_date = (datetime.datetime.utcnow() + datetime.timedelta(seconds=args.duration)).isoformat()
|
|
1190
1203
|
|
|
1191
1204
|
chunk_size = 10000
|
|
@@ -1234,13 +1247,13 @@ def list_pfns(args):
|
|
|
1234
1247
|
if not rse_info['deterministic']:
|
|
1235
1248
|
logger.warning('This is a non-deterministic site, so the real PFN might be different from the on suggested')
|
|
1236
1249
|
rse_attr = client.list_rse_attributes(rse)
|
|
1237
|
-
naming_convention = rse_attr.get(
|
|
1250
|
+
naming_convention = rse_attr.get(RseAttr.NAMING_CONVENTION, None)
|
|
1238
1251
|
parents = [did for did in client.list_parent_dids(scope, name)]
|
|
1239
1252
|
if len(parents) > 1:
|
|
1240
1253
|
logger.warning('The file has multiple parents')
|
|
1241
1254
|
for did in parents:
|
|
1242
1255
|
if did['type'] == 'DATASET':
|
|
1243
|
-
path =
|
|
1256
|
+
path = construct_non_deterministic_pfn(did['name'], scope, name, naming_convention=naming_convention)
|
|
1244
1257
|
pfn = ''.join([proto.attributes['scheme'],
|
|
1245
1258
|
'://',
|
|
1246
1259
|
proto.attributes['hostname'],
|
|
@@ -1277,7 +1290,7 @@ def import_data(args):
|
|
|
1277
1290
|
print('There was problem with decoding your file.')
|
|
1278
1291
|
print(error)
|
|
1279
1292
|
return FAILURE
|
|
1280
|
-
except
|
|
1293
|
+
except OSError as error:
|
|
1281
1294
|
print('There was a problem with reading your file.')
|
|
1282
1295
|
print(error)
|
|
1283
1296
|
return FAILURE
|
|
@@ -1309,7 +1322,7 @@ def export_data(args):
|
|
|
1309
1322
|
print('File successfully written.')
|
|
1310
1323
|
print('Data successfully exported to %s' % args.file_path)
|
|
1311
1324
|
return SUCCESS
|
|
1312
|
-
except
|
|
1325
|
+
except OSError as error:
|
|
1313
1326
|
print('There was a problem with reading your file.')
|
|
1314
1327
|
print(error)
|
|
1315
1328
|
return FAILURE
|
|
@@ -1373,7 +1386,7 @@ def get_parser():
|
|
|
1373
1386
|
oparser.add_argument('--oidc-refresh-lifetime', dest='oidc_refresh_lifetime', default=None, help='Max lifetime in hours for this an access token will be refreshed by asynchronous Rucio daemon. '
|
|
1374
1387
|
+ 'If not specified, refresh will be stopped after 4 days. This option is effective only if --oidc-scope includes offline_access scope for a refresh token to be granted to Rucio.') # NOQA: W503
|
|
1375
1388
|
oparser.add_argument('--oidc-issuer', dest='oidc_issuer', default=None,
|
|
1376
|
-
help='Defines which Identity Provider is
|
|
1389
|
+
help='Defines which Identity Provider is going to be used. The issuer string must correspond '
|
|
1377
1390
|
+ 'to the keys configured in the /etc/idpsecrets.json auth server configuration file.') # NOQA: W503
|
|
1378
1391
|
|
|
1379
1392
|
# Options for the x509 auth_strategy
|
|
@@ -1422,7 +1435,7 @@ def get_parser():
|
|
|
1422
1435
|
'"""""""""""""\n'
|
|
1423
1436
|
'::\n'
|
|
1424
1437
|
'\n'
|
|
1425
|
-
' $ rucio-admin account list --type \
|
|
1438
|
+
' $ rucio-admin account list --type USER\n'
|
|
1426
1439
|
'\n')
|
|
1427
1440
|
list_account_parser.add_argument('--type', dest='account_type', action='store', help='Account Type (USER, GROUP, SERVICE)')
|
|
1428
1441
|
list_account_parser.add_argument('--id', dest='identity', action='store', help='Identity (e.g. DN)')
|
|
@@ -1698,7 +1711,7 @@ def get_parser():
|
|
|
1698
1711
|
' $ rucio-admin identity delete --account jdoe --type X509 --id \'CN=Joe Doe,CN=707658,CN=jdoe,OU=Users,OU=Organic Units,DC=cern,DC=ch\'\n'
|
|
1699
1712
|
' Deleted identity: CN=Joe Doe,CN=707658,CN=jdoe,OU=Users,OU=Organic Units,DC=cern,DC=ch\n'
|
|
1700
1713
|
'\n'
|
|
1701
|
-
'Note: if the identity was
|
|
1714
|
+
'Note: if the identity was accidentally deleted, use add option.\n'
|
|
1702
1715
|
'\n')
|
|
1703
1716
|
identity_delete_parser.set_defaults(which='identity_delete')
|
|
1704
1717
|
identity_delete_parser.add_argument('--account', dest='account', action='store', help='Account name', required=True)
|
|
@@ -1946,7 +1959,7 @@ def get_parser():
|
|
|
1946
1959
|
' $ rucio-admin rse add-protocol --hostname jdoes.test.org --scheme gsiftp --prefix \'/atlasdatadisk/rucio/\' --port 8443 JDOE_DATADISK\n'
|
|
1947
1960
|
'\n'
|
|
1948
1961
|
'Note: no printed stdout.\n'
|
|
1949
|
-
'Note: examples of optional
|
|
1962
|
+
'Note: examples of optional parameters::\n'
|
|
1950
1963
|
'\n'
|
|
1951
1964
|
' --space-token DATADISK\n'
|
|
1952
1965
|
' --web-service-path \'/srm/managerv2?SFN=\'\n'
|
|
@@ -2100,7 +2113,7 @@ def get_parser():
|
|
|
2100
2113
|
|
|
2101
2114
|
# The config subparser
|
|
2102
2115
|
config_parser = subparsers.add_parser('config',
|
|
2103
|
-
help='Configuration methods. The global configuration of data
|
|
2116
|
+
help='Configuration methods. The global configuration of data management system can by modified.',
|
|
2104
2117
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
2105
2118
|
epilog='''e.g. quotas, daemons, rses''')
|
|
2106
2119
|
config_subparser = config_parser.add_subparsers(dest='config_subcommand', **required_arg)
|
|
@@ -2301,9 +2314,9 @@ def get_parser():
|
|
|
2301
2314
|
declare_bad_file_replicas_parser.add_argument('--inputfile', dest='inputfile', nargs='?', action='store', help='File containing list of bad items')
|
|
2302
2315
|
declare_bad_file_replicas_parser.add_argument('--allow-collection', dest='allow_collection', action='store_true', help='Allow passing a collection DID as bad item')
|
|
2303
2316
|
|
|
2304
|
-
declare_bad_file_replicas_parser.add_argument('--lfns', dest='lfns', nargs='?', action='store', help='File
|
|
2305
|
-
declare_bad_file_replicas_parser.add_argument('--scope', dest='scope', nargs='?', action='store', help='Common scope for bad replicas
|
|
2306
|
-
declare_bad_file_replicas_parser.add_argument('--rse', dest='rse', nargs='?', action='store', help='Common RSE for bad replicas
|
|
2317
|
+
declare_bad_file_replicas_parser.add_argument('--lfns', dest='lfns', nargs='?', action='store', help='File containing list of LFNs for bad replicas. Requires --rse and --scope')
|
|
2318
|
+
declare_bad_file_replicas_parser.add_argument('--scope', dest='scope', nargs='?', action='store', help='Common scope for bad replicas specified with LFN list, ignored without --lfns')
|
|
2319
|
+
declare_bad_file_replicas_parser.add_argument('--rse', dest='rse', nargs='?', action='store', help='Common RSE for bad replicas specified with LFN list, ignored without --lfns')
|
|
2307
2320
|
|
|
2308
2321
|
# The declare-temporary-unavailable command
|
|
2309
2322
|
declare_temporary_unavailable_replicas_parser = rep_subparser.add_parser('declare-temporary-unavailable',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rucio-clients
|
|
3
|
-
Version:
|
|
3
|
+
Version: 35.8.0
|
|
4
4
|
Summary: Rucio Client Lite Package
|
|
5
5
|
Home-page: https://rucio.cern.ch/
|
|
6
6
|
Author: Rucio
|
|
@@ -19,25 +19,25 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
19
19
|
Requires-Python: >=3.9, <4
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: AUTHORS.rst
|
|
22
|
-
Requires-Dist: requests
|
|
23
|
-
Requires-Dist: urllib3
|
|
24
|
-
Requires-Dist: dogpile
|
|
25
|
-
Requires-Dist: tabulate
|
|
26
|
-
Requires-Dist: jsonschema
|
|
22
|
+
Requires-Dist: requests>=2.32.2
|
|
23
|
+
Requires-Dist: urllib3>=1.26.18
|
|
24
|
+
Requires-Dist: dogpile-cache>=1.2.2
|
|
25
|
+
Requires-Dist: tabulate>=0.9.0
|
|
26
|
+
Requires-Dist: jsonschema>=4.20.0
|
|
27
27
|
Provides-Extra: ssh
|
|
28
|
-
Requires-Dist: paramiko
|
|
28
|
+
Requires-Dist: paramiko>=3.4.1; extra == "ssh"
|
|
29
29
|
Provides-Extra: kerberos
|
|
30
|
-
Requires-Dist: kerberos
|
|
31
|
-
Requires-Dist: pykerberos
|
|
30
|
+
Requires-Dist: kerberos>=1.3.1; extra == "kerberos"
|
|
31
|
+
Requires-Dist: pykerberos>=1.2.4; extra == "kerberos"
|
|
32
32
|
Requires-Dist: requests-kerberos>=0.14.0; extra == "kerberos"
|
|
33
33
|
Provides-Extra: swift
|
|
34
|
-
Requires-Dist: python-swiftclient
|
|
34
|
+
Requires-Dist: python-swiftclient>=4.4.0; extra == "swift"
|
|
35
35
|
Provides-Extra: argcomplete
|
|
36
|
-
Requires-Dist: argcomplete
|
|
36
|
+
Requires-Dist: argcomplete>=3.1.6; extra == "argcomplete"
|
|
37
37
|
Provides-Extra: sftp
|
|
38
|
-
Requires-Dist: paramiko
|
|
38
|
+
Requires-Dist: paramiko>=3.4.1; extra == "sftp"
|
|
39
39
|
Provides-Extra: dumper
|
|
40
|
-
Requires-Dist: python-magic
|
|
40
|
+
Requires-Dist: python-magic>=0.4.27; extra == "dumper"
|
|
41
41
|
Dynamic: author
|
|
42
42
|
Dynamic: author-email
|
|
43
43
|
Dynamic: classifier
|