rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio-clients might be problematic. Click here for more details.

Files changed (92) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/exception.py +281 -222
  35. rucio/common/extra.py +0 -1
  36. rucio/common/logging.py +54 -38
  37. rucio/common/pcache.py +122 -101
  38. rucio/common/plugins.py +153 -0
  39. rucio/common/policy.py +4 -4
  40. rucio/common/schema/__init__.py +17 -10
  41. rucio/common/schema/atlas.py +7 -5
  42. rucio/common/schema/belleii.py +7 -5
  43. rucio/common/schema/domatpc.py +7 -5
  44. rucio/common/schema/escape.py +7 -5
  45. rucio/common/schema/generic.py +8 -6
  46. rucio/common/schema/generic_multi_vo.py +7 -5
  47. rucio/common/schema/icecube.py +7 -5
  48. rucio/common/stomp_utils.py +0 -1
  49. rucio/common/stopwatch.py +0 -1
  50. rucio/common/test_rucio_server.py +2 -2
  51. rucio/common/types.py +262 -17
  52. rucio/common/utils.py +743 -451
  53. rucio/rse/__init__.py +3 -4
  54. rucio/rse/protocols/__init__.py +0 -1
  55. rucio/rse/protocols/bittorrent.py +184 -0
  56. rucio/rse/protocols/cache.py +1 -2
  57. rucio/rse/protocols/dummy.py +1 -2
  58. rucio/rse/protocols/gfal.py +12 -10
  59. rucio/rse/protocols/globus.py +7 -7
  60. rucio/rse/protocols/gsiftp.py +2 -3
  61. rucio/rse/protocols/http_cache.py +1 -2
  62. rucio/rse/protocols/mock.py +1 -2
  63. rucio/rse/protocols/ngarc.py +1 -2
  64. rucio/rse/protocols/posix.py +12 -13
  65. rucio/rse/protocols/protocol.py +116 -52
  66. rucio/rse/protocols/rclone.py +6 -7
  67. rucio/rse/protocols/rfio.py +4 -5
  68. rucio/rse/protocols/srm.py +9 -10
  69. rucio/rse/protocols/ssh.py +8 -9
  70. rucio/rse/protocols/storm.py +2 -3
  71. rucio/rse/protocols/webdav.py +17 -14
  72. rucio/rse/protocols/xrootd.py +23 -17
  73. rucio/rse/rsemanager.py +19 -7
  74. rucio/vcsversion.py +4 -4
  75. rucio/version.py +5 -13
  76. rucio_clients-35.8.0.data/data/requirements.client.txt +15 -0
  77. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py +2 -5
  78. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio +87 -85
  79. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio-admin +45 -32
  80. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/METADATA +13 -13
  81. rucio_clients-35.8.0.dist-info/RECORD +88 -0
  82. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/WHEEL +1 -1
  83. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  84. rucio/common/schema/cms.py +0 -478
  85. rucio/common/schema/lsst.py +0 -423
  86. rucio_clients-32.8.6.data/data/requirements.txt +0 -55
  87. rucio_clients-32.8.6.dist-info/RECORD +0 -88
  88. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rse-accounts.cfg.template +0 -0
  89. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
  90. {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.template +0 -0
  91. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/LICENSE +0 -0
  92. {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,7 +19,6 @@ import itertools
20
19
  import logging
21
20
  import math
22
21
  import os
23
- import re
24
22
  import signal
25
23
  import subprocess
26
24
  import sys
@@ -28,28 +26,41 @@ import time
28
26
  import traceback
29
27
  import unittest
30
28
  import uuid
29
+ from configparser import NoOptionError, NoSectionError
31
30
  from copy import deepcopy
32
31
  from datetime import datetime
33
32
  from functools import wraps
33
+ from typing import Optional
34
34
 
35
- from configparser import NoOptionError, NoSectionError
36
35
  from tabulate import tabulate
37
36
 
38
37
  # rucio module has the same name as this executable module, so this rule fails. pylint: disable=no-name-in-module
39
38
  from rucio import version
40
39
  from rucio.client import Client
41
40
  from rucio.common.config import config_get, config_get_float
42
- from rucio.common.exception import (DataIdentifierAlreadyExists, AccessDenied, DataIdentifierNotFound, InvalidObject,
43
- RSENotFound, InvalidRSEExpression, InputValidationError, DuplicateContent,
44
- RuleNotFound, CannotAuthenticate, MissingDependency, UnsupportedOperation,
45
- RucioException, DuplicateRule, InvalidType, DuplicateCriteriaInDIDFilter,
46
- DIDFilterSyntaxError)
41
+ from rucio.common.constants import ReplicaState
42
+ from rucio.common.exception import (
43
+ AccessDenied,
44
+ CannotAuthenticate,
45
+ DataIdentifierAlreadyExists,
46
+ DataIdentifierNotFound,
47
+ DIDFilterSyntaxError,
48
+ DuplicateContent,
49
+ DuplicateCriteriaInDIDFilter,
50
+ DuplicateRule,
51
+ InputValidationError,
52
+ InvalidObject,
53
+ InvalidRSEExpression,
54
+ InvalidType,
55
+ MissingDependency,
56
+ RSENotFound,
57
+ RucioException,
58
+ RuleNotFound,
59
+ UnsupportedOperation,
60
+ )
47
61
  from rucio.common.extra import import_extras
48
62
  from rucio.common.test_rucio_server import TestRucioServer
49
- from rucio.common.utils import sizefmt, Color, detect_client_location, chunks, parse_did_filter_from_string, \
50
- parse_did_filter_from_string_fe, extract_scope, setup_logger, StoreAndDeprecateWarningAction
51
- from rucio.common.constants import ReplicaState
52
- from rucio.rse.protocols.protocol import RSEProtocol
63
+ from rucio.common.utils import Color, StoreAndDeprecateWarningAction, chunks, detect_client_location, extract_scope, parse_did_filter_from_string, parse_did_filter_from_string_fe, setup_logger, sizefmt
53
64
 
54
65
  EXTRA_MODULES = import_extras(['argcomplete'])
55
66
 
@@ -156,7 +167,7 @@ def exception_handler(function):
156
167
  used_account = '%s (from rucio.cfg)' % config_get('client', 'account')
157
168
  except:
158
169
  pass
159
- try: # are we overriden by the environment?
170
+ try: # are we overridden by the environment?
160
171
  used_account = '%s (from RUCIO_ACCOUNT)' % os.environ['RUCIO_ACCOUNT']
161
172
  except:
162
173
  pass
@@ -224,6 +235,8 @@ def get_client(args):
224
235
  'oidc_issuer': args.oidc_issuer,
225
236
  'oidc_username': args.oidc_username,
226
237
  'oidc_password': args.oidc_password}
238
+ elif auth_type == "x509":
239
+ creds = {'client_cert': args.certificate, "client_key": args.client_key}
227
240
  else:
228
241
  creds = None
229
242
 
@@ -399,7 +412,7 @@ def list_file_replicas(args):
399
412
  pfn_dir, dst_dir = args.link.split(':')
400
413
  if args.rses:
401
414
  for replica, rse in itertools.product(replicas, rses):
402
- if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
415
+ if replica['rses'].get(rse):
403
416
  for pfn in replica['rses'][rse]:
404
417
  os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
405
418
  else:
@@ -410,16 +423,17 @@ def list_file_replicas(args):
410
423
  os.symlink(dst_dir + pfn.rsplit(pfn_dir)[-1], replica['name'])
411
424
  elif args.pfns:
412
425
  if args.rses:
413
- for replica, rse in itertools.product(replicas, rses):
414
- if rse in list(replica['rses'].keys()) and replica['rses'][rse]:
415
- for pfn in replica['rses'][rse]:
426
+ for replica in replicas:
427
+ for pfn in replica['pfns']:
428
+ rse = replica['pfns'][pfn]['rse']
429
+ if replica['rses'].get(rse):
416
430
  print(pfn)
417
431
  else:
418
432
  for replica in replicas:
419
- for rse in replica['rses']:
433
+ for pfn in replica['pfns']:
434
+ rse = replica['pfns'][pfn]['rse']
420
435
  if replica['rses'][rse]:
421
- for pfn in replica['rses'][rse]:
422
- print(pfn)
436
+ print(pfn)
423
437
  else:
424
438
  if args.all_states:
425
439
  header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', '(STATE) RSE: REPLICA']
@@ -427,18 +441,18 @@ def list_file_replicas(args):
427
441
  header = ['SCOPE', 'NAME', 'FILESIZE', 'ADLER32', 'RSE: REPLICA']
428
442
  for replica in replicas:
429
443
  if 'bytes' in replica:
430
- for rse in replica['rses']:
431
- for pfn in replica['rses'][rse]:
432
- if args.all_states:
433
- rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
434
- else:
435
- rse_string = '{0}: {1}'.format(rse, pfn)
436
- if args.rses:
437
- for selected_rse in rses:
438
- if rse == selected_rse:
439
- table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
440
- else:
441
- table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
444
+ for pfn in replica['pfns']:
445
+ rse = replica['pfns'][pfn]['rse']
446
+ if args.all_states:
447
+ rse_string = '({2}) {0}: {1}'.format(rse, pfn, ReplicaState[replica['states'][rse]].value)
448
+ else:
449
+ rse_string = '{0}: {1}'.format(rse, pfn)
450
+ if args.rses:
451
+ for selected_rse in rses:
452
+ if rse == selected_rse:
453
+ table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
454
+ else:
455
+ table.append([replica['scope'], replica['name'], sizefmt(replica['bytes'], args.human), replica['adler32'], rse_string])
442
456
  print(tabulate(table, tablefmt=tablefmt, headers=header, disable_numparse=True))
443
457
  return SUCCESS
444
458
 
@@ -490,7 +504,7 @@ def attach(args):
490
504
  try:
491
505
  f = open(dids[0], 'r')
492
506
  dids = [did.rstrip() for did in f.readlines()]
493
- except IOError:
507
+ except OSError:
494
508
  logger.error("Can't open file '" + dids[0] + "'.")
495
509
  return FAILURE
496
510
 
@@ -868,44 +882,6 @@ def erase(args):
868
882
  return SUCCESS
869
883
 
870
884
 
871
- @exception_handler
872
- def list_impls(args):
873
- """
874
- %(prog)s list-impls
875
-
876
- List protocol implementations.
877
- """
878
-
879
- PROTOCOL_DIRECTORY = '/opt/rucio/lib/rucio/rse/protocols'
880
-
881
- for filename in os.listdir(PROTOCOL_DIRECTORY):
882
- if os.path.isdir(os.path.join(PROTOCOL_DIRECTORY, filename)) or filename in ["__init__.py", "cache.py", "http_cache.py", "dummy.py", "protocol.py"]:
883
- continue
884
- else:
885
- filename = re.sub(r".py", r"", filename)
886
- __import__("rucio.rse.protocols", fromlist=[filename])
887
-
888
- impls = []
889
-
890
- def get_subclasses(cls):
891
- for subclass in cls.__subclasses__():
892
- if (str(subclass)[28:-2]).endswith('Default'):
893
- class_name = str(subclass)[28:-10]
894
- else:
895
- class_name = str(subclass)[28:-2]
896
- impls.append([class_name, subclass.__doc__])
897
- get_subclasses(subclass)
898
-
899
- get_subclasses(RSEProtocol)
900
- impls = sorted(impls)
901
- table = []
902
- for impl in impls:
903
- table.append([impl[0], impl[1]])
904
-
905
- print(tabulate(table, tablefmt=tablefmt, headers=['impl', 'DESCRIPTION']))
906
- return SUCCESS
907
-
908
-
909
885
  @exception_handler
910
886
  def upload(args):
911
887
  """
@@ -1105,6 +1081,24 @@ def download(args):
1105
1081
  logger.debug(args.dids)
1106
1082
  item_defaults['pfn'] = args.pfn
1107
1083
  item_defaults['did'] = did_str
1084
+ if args.rses is None:
1085
+ logger.warning("No RSE was given, selecting one.")
1086
+ client = get_client(args)
1087
+ replicas = client.list_replicas(
1088
+ [{"scope": did_str.split(':')[0], "name": did_str.split(':')[-1]}],
1089
+ schemes=args.protocol,
1090
+ ignore_availability=False,
1091
+ client_location=detect_client_location(),
1092
+ resolve_archives=not args.no_resolve_archives
1093
+ )
1094
+
1095
+ download_rse = _get_rse_for_pfn(replicas, args.pfn)
1096
+ if download_rse is None:
1097
+ logger.error("Could not find RSE for pfn %s", args.pfn)
1098
+ return FAILURE
1099
+ else:
1100
+ item_defaults['rse'] = download_rse
1101
+
1108
1102
  result = download_client.download_pfns([item_defaults], 1, trace_pattern, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
1109
1103
 
1110
1104
  if not result:
@@ -1146,6 +1140,21 @@ def download(args):
1146
1140
  return SUCCESS
1147
1141
 
1148
1142
 
1143
+ def _get_rse_for_pfn(replicas, pfn) -> Optional[str]:
1144
+ # Check each rse in the replica list for the pfn. If no pfn is found, returns None.
1145
+ # If it is found, stop the generator and return the item.
1146
+ for replica in replicas:
1147
+ try:
1148
+ download_rse = next(
1149
+ rse for rse in replica['rses']
1150
+ if pfn in replica['rses'][rse]
1151
+ )
1152
+ except StopIteration:
1153
+ continue
1154
+ else:
1155
+ return download_rse
1156
+
1157
+
1149
1158
  @exception_handler
1150
1159
  def get_metadata(args):
1151
1160
  """
@@ -1179,7 +1188,7 @@ def set_metadata(args):
1179
1188
  client = get_client(args)
1180
1189
  value = args.value
1181
1190
  if args.key == 'lifetime':
1182
- value = float(args.value)
1191
+ value = None if args.value.lower() == 'none' else float(args.value)
1183
1192
  scope, name = get_scope(args.did, client)
1184
1193
  client.set_metadata(scope=scope, name=name, key=args.key, value=value)
1185
1194
  return SUCCESS
@@ -1374,9 +1383,6 @@ def info_rule(args):
1374
1383
 
1375
1384
  Retrieve information about a rule.
1376
1385
  """
1377
- if args.estimate_ttc:
1378
- logger.error('"--estimate-ttc" is deprecated!')
1379
- return FAILURE
1380
1386
  client = get_client(args)
1381
1387
  if args.examine:
1382
1388
  analysis = client.examine_replication_rule(rule_id=args.rule_id)
@@ -1581,7 +1587,7 @@ def list_rse_attributes(args):
1581
1587
  """
1582
1588
  client = get_client(args)
1583
1589
  attributes = client.list_rse_attributes(rse=args.rse)
1584
- table = [(k + ':', str(v)) for (k, v) in sorted(attributes.items())] # columns hav mixed datatypes
1590
+ table = [(k + ':', str(v)) for (k, v) in sorted(attributes.items())] # columns have mixed datatypes
1585
1591
  print(tabulate(table, tablefmt='plain', disable_numparse=True)) # disabling number parsing
1586
1592
  return SUCCESS
1587
1593
 
@@ -1885,7 +1891,8 @@ def get_parser():
1885
1891
  + 'to the keys configured in the /etc/idpsecrets.json auth server configuration file.') # NOQA: W503
1886
1892
 
1887
1893
  # Options for the x509 auth_strategy
1888
- oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file.')
1894
+ oparser.add_argument('--certificate', dest='certificate', default=None, help='Client certificate file for x509 Authentication.')
1895
+ oparser.add_argument('--client_key', dest='client_key', default=None, help='Client key for x509 Authentication.')
1889
1896
  oparser.add_argument('--ca-certificate', dest='ca_certificate', default=None, help='CA certificate to verify peer against (SSL).')
1890
1897
 
1891
1898
  # Ping command
@@ -1957,7 +1964,7 @@ To list the missing replica of a dataset of a given RSE-expression::
1957
1964
  list_file_replicas_parser.add_argument('--no-resolve-archives', dest='no_resolve_archives', default=False, action='store_true', help='Do not resolve archives which may contain the files.', required=False)
1958
1965
  list_file_replicas_parser.add_argument('--sort', dest='sort', default=None, action='store', help='Replica sort algorithm. Available options: geoip (default), random', required=False)
1959
1966
  list_file_replicas_parser.add_argument('--rses', dest='rses', default=None, action='store', help='The RSE filter expression. A comprehensive help about RSE expressions\
1960
- can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
1967
+ can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
1961
1968
 
1962
1969
  # The list-dataset-replicas command
1963
1970
  list_dataset_replicas_parser = subparsers.add_parser('list-dataset-replicas', help='List the dataset replicas.',
@@ -2265,10 +2272,6 @@ You can filter by key/value, e.g.::
2265
2272
  list_content_history_parser.set_defaults(function=list_content_history)
2266
2273
  list_content_history_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
2267
2274
 
2268
- # The list-impls command
2269
- list_impls_parser = subparsers.add_parser('list-impls', help='List all supported protocol implementations.')
2270
- list_impls_parser.set_defaults(function=list_impls)
2271
-
2272
2275
  # The upload subparser
2273
2276
  upload_parser = subparsers.add_parser('upload', help='Upload method.')
2274
2277
  upload_parser.set_defaults(function=upload)
@@ -2396,7 +2399,6 @@ You can filter by key/value, e.g.::
2396
2399
  info_rule_parser.set_defaults(function=info_rule)
2397
2400
  info_rule_parser.add_argument(dest='rule_id', action='store', help='The rule ID')
2398
2401
  info_rule_parser.add_argument('--examine', dest='examine', action='store_true', help='Detailed analysis of transfer errors')
2399
- info_rule_parser.add_argument('--estimate-ttc', dest='estimate_ttc', action='store_true', help='Deprecated, specifying this will cause the program to fail.')
2400
2402
 
2401
2403
  # The list_rules command
2402
2404
  list_rules_parser = subparsers.add_parser('list-rules', help='List replication rules.', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
@@ -2464,13 +2466,13 @@ You can filter by account::
2464
2466
  list_rses_parser = subparsers.add_parser('list-rses', help='Show the list of all the registered Rucio Storage Elements (RSEs).')
2465
2467
  list_rses_parser.set_defaults(function=list_rses)
2466
2468
  list_rses_parser.add_argument('--rses', dest='rses', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
2467
- can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
2469
+ can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
2468
2470
 
2469
2471
  # The list-suspicious-replicas command
2470
2472
  list_suspicious_replicas_parser = subparsers.add_parser('list-suspicious-replicas', help='Show the list of all replicas marked "suspicious".')
2471
2473
  list_suspicious_replicas_parser.set_defaults(function=list_suspicious_replicas)
2472
2474
  list_suspicious_replicas_parser.add_argument('--expression', dest='rse_expression', action='store', help='The RSE filter expression. A comprehensive help about RSE expressions \
2473
- can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_expressions/' + Color.END)
2475
+ can be found in ' + Color.BOLD + 'https://rucio.cern.ch/documentation/started/concepts/rse_expressions' + Color.END)
2474
2476
  list_suspicious_replicas_parser.add_argument('--younger_than', '--younger-than', new_option_string='--younger-than', dest='younger_than', action=StoreAndDeprecateWarningAction, help='List files that have been marked suspicious since the date "younger_than", e.g. 2021-11-29T00:00:00.') # NOQA: E501
2475
2477
  list_suspicious_replicas_parser.add_argument('--nattempts', dest='nattempts', action='store', help='Minimum number of failed attempts to access a suspicious file.')
2476
2478
 
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,25 +24,37 @@ import signal
25
24
  import sys
26
25
  import time
27
26
  import traceback
28
- from textwrap import dedent
27
+ from configparser import NoOptionError, NoSectionError
29
28
  from functools import wraps
29
+ from textwrap import dedent
30
30
 
31
- from configparser import NoOptionError, NoSectionError
32
31
  from tabulate import tabulate
33
32
 
34
33
  from rucio import version
35
34
  from rucio.client import Client
36
35
  from rucio.common.config import config_get
37
- from rucio.common.exception import (AccountNotFound, DataIdentifierAlreadyExists, AccessDenied,
38
- DataIdentifierNotFound, InvalidObject, ReplicaNotFound,
39
- RSENotFound, RSEOperationNotSupported, InvalidRSEExpression,
40
- DuplicateContent, RuleNotFound, CannotAuthenticate,
41
- Duplicate, ReplicaIsLocked, ConfigNotFound, ScopeNotFound,
42
- InputValidationError)
36
+ from rucio.common.constants import RseAttr
37
+ from rucio.common.exception import (
38
+ AccessDenied,
39
+ AccountNotFound,
40
+ CannotAuthenticate,
41
+ ConfigNotFound,
42
+ DataIdentifierAlreadyExists,
43
+ DataIdentifierNotFound,
44
+ Duplicate,
45
+ DuplicateContent,
46
+ InputValidationError,
47
+ InvalidObject,
48
+ InvalidRSEExpression,
49
+ ReplicaIsLocked,
50
+ ReplicaNotFound,
51
+ RSENotFound,
52
+ RSEOperationNotSupported,
53
+ RuleNotFound,
54
+ ScopeNotFound,
55
+ )
43
56
  from rucio.common.extra import import_extras
44
- from rucio.common.utils import (chunks, construct_surl, sizefmt, get_bytes_value_from_string,
45
- render_json, parse_response, extract_scope, clean_surls,
46
- StoreAndDeprecateWarningAction)
57
+ from rucio.common.utils import StoreAndDeprecateWarningAction, chunks, clean_pfns, construct_non_deterministic_pfn, extract_scope, get_bytes_value_from_string, parse_response, render_json, sizefmt
47
58
  from rucio.rse import rsemanager as rsemgr
48
59
 
49
60
  EXTRA_MODULES = import_extras(['argcomplete'])
@@ -375,7 +386,7 @@ def set_limits(args):
375
386
  try:
376
387
  byte_limit = int(limit_input)
377
388
  except ValueError:
378
- logger.error('The limit could not be set. Either you misspelled infinity or your input could not be converted to integer or you used a wrong pattern. Please use a format like 10GB with B,KB,MB,GB,TB,PB as units (not case sensistive)')
389
+ logger.error('The limit could not be set. Either you misspelled infinity or your input could not be converted to integer or you used a wrong pattern. Please use a format like 10GB with B,KB,MB,GB,TB,PB as units (not case sensitive)')
379
390
  return FAILURE
380
391
 
381
392
  client.set_account_limit(account=args.account, rse=args.rse, bytes_=byte_limit, locality=locality)
@@ -1137,21 +1148,21 @@ def declare_bad_file_replicas(args):
1137
1148
  list_bad_pfns = []
1138
1149
  cnt += 1
1139
1150
  previous_pattern = None
1140
- for surl in clean_surls(chunk):
1151
+ for pfn in clean_pfns(chunk):
1141
1152
  unknown = True
1142
1153
  if previous_pattern:
1143
- if previous_pattern in surl:
1144
- list_bad_pfns.append(surl)
1154
+ if previous_pattern in pfn:
1155
+ list_bad_pfns.append(pfn)
1145
1156
  unknown = False
1146
1157
  continue
1147
1158
  for pattern in prot_dict:
1148
- if pattern in surl:
1159
+ if pattern in pfn:
1149
1160
  previous_pattern = prot_dict[pattern]
1150
- list_bad_pfns.append(surl)
1161
+ list_bad_pfns.append(pfn)
1151
1162
  unknown = False
1152
1163
  break
1153
1164
  if unknown:
1154
- print('Cannot find any RSE associated to %s' % surl)
1165
+ print('Cannot find any RSE associated to %s' % pfn)
1155
1166
  client.add_bad_pfns(pfns=list_bad_pfns, reason=args.reason, state='BAD', expires_at=None)
1156
1167
  ndeclared = len(list_bad_pfns)
1157
1168
  tot_file_declared += ndeclared
@@ -1185,7 +1196,9 @@ def declare_temporary_unavailable_replicas(args):
1185
1196
  else:
1186
1197
  bad_files = args.listbadfiles
1187
1198
 
1188
- assert args.duration is not None, "Duration should have been set, something went wrong!"
1199
+ if args.duration is None:
1200
+ raise ValueError("Duration should have been set, something went wrong!")
1201
+
1189
1202
  expiration_date = (datetime.datetime.utcnow() + datetime.timedelta(seconds=args.duration)).isoformat()
1190
1203
 
1191
1204
  chunk_size = 10000
@@ -1234,13 +1247,13 @@ def list_pfns(args):
1234
1247
  if not rse_info['deterministic']:
1235
1248
  logger.warning('This is a non-deterministic site, so the real PFN might be different from the on suggested')
1236
1249
  rse_attr = client.list_rse_attributes(rse)
1237
- naming_convention = rse_attr.get('naming_convention', None)
1250
+ naming_convention = rse_attr.get(RseAttr.NAMING_CONVENTION, None)
1238
1251
  parents = [did for did in client.list_parent_dids(scope, name)]
1239
1252
  if len(parents) > 1:
1240
1253
  logger.warning('The file has multiple parents')
1241
1254
  for did in parents:
1242
1255
  if did['type'] == 'DATASET':
1243
- path = construct_surl(did['name'], scope, name, naming_convention=naming_convention)
1256
+ path = construct_non_deterministic_pfn(did['name'], scope, name, naming_convention=naming_convention)
1244
1257
  pfn = ''.join([proto.attributes['scheme'],
1245
1258
  '://',
1246
1259
  proto.attributes['hostname'],
@@ -1277,7 +1290,7 @@ def import_data(args):
1277
1290
  print('There was problem with decoding your file.')
1278
1291
  print(error)
1279
1292
  return FAILURE
1280
- except IOError as error:
1293
+ except OSError as error:
1281
1294
  print('There was a problem with reading your file.')
1282
1295
  print(error)
1283
1296
  return FAILURE
@@ -1309,7 +1322,7 @@ def export_data(args):
1309
1322
  print('File successfully written.')
1310
1323
  print('Data successfully exported to %s' % args.file_path)
1311
1324
  return SUCCESS
1312
- except IOError as error:
1325
+ except OSError as error:
1313
1326
  print('There was a problem with reading your file.')
1314
1327
  print(error)
1315
1328
  return FAILURE
@@ -1373,7 +1386,7 @@ def get_parser():
1373
1386
  oparser.add_argument('--oidc-refresh-lifetime', dest='oidc_refresh_lifetime', default=None, help='Max lifetime in hours for this an access token will be refreshed by asynchronous Rucio daemon. '
1374
1387
  + 'If not specified, refresh will be stopped after 4 days. This option is effective only if --oidc-scope includes offline_access scope for a refresh token to be granted to Rucio.') # NOQA: W503
1375
1388
  oparser.add_argument('--oidc-issuer', dest='oidc_issuer', default=None,
1376
- help='Defines which Identity Provider is goign to be used. The issuer string must correspond '
1389
+ help='Defines which Identity Provider is going to be used. The issuer string must correspond '
1377
1390
  + 'to the keys configured in the /etc/idpsecrets.json auth server configuration file.') # NOQA: W503
1378
1391
 
1379
1392
  # Options for the x509 auth_strategy
@@ -1422,7 +1435,7 @@ def get_parser():
1422
1435
  '"""""""""""""\n'
1423
1436
  '::\n'
1424
1437
  '\n'
1425
- ' $ rucio-admin account list --type \'user\'\n'
1438
+ ' $ rucio-admin account list --type USER\n'
1426
1439
  '\n')
1427
1440
  list_account_parser.add_argument('--type', dest='account_type', action='store', help='Account Type (USER, GROUP, SERVICE)')
1428
1441
  list_account_parser.add_argument('--id', dest='identity', action='store', help='Identity (e.g. DN)')
@@ -1698,7 +1711,7 @@ def get_parser():
1698
1711
  ' $ rucio-admin identity delete --account jdoe --type X509 --id \'CN=Joe Doe,CN=707658,CN=jdoe,OU=Users,OU=Organic Units,DC=cern,DC=ch\'\n'
1699
1712
  ' Deleted identity: CN=Joe Doe,CN=707658,CN=jdoe,OU=Users,OU=Organic Units,DC=cern,DC=ch\n'
1700
1713
  '\n'
1701
- 'Note: if the identity was accidentaly deleted, use add option.\n'
1714
+ 'Note: if the identity was accidentally deleted, use add option.\n'
1702
1715
  '\n')
1703
1716
  identity_delete_parser.set_defaults(which='identity_delete')
1704
1717
  identity_delete_parser.add_argument('--account', dest='account', action='store', help='Account name', required=True)
@@ -1946,7 +1959,7 @@ def get_parser():
1946
1959
  ' $ rucio-admin rse add-protocol --hostname jdoes.test.org --scheme gsiftp --prefix \'/atlasdatadisk/rucio/\' --port 8443 JDOE_DATADISK\n'
1947
1960
  '\n'
1948
1961
  'Note: no printed stdout.\n'
1949
- 'Note: examples of optional parametres::\n'
1962
+ 'Note: examples of optional parameters::\n'
1950
1963
  '\n'
1951
1964
  ' --space-token DATADISK\n'
1952
1965
  ' --web-service-path \'/srm/managerv2?SFN=\'\n'
@@ -2100,7 +2113,7 @@ def get_parser():
2100
2113
 
2101
2114
  # The config subparser
2102
2115
  config_parser = subparsers.add_parser('config',
2103
- help='Configuration methods. The global configuration of data mangement system can by modified.',
2116
+ help='Configuration methods. The global configuration of data management system can by modified.',
2104
2117
  formatter_class=argparse.RawDescriptionHelpFormatter,
2105
2118
  epilog='''e.g. quotas, daemons, rses''')
2106
2119
  config_subparser = config_parser.add_subparsers(dest='config_subcommand', **required_arg)
@@ -2301,9 +2314,9 @@ def get_parser():
2301
2314
  declare_bad_file_replicas_parser.add_argument('--inputfile', dest='inputfile', nargs='?', action='store', help='File containing list of bad items')
2302
2315
  declare_bad_file_replicas_parser.add_argument('--allow-collection', dest='allow_collection', action='store_true', help='Allow passing a collection DID as bad item')
2303
2316
 
2304
- declare_bad_file_replicas_parser.add_argument('--lfns', dest='lfns', nargs='?', action='store', help='File cotaining list of LFNs for bad replicas. Requires --rse and --scope')
2305
- declare_bad_file_replicas_parser.add_argument('--scope', dest='scope', nargs='?', action='store', help='Common scope for bad replicas secified with LFN list, ignored wthout --lfns')
2306
- declare_bad_file_replicas_parser.add_argument('--rse', dest='rse', nargs='?', action='store', help='Common RSE for bad replicas secified with LFN list, ignored wthout --lfns')
2317
+ declare_bad_file_replicas_parser.add_argument('--lfns', dest='lfns', nargs='?', action='store', help='File containing list of LFNs for bad replicas. Requires --rse and --scope')
2318
+ declare_bad_file_replicas_parser.add_argument('--scope', dest='scope', nargs='?', action='store', help='Common scope for bad replicas specified with LFN list, ignored without --lfns')
2319
+ declare_bad_file_replicas_parser.add_argument('--rse', dest='rse', nargs='?', action='store', help='Common RSE for bad replicas specified with LFN list, ignored without --lfns')
2307
2320
 
2308
2321
  # The declare-temporary-unavailable command
2309
2322
  declare_temporary_unavailable_replicas_parser = rep_subparser.add_parser('declare-temporary-unavailable',
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rucio-clients
3
- Version: 32.8.6
3
+ Version: 35.8.0
4
4
  Summary: Rucio Client Lite Package
5
5
  Home-page: https://rucio.cern.ch/
6
6
  Author: Rucio
@@ -19,25 +19,25 @@ Classifier: Programming Language :: Python :: 3.10
19
19
  Requires-Python: >=3.9, <4
20
20
  License-File: LICENSE
21
21
  License-File: AUTHORS.rst
22
- Requires-Dist: requests<=2.31.0,>=2.25.1
23
- Requires-Dist: urllib3~=1.26.16
24
- Requires-Dist: dogpile.cache~=1.2.2
25
- Requires-Dist: tabulate~=0.9.0
26
- Requires-Dist: jsonschema~=4.18.4
22
+ Requires-Dist: requests>=2.32.2
23
+ Requires-Dist: urllib3>=1.26.18
24
+ Requires-Dist: dogpile-cache>=1.2.2
25
+ Requires-Dist: tabulate>=0.9.0
26
+ Requires-Dist: jsonschema>=4.20.0
27
27
  Provides-Extra: ssh
28
- Requires-Dist: paramiko~=3.4.0; extra == "ssh"
28
+ Requires-Dist: paramiko>=3.4.1; extra == "ssh"
29
29
  Provides-Extra: kerberos
30
- Requires-Dist: kerberos~=1.3.1; extra == "kerberos"
31
- Requires-Dist: pykerberos~=1.2.4; extra == "kerberos"
30
+ Requires-Dist: kerberos>=1.3.1; extra == "kerberos"
31
+ Requires-Dist: pykerberos>=1.2.4; extra == "kerberos"
32
32
  Requires-Dist: requests-kerberos>=0.14.0; extra == "kerberos"
33
33
  Provides-Extra: swift
34
- Requires-Dist: python-swiftclient~=4.3.0; extra == "swift"
34
+ Requires-Dist: python-swiftclient>=4.4.0; extra == "swift"
35
35
  Provides-Extra: argcomplete
36
- Requires-Dist: argcomplete~=3.1.1; extra == "argcomplete"
36
+ Requires-Dist: argcomplete>=3.1.6; extra == "argcomplete"
37
37
  Provides-Extra: sftp
38
- Requires-Dist: paramiko~=3.4.0; extra == "sftp"
38
+ Requires-Dist: paramiko>=3.4.1; extra == "sftp"
39
39
  Provides-Extra: dumper
40
- Requires-Dist: python-magic~=0.4.27; extra == "dumper"
40
+ Requires-Dist: python-magic>=0.4.27; extra == "dumper"
41
41
  Dynamic: author
42
42
  Dynamic: author-email
43
43
  Dynamic: classifier