rucio 37.6.0__py3-none-any.whl → 37.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (139) hide show
  1. rucio/cli/bin_legacy/rucio.py +40 -21
  2. rucio/cli/rule.py +9 -5
  3. rucio/client/baseclient.py +4 -3
  4. rucio/client/downloadclient.py +2 -1
  5. rucio/client/exportclient.py +45 -4
  6. rucio/client/pingclient.py +35 -4
  7. rucio/client/touchclient.py +2 -1
  8. rucio/client/uploadclient.py +3 -2
  9. rucio/common/cache.py +1 -2
  10. rucio/common/client.py +4 -30
  11. rucio/common/config.py +26 -1
  12. rucio/common/constants.py +3 -1
  13. rucio/common/plugins.py +2 -2
  14. rucio/common/policy.py +3 -2
  15. rucio/common/schema/__init__.py +4 -3
  16. rucio/common/types.py +7 -5
  17. rucio/core/account.py +2 -1
  18. rucio/core/account_limit.py +3 -2
  19. rucio/core/did.py +8 -7
  20. rucio/core/dirac.py +2 -1
  21. rucio/core/distance.py +2 -1
  22. rucio/core/exporter.py +3 -2
  23. rucio/core/importer.py +5 -5
  24. rucio/core/permission/__init__.py +2 -1
  25. rucio/core/replica.py +5 -5
  26. rucio/core/request.py +2 -2
  27. rucio/core/rse.py +7 -7
  28. rucio/core/rule.py +8 -8
  29. rucio/core/transfer.py +2 -2
  30. rucio/core/vo.py +2 -1
  31. rucio/daemons/atropos/atropos.py +2 -1
  32. rucio/daemons/automatix/automatix.py +5 -5
  33. rucio/daemons/badreplicas/minos.py +3 -2
  34. rucio/daemons/bb8/bb8.py +2 -1
  35. rucio/daemons/bb8/nuclei_background_rebalance.py +2 -2
  36. rucio/daemons/conveyor/common.py +3 -3
  37. rucio/daemons/conveyor/submitter.py +2 -1
  38. rucio/daemons/hermes/hermes.py +36 -6
  39. rucio/daemons/reaper/dark_reaper.py +5 -4
  40. rucio/daemons/reaper/reaper.py +7 -7
  41. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +3 -3
  42. rucio/daemons/tracer/kronos.py +3 -2
  43. rucio/daemons/transmogrifier/transmogrifier.py +70 -68
  44. rucio/daemons/undertaker/undertaker.py +2 -1
  45. rucio/db/sqla/models.py +2 -2
  46. rucio/db/sqla/util.py +3 -2
  47. rucio/gateway/account.py +13 -12
  48. rucio/gateway/account_limit.py +90 -116
  49. rucio/gateway/authentication.py +9 -8
  50. rucio/gateway/config.py +11 -10
  51. rucio/gateway/credential.py +2 -1
  52. rucio/gateway/did.py +32 -32
  53. rucio/gateway/dirac.py +2 -1
  54. rucio/gateway/exporter.py +2 -1
  55. rucio/gateway/heartbeat.py +3 -2
  56. rucio/gateway/identity.py +4 -3
  57. rucio/gateway/importer.py +2 -1
  58. rucio/gateway/lifetime_exception.py +4 -3
  59. rucio/gateway/lock.py +6 -5
  60. rucio/gateway/meta_conventions.py +3 -2
  61. rucio/gateway/permission.py +2 -1
  62. rucio/gateway/quarantined_replica.py +2 -1
  63. rucio/gateway/replica.py +18 -18
  64. rucio/gateway/request.py +10 -10
  65. rucio/gateway/rse.py +27 -26
  66. rucio/gateway/rule.py +12 -11
  67. rucio/gateway/scope.py +4 -3
  68. rucio/gateway/subscription.py +7 -6
  69. rucio/gateway/vo.py +5 -4
  70. rucio/rse/__init__.py +7 -6
  71. rucio/rse/rsemanager.py +5 -4
  72. rucio/rse/translation.py +2 -2
  73. rucio/tests/common.py +2 -1
  74. rucio/vcsversion.py +3 -3
  75. rucio/web/rest/flaskapi/v1/accountlimits.py +5 -5
  76. rucio/web/rest/flaskapi/v1/archives.py +2 -1
  77. rucio/web/rest/flaskapi/v1/common.py +4 -3
  78. rucio/web/rest/flaskapi/v1/dids.py +205 -154
  79. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/METADATA +1 -1
  80. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/RECORD +139 -139
  81. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/alembic.ini.template +0 -0
  82. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  83. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/globus-config.yml.template +0 -0
  84. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/ldap.cfg.template +0 -0
  85. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  86. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  87. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  88. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  89. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  90. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  91. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  92. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  93. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/rucio.cfg.template +0 -0
  94. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  95. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/requirements.server.txt +0 -0
  96. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/tools/bootstrap.py +0 -0
  97. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  98. {rucio-37.6.0.data → rucio-37.7.1.data}/data/rucio/tools/reset_database.py +0 -0
  99. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio +0 -0
  100. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-abacus-account +0 -0
  101. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-abacus-collection-replica +0 -0
  102. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-abacus-rse +0 -0
  103. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-admin +0 -0
  104. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-atropos +0 -0
  105. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-auditor +0 -0
  106. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-automatix +0 -0
  107. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-bb8 +0 -0
  108. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-cache-client +0 -0
  109. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-cache-consumer +0 -0
  110. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-finisher +0 -0
  111. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-poller +0 -0
  112. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-preparer +0 -0
  113. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-receiver +0 -0
  114. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-stager +0 -0
  115. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-submitter +0 -0
  116. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-conveyor-throttler +0 -0
  117. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-dark-reaper +0 -0
  118. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-dumper +0 -0
  119. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-follower +0 -0
  120. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-hermes +0 -0
  121. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-judge-cleaner +0 -0
  122. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-judge-evaluator +0 -0
  123. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-judge-injector +0 -0
  124. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-judge-repairer +0 -0
  125. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-kronos +0 -0
  126. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-minos +0 -0
  127. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-minos-temporary-expiration +0 -0
  128. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-necromancer +0 -0
  129. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-oauth-manager +0 -0
  130. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-reaper +0 -0
  131. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-replica-recoverer +0 -0
  132. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-rse-decommissioner +0 -0
  133. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-storage-consistency-actions +0 -0
  134. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-transmogrifier +0 -0
  135. {rucio-37.6.0.data → rucio-37.7.1.data}/scripts/rucio-undertaker +0 -0
  136. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/WHEEL +0 -0
  137. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/licenses/AUTHORS.rst +0 -0
  138. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/licenses/LICENSE +0 -0
  139. {rucio-37.6.0.dist-info → rucio-37.7.1.dist-info}/top_level.txt +0 -0
@@ -25,6 +25,7 @@ import random
25
25
  import re
26
26
  import smtplib
27
27
  import socket
28
+ import ssl
28
29
  import sys
29
30
  import threading
30
31
  import time
@@ -343,6 +344,14 @@ def deliver_emails(
343
344
  :returns: List of message_id to delete
344
345
  """
345
346
 
347
+ smtp_host = config_get("messaging-hermes", "smtp_host", default='')
348
+ smtp_port = config_get_int("messaging-hermes", "smtp_port", default=25)
349
+ smtp_username = config_get("messaging-hermes", "smtp_username", default='')
350
+ smtp_password = config_get("messaging-hermes", "smtp_password", default='')
351
+ smtp_certfile = config_get("messaging-hermes", "smtp_certfile", default='')
352
+ smtp_keyfile = config_get("messaging-hermes", "smtp_keyfile", default='')
353
+ smtp_usessl = config_get_bool("messaging-hermes", "smtp_usessl", default=False)
354
+ smtp_usetls = config_get_bool("messaging-hermes", "smtp_usetls", default=False)
346
355
  email_from = config_get("messaging-hermes", "email_from")
347
356
  send_email = config_get_bool(
348
357
  "messaging-hermes", "send_email", raise_exception=False, default=True
@@ -357,12 +366,33 @@ def deliver_emails(
357
366
 
358
367
  try:
359
368
  if send_email:
360
- smtp = smtplib.SMTP()
361
- smtp.connect()
362
- smtp.sendmail(
363
- msg["From"], message["payload"]["to"], msg.as_string()
364
- )
365
- smtp.quit()
369
+ # Fall back to unauthenticated connection if no host is provided
370
+ if not smtp_host:
371
+ smtp = smtplib.SMTP()
372
+ smtp.connect()
373
+ smtp.sendmail(
374
+ msg["From"], message["payload"]["to"], msg.as_string()
375
+ )
376
+ smtp.quit()
377
+ else:
378
+ ssl_context = None
379
+ if smtp_certfile and smtp_keyfile:
380
+ ssl_context = ssl.create_default_context()
381
+ ssl_context.load_cert_chain(certfile=smtp_certfile, keyfile=smtp_keyfile)
382
+
383
+ smtp_context = smtplib.SMTP(host=smtp_host, port=smtp_port)
384
+ if not smtp_usetls and smtp_usessl:
385
+ smtp_context = smtplib.SMTP_SSL(host=smtp_host, port=smtp_port, context=ssl_context)
386
+
387
+ with smtp_context as smtp_server:
388
+ if smtp_usetls:
389
+ smtp_server.ehlo() # not strictly necessary
390
+ smtp_server.starttls(context=ssl_context)
391
+ if smtp_username and smtp_password:
392
+ smtp_server.login(smtp_username, smtp_password)
393
+ smtp_server.sendmail(
394
+ msg["From"], message["payload"]["to"], msg.as_string()
395
+ )
366
396
  to_delete.append(message["id"])
367
397
  except Exception as error:
368
398
  logger(logging.ERROR, "Cannot send email : %s", str(error))
@@ -29,6 +29,7 @@ import rucio.core.rse as rse_core
29
29
  import rucio.db.sqla.util
30
30
  from rucio.common import exception
31
31
  from rucio.common.config import config_get_bool
32
+ from rucio.common.constants import DEFAULT_VO
32
33
  from rucio.common.exception import ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, ServiceUnavailable, SourceNotFound, VONotFound
33
34
  from rucio.common.logging import setup_logging
34
35
  from rucio.core.message import add_message
@@ -145,7 +146,7 @@ def run_once(
145
146
  'url': pfn,
146
147
  'duration': duration,
147
148
  'protocol': prot.attributes['scheme']}
148
- if replica['scope'].vo != 'def':
149
+ if replica['scope'].vo != DEFAULT_VO:
149
150
  payload['vo'] = replica['scope'].vo
150
151
  add_message('deletion-done', payload)
151
152
  deleted_replicas.append(replica)
@@ -167,7 +168,7 @@ def run_once(
167
168
  'url': pfn,
168
169
  'reason': str(error),
169
170
  'protocol': prot.attributes['scheme']}
170
- if replica['scope'].vo != 'def':
171
+ if replica['scope'].vo != DEFAULT_VO:
171
172
  payload['vo'] = replica['scope'].vo
172
173
  add_message('deletion-failed', payload)
173
174
 
@@ -215,7 +216,7 @@ def run(
215
216
  :param exclude_rses: RSE expression to exclude RSEs from the Reaper.
216
217
  :param include_rses: RSE expression to include RSEs.
217
218
  :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
218
- If None, we either use all VOs if run from "def", or the current VO otherwise.
219
+ If None, we either use all VOs if run from DEFAULT_VO, or the current VO otherwise.
219
220
  """
220
221
  rses = rses or []
221
222
  setup_logging(process_name=DAEMON_NAME)
@@ -229,7 +230,7 @@ def run(
229
230
  if not multi_vo:
230
231
  if vos:
231
232
  logging.warning('Ignoring argument vos, this is only applicable in a multi-VO setup.')
232
- vos = ['def']
233
+ vos = [DEFAULT_VO]
233
234
  else:
234
235
  if vos:
235
236
  invalid = set(vos) - set([v['vo'] for v in list_vos()])
@@ -35,7 +35,7 @@ from sqlalchemy.exc import DatabaseError, IntegrityError
35
35
  import rucio.db.sqla.util
36
36
  from rucio.common.cache import MemcacheRegion
37
37
  from rucio.common.config import config_get_bool, config_get_int
38
- from rucio.common.constants import RseAttr
38
+ from rucio.common.constants import RseAttr, DEFAULT_VO
39
39
  from rucio.common.exception import DatabaseException, ReplicaNotFound, ReplicaUnAvailable, ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, RSEProtocolNotSupported, ServiceUnavailable, SourceNotFound, VONotFound
40
40
  from rucio.common.logging import setup_logging
41
41
  from rucio.common.stopwatch import Stopwatch
@@ -81,7 +81,7 @@ def get_rses_to_process(
81
81
  :param exclude_rses: RSE expression to exclude RSEs from the Reaper.
82
82
  :param include_rses: RSE expression to include RSEs.
83
83
  :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
84
- If None, we either use all VOs if run from "def"
84
+ If None, we either use all VOs if run from DEFAULT_VO
85
85
 
86
86
  :returns: A list of RSEs to process
87
87
  """
@@ -89,7 +89,7 @@ def get_rses_to_process(
89
89
  if not multi_vo:
90
90
  if vos:
91
91
  logging.log(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
92
- vos = ['def']
92
+ vos = [DEFAULT_VO]
93
93
  else:
94
94
  if vos:
95
95
  invalid = set(vos) - set([v['vo'] for v in list_vos()])
@@ -156,7 +156,7 @@ def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info,
156
156
  'protocol': prot.attributes['scheme'],
157
157
  'datatype': replica['datatype']}
158
158
  try:
159
- if replica['scope'].vo != 'def':
159
+ if replica['scope'].vo != DEFAULT_VO:
160
160
  deletion_dict['vo'] = replica['scope'].vo
161
161
  logger(logging.DEBUG, 'Deletion ATTEMPT of %s:%s as %s on %s', replica['scope'], replica['name'], replica['pfn'], rse_name)
162
162
  # For STAGING RSEs, no physical deletion
@@ -232,7 +232,7 @@ def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info,
232
232
  'url': replica['pfn'],
233
233
  'reason': str(error),
234
234
  'protocol': prot.attributes['scheme']}
235
- if replica['scope'].vo != 'def':
235
+ if replica['scope'].vo != DEFAULT_VO:
236
236
  payload['vo'] = replica['scope'].vo
237
237
  add_message('deletion-failed', payload)
238
238
  logger(logging.INFO, 'Cannot connect to %s. RSE will be temporarily excluded.', rse_name)
@@ -400,7 +400,7 @@ def reaper(
400
400
  :param include_rses: RSE expression to include RSEs.
401
401
  :param exclude_rses: RSE expression to exclude RSEs from the Reaper.
402
402
  :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
403
- If None, we either use all VOs if run from "def", or the current VO otherwise.
403
+ If None, we either use all VOs if run from DEFAULT_VO, or the current VO otherwise.
404
404
  :param chunk_size: The size of chunk for deletion.
405
405
  :param once: If True, only runs one iteration of the main loop.
406
406
  :param greedy: If True, delete right away replicas with tombstone.
@@ -703,7 +703,7 @@ def run(
703
703
  :param exclude_rses: RSE expression to exclude RSEs from the Reaper.
704
704
  :param include_rses: RSE expression to include RSEs.
705
705
  :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
706
- If None, we either use all VOs if run from "def",
706
+ If None, we either use all VOs if run from DEFAULT_VO,
707
707
  or the current VO otherwise.
708
708
  :param delay_seconds: The delay to query replicas in BEING_DELETED state.
709
709
  :param sleep_time: Time between two cycles.
@@ -31,7 +31,7 @@ from typing import TYPE_CHECKING, Any, Optional
31
31
 
32
32
  import rucio.db.sqla.util
33
33
  from rucio.common.config import config_get, config_get_bool
34
- from rucio.common.constants import SuspiciousAvailability
34
+ from rucio.common.constants import DEFAULT_VO, SuspiciousAvailability
35
35
  from rucio.common.exception import DatabaseException, DuplicateRule, VONotFound
36
36
  from rucio.common.logging import setup_logging
37
37
  from rucio.common.types import InternalAccount, LoggerFunction
@@ -145,7 +145,7 @@ def declare_suspicious_replicas_bad(
145
145
  :param nattempts: The minimum number of appearances in the bad_replica DB table
146
146
  in order to appear in the resulting list of replicas for recovery.
147
147
  :param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
148
- If empty, we either use all VOs if run from "def",
148
+ If empty, we either use all VOs if run from DEFAULT_VO,
149
149
  :param limit_suspicious_files_on_rse: Maximum number of suspicious replicas on an RSE before that RSE
150
150
  is considered problematic and the suspicious replicas on that RSE
151
151
  are labeled as 'TEMPORARY_UNAVAILABLE'.
@@ -182,7 +182,7 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: "Op
182
182
  if not multi_vo:
183
183
  if vos:
184
184
  logger(logging.WARNING, 'Ignoring argument vos, this is only applicable in a multi-VO setup.')
185
- vos = ['def']
185
+ vos = [DEFAULT_VO]
186
186
  else:
187
187
  if vos:
188
188
  invalid = set(vos) - set([v['vo'] for v in list_vos()])
@@ -30,6 +30,7 @@ from typing import TYPE_CHECKING, Optional
30
30
 
31
31
  import rucio.db.sqla.util
32
32
  from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
33
+ from rucio.common.constants import DEFAULT_VO
33
34
  from rucio.common.exception import DatabaseException, RSENotFound
34
35
  from rucio.common.logging import setup_logging
35
36
  from rucio.common.stomp_utils import StompConnectionManager
@@ -143,7 +144,7 @@ class AMQConsumer:
143
144
  rses = []
144
145
  for report in self.__reports:
145
146
  if 'vo' not in report:
146
- report['vo'] = 'def'
147
+ report['vo'] = DEFAULT_VO
147
148
 
148
149
  try:
149
150
  # Identify suspicious files
@@ -299,7 +300,7 @@ class AMQConsumer:
299
300
  'usrdn': 'someuser',
300
301
  'clientState': 'DONE',
301
302
  'eventVersion': replica['eventVersion']}
302
- if replica['scope'].vo != 'def':
303
+ if replica['scope'].vo != DEFAULT_VO:
303
304
  resubmit['vo'] = replica['scope'].vo
304
305
  self.__conn.send(body=jdumps(resubmit), destination=self.__queue, headers={'appversion': 'rucio', 'resubmitted': '1'})
305
306
  METRICS.counter('sent_resubmitted').inc()
@@ -124,7 +124,7 @@ def __split_rule_select_rses(
124
124
  weight: int,
125
125
  rse_expression: str,
126
126
  copies: int,
127
- blocklisted_rse_id: list,
127
+ blocklisted_rse_ids: list,
128
128
  logger: LoggerFunction,
129
129
  ) -> tuple[list, bool, bool]:
130
130
  """
@@ -138,7 +138,7 @@ def __split_rule_select_rses(
138
138
  :param weight: The weight of the rule.
139
139
  :param rse_expression: The RSE expression of the rule.
140
140
  :param copies: The number of copies.
141
- :param blocklisted_rse_id: The list of blocklisted_rse_id.
141
+ :param blocklisted_rse_ids: The list of blocklisted rse_ids.
142
142
  :param logger: The logger.
143
143
  :return: A tuple with list selected_rses, and 2 booleans create_rule, wont_reevaluate.
144
144
  """
@@ -172,7 +172,7 @@ def __split_rule_select_rses(
172
172
  copies=copies,
173
173
  size=0,
174
174
  preferred_rses=preferred_rses,
175
- blocklist=blocklisted_rse_id,
175
+ blocklist=blocklisted_rse_ids,
176
176
  )
177
177
  wont_reevaluate = True
178
178
  break
@@ -184,26 +184,22 @@ def __split_rule_select_rses(
184
184
  ) as error:
185
185
  logger(
186
186
  logging.WARNING,
187
- 'Problem getting RSEs for subscription "%s" for account %s : %s. %s'
188
- % (
189
- subscription_name,
190
- account,
191
- str(error),
192
- 'Try including blocklisted sites' if attempt == 0 else 'Skipping rule creation.'
193
- ),
187
+ 'Problem getting RSEs for subscription "%s" for account %s : %s. %s',
188
+ subscription_name,
189
+ account,
190
+ str(error),
191
+ 'Try including blocklisted sites' if attempt == 0 else 'Skipping rule creation.'
194
192
  )
195
193
  # Now including the blocklisted sites
196
- blocklisted_rse_id = []
194
+ blocklisted_rse_ids = []
197
195
  METRICS.counter(name="addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
198
196
  wont_reevaluate = True
199
197
  except Exception as error:
200
198
  logger(
201
199
  logging.ERROR,
202
- "Problem resolving RSE expression %s : %s"
203
- % (
204
- rse_expression,
205
- str(error),
206
- )
200
+ "Problem resolving RSE expression %s : %s",
201
+ rse_expression,
202
+ str(error),
207
203
  )
208
204
  if len(preferred_rses) - len(preferred_unmatched) >= copies:
209
205
  create_rule = False
@@ -242,6 +238,7 @@ def get_subscriptions(logger: LoggerFunction = logging.log) -> list[dict]:
242
238
  break
243
239
  if rule.get("copies") == "*":
244
240
  rule["copies"] = len(list_rses_from_expression)
241
+ rule["wildcard"] = True
245
242
  overwrite_rules = True
246
243
  if skip_sub:
247
244
  continue
@@ -272,7 +269,7 @@ def get_subscriptions(logger: LoggerFunction = logging.log) -> list[dict]:
272
269
  subscriptions.extend(sub_dict[priority])
273
270
  logger(logging.INFO, "%i active subscriptions", len(subscriptions))
274
271
  except SubscriptionNotFound as error:
275
- logger(logging.WARNING, "No subscriptions defined: %s" % (str(error)))
272
+ logger(logging.WARNING, "No subscriptions defined: %s", (str(error)))
276
273
  return []
277
274
  except TypeError as error:
278
275
  logger(
@@ -307,7 +304,7 @@ def __is_matching_subscription(
307
304
  try:
308
305
  filter_string = loads(subscription["filter"])
309
306
  except ValueError as error:
310
- logging.error("%s : Subscription will be skipped" % error)
307
+ logging.error("%s : Subscription will be skipped", error)
311
308
  return False
312
309
  # Loop over the keys of filter_string for subscription
313
310
  for key in filter_string:
@@ -399,7 +396,7 @@ def select_algorithm(
399
396
  selected_rses = {}
400
397
  for rule_id in rule_ids:
401
398
  rule = get_rule(rule_id)
402
- logging.debug("In select_algorithm, %s", str(rule))
399
+ logger(logging.DEBUG, "In select_algorithm, %s", str(rule))
403
400
  rse = rule["rse_expression"]
404
401
  vo = rule["account"].vo
405
402
  if rse_exists(rse, vo=vo):
@@ -439,7 +436,7 @@ def select_algorithm(
439
436
  weight=rule.get("weight"),
440
437
  rse_expression=rse_expression,
441
438
  copies=rule.get('copies'),
442
- blocklisted_rse_id=params['blocklisted_rse_id'],
439
+ blocklisted_rse_ids=params['blocklisted_rse_ids'],
443
440
  logger=logger,
444
441
  )
445
442
  dict_selected_rses = {}
@@ -487,7 +484,8 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
487
484
 
488
485
  worker_number, total_workers, logger = heartbeat_handler.live()
489
486
  stopwatch = Stopwatch()
490
- blocklisted_rse_id = [rse["id"] for rse in list_rses({"availability_write": False})]
487
+ block_listed = {rse['rse']: rse["id"] for rse in list_rses({"availability_write": False})}
488
+ blocklisted_rse_ids = list(block_listed.values())
491
489
  identifiers = []
492
490
  # List all the active subscriptions
493
491
  subscriptions = get_subscriptions(logger=logger)
@@ -535,6 +533,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
535
533
  # Get all the rule and subscription parameters
536
534
  rule_dict = __get_rule_dict(rule_dict, subscription)
537
535
  weight = rule_dict.get("weight", None)
536
+ ignore_availability = rule_dict.get("ignore_availability", False)
538
537
  source_replica_expression = rule_dict.get(
539
538
  "source_replica_expression", None
540
539
  )
@@ -551,7 +550,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
551
550
  params['rse_expression'] = rule_dict.get("rse_expression")
552
551
  params['subscription_id'] = subscription["id"]
553
552
  params['subscription_name'] = subscription["name"]
554
- params['blocklisted_rse_id'] = blocklisted_rse_id
553
+ params['blocklisted_rse_ids'] = blocklisted_rse_ids
555
554
  if rule_dict.get("associated_site_idx", None):
556
555
  params["associated_site_idx"] = rule_dict.get(
557
556
  "associated_site_idx", None
@@ -583,7 +582,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
583
582
  weight=weight,
584
583
  rse_expression=rule_dict.get("rse_expression"),
585
584
  copies=copies,
586
- blocklisted_rse_id=blocklisted_rse_id,
585
+ blocklisted_rse_ids=blocklisted_rse_ids,
587
586
  logger=logger,
588
587
  )
589
588
  copies = 1
@@ -595,20 +594,26 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
595
594
  nb_rule = 0
596
595
  # Try to create the rule
597
596
  logger(logging.DEBUG, 'selected_rses : %s' % selected_rses)
598
- try:
599
- for rse in selected_rses:
600
- if isinstance(selected_rses, dict):
601
- # selected_rses is a dictionary only when split_rule is True or for chained subscriptions
602
- source_replica_expression = selected_rses[rse].get(
603
- "source_replica_expression",
604
- None,
605
- )
606
- weight = selected_rses[rse].get("weight", None)
607
- logger(
608
- logging.INFO,
609
- "Will insert one rule for %s:%s on %s"
610
- % (did["scope"], did["name"], rse),
597
+ for rse in selected_rses:
598
+ if isinstance(selected_rses, dict):
599
+ # selected_rses is a dictionary only when split_rule is True or for chained subscriptions
600
+ source_replica_expression = selected_rses[rse].get(
601
+ "source_replica_expression",
602
+ None,
611
603
  )
604
+ weight = selected_rses[rse].get("weight", None)
605
+ logger(
606
+ logging.INFO,
607
+ "Will insert one rule for %s:%s on %s",
608
+ did["scope"], did["name"], rse,
609
+ )
610
+ if rse in block_listed and rule_dict.get("wildcard"):
611
+ if ignore_availability:
612
+ logger(logging.WARNING, "RSE %s is unavailable, but wildcard number of copies is used with ignore_availability option. Creating a rule", rse)
613
+ else:
614
+ logger(logging.INFO, "RSE %s is unavailable and wildcard number of copies is used. Skipping rule creation", rse)
615
+ continue
616
+ try:
612
617
  rule_ids = add_rule(
613
618
  dids=[
614
619
  {
@@ -627,9 +632,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
627
632
  source_replica_expression=source_replica_expression,
628
633
  activity=rule_dict.get("activity"),
629
634
  purge_replicas=rule_dict.get("purge_replicas", False),
630
- ignore_availability=rule_dict.get(
631
- "ignore_availability", None
632
- ),
635
+ ignore_availability=ignore_availability,
633
636
  comment=rule_dict.get("comment"),
634
637
  delay_injection=rule_dict.get("delay_injection"),
635
638
  )
@@ -640,41 +643,40 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
640
643
  if split_rule:
641
644
  success = True
642
645
 
643
- METRICS.counter("addnewrule.done").inc(nb_rule)
644
- METRICS.counter("addnewrule.activity.{activity}").labels(activity="".join(rule_dict.get("activity").split())).inc(nb_rule)
645
- success = True
646
- except (
647
- InvalidReplicationRule,
648
- InvalidRuleWeight,
649
- InvalidRSEExpression,
650
- StagingAreaRuleRequiresLifetime,
651
- DuplicateRule,
652
- ) as error:
653
- # Errors that won't be retried
654
- success = True
655
- logger(logging.ERROR, str(error))
656
- METRICS.counter("addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
657
- except Exception:
658
- # Errors that will be retried
659
- METRICS.counter("addnewrule.errortype.{exception}").labels(exception="unknown").inc()
660
- logger(logging.ERROR, "Unexpected error", exc_info=True)
646
+ except (
647
+ InvalidReplicationRule,
648
+ InvalidRuleWeight,
649
+ InvalidRSEExpression,
650
+ StagingAreaRuleRequiresLifetime,
651
+ DuplicateRule,
652
+ ) as error:
653
+ # Errors that won't be retried
654
+ success = True
655
+ logger(logging.ERROR, str(error))
656
+ METRICS.counter("addnewrule.errortype.{exception}").labels(exception=str(error.__class__.__name__)).inc()
657
+ except Exception:
658
+ # Errors that will be retried
659
+ METRICS.counter("addnewrule.errortype.{exception}").labels(exception="unknown").inc()
660
+ logger(logging.ERROR, "Unexpected error", exc_info=True)
661
+
662
+ METRICS.counter("addnewrule.done").inc(nb_rule)
663
+ METRICS.counter("addnewrule.activity.{activity}").labels(activity="".join(rule_dict.get("activity").split())).inc(nb_rule)
664
+ success = True
661
665
 
662
666
  did_success = did_success and success
663
667
  if not success:
664
668
  logger(
665
669
  logging.ERROR,
666
- "Rule for %s:%s on %s cannot be inserted"
667
- % (
668
- did["scope"],
669
- did["name"],
670
- rule_dict.get("rse_expression"),
671
- ),
670
+ "Rule for %s:%s on %s cannot be inserted",
671
+ did["scope"],
672
+ did["name"],
673
+ rule_dict.get("rse_expression"),
672
674
  )
673
675
  else:
674
676
  logger(
675
677
  logging.INFO,
676
- "%s rule(s) inserted in %f seconds"
677
- % (str(nb_rule), time.time() - stime),
678
+ "%s rule(s) inserted in %f seconds",
679
+ str(nb_rule), time.time() - stime,
678
680
  )
679
681
 
680
682
  if did_success:
@@ -697,7 +699,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
697
699
  flag_stopwatch = Stopwatch()
698
700
  for identifier in chunks(identifiers, 100):
699
701
  set_new_dids(identifier, None)
700
- logger(logging.DEBUG, "Time to set the new flag : %f" % flag_stopwatch.elapsed)
702
+ logger(logging.DEBUG, "Time to set the new flag : %f", flag_stopwatch.elapsed)
701
703
 
702
704
  stopwatch.stop()
703
705
 
@@ -709,9 +711,9 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
709
711
  )
710
712
  logger(
711
713
  logging.INFO,
712
- "It took %f seconds to process %i DIDs" % (stopwatch.elapsed, len(identifiers)),
714
+ "It took %f seconds to process %i DIDs", stopwatch.elapsed, len(identifiers),
713
715
  )
714
- logger(logging.DEBUG, "DIDs processed : %s" % (str(identifiers)))
716
+ logger(logging.DEBUG, "DIDs processed : %s", str(identifiers))
715
717
  METRICS.counter(name="transmogrifier.job.done").inc(1)
716
718
  METRICS.timer("job.duration").observe(stopwatch.elapsed)
717
719
  must_sleep = True
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING
29
29
  from sqlalchemy.exc import DatabaseError
30
30
 
31
31
  import rucio.db.sqla.util
32
+ from rucio.common.constants import DEFAULT_VO
32
33
  from rucio.common.exception import DatabaseException, RuleNotFound, UnsupportedOperation
33
34
  from rucio.common.logging import setup_logging
34
35
  from rucio.common.types import InternalAccount
@@ -90,7 +91,7 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
90
91
  _, _, logger = heartbeat_handler.live()
91
92
  try:
92
93
  logger(logging.INFO, 'Receive %s dids to delete', len(chunk))
93
- delete_dids(dids=chunk, account=InternalAccount('root', vo='def'), expire_rules=True)
94
+ delete_dids(dids=chunk, account=InternalAccount('root', vo=DEFAULT_VO), expire_rules=True)
94
95
  logger(logging.INFO, 'Delete %s dids', len(chunk))
95
96
  METRICS.counter(name='undertaker.delete_dids').inc(len(chunk))
96
97
  except RuleNotFound as error:
rucio/db/sqla/models.py CHANGED
@@ -29,7 +29,7 @@ from sqlalchemy.types import LargeBinary
29
29
  # and it must be renamed to avoid conflicts with the policy package schema modules
30
30
  from rucio.common import schema as common_schema
31
31
  from rucio.common import utils
32
- from rucio.common.constants import TransferLimitDirection
32
+ from rucio.common.constants import DEFAULT_VO, TransferLimitDirection
33
33
  from rucio.common.types import InternalAccount, InternalScope # noqa: TCH001 (types are needed by SQLAlchemy)
34
34
  from rucio.db.sqla.constants import (
35
35
  AccountStatus,
@@ -776,7 +776,7 @@ class RSE(BASE, SoftModelBase):
776
776
  __tablename__ = 'rses'
777
777
  id: Mapped[str] = mapped_column(GUID(), default=utils.generate_uuid)
778
778
  rse: Mapped[str] = mapped_column(String(255))
779
- vo: Mapped[str] = mapped_column(String(3), nullable=False, server_default='def')
779
+ vo: Mapped[str] = mapped_column(String(3), nullable=False, server_default=DEFAULT_VO)
780
780
  rse_type: Mapped[RSEType] = mapped_column(Enum(RSEType, name='RSES_TYPE_CHK',
781
781
  create_constraint=True,
782
782
  values_callable=lambda obj: [e.value for e in obj]),
rucio/db/sqla/util.py CHANGED
@@ -33,6 +33,7 @@ from sqlalchemy.sql.expression import select, text
33
33
  from rucio import alembicrevision
34
34
  from rucio.common.cache import MemcacheRegion
35
35
  from rucio.common.config import config_get, config_get_list
36
+ from rucio.common.constants import DEFAULT_VO
36
37
  from rucio.common.schema import get_schema_value
37
38
  from rucio.common.types import InternalAccount, LoggerFunction
38
39
  from rucio.common.utils import generate_uuid
@@ -136,7 +137,7 @@ def create_base_vo() -> None:
136
137
 
137
138
  session_scoped = get_session()
138
139
 
139
- vo = models.VO(vo='def', description='Default base VO', email='N/A')
140
+ vo = models.VO(vo=DEFAULT_VO, description='Default base VO', email='N/A')
140
141
  with session_scoped() as s:
141
142
  with s.begin():
142
143
  s.add_all([vo])
@@ -172,7 +173,7 @@ def create_root_account() -> None:
172
173
  else:
173
174
  access = 'root'
174
175
 
175
- account = models.Account(account=InternalAccount(access, 'def'), account_type=AccountType.SERVICE, status=AccountStatus.ACTIVE)
176
+ account = models.Account(account=InternalAccount(access, DEFAULT_VO), account_type=AccountType.SERVICE, status=AccountStatus.ACTIVE)
176
177
 
177
178
  salt = urandom(255)
178
179
  salted_password = salt + up_pwd.encode()