rucio 37.4.0__py3-none-any.whl → 37.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (179) hide show
  1. rucio/cli/bin_legacy/rucio.py +1 -1
  2. rucio/cli/bin_legacy/rucio_admin.py +1 -1
  3. rucio/cli/did.py +2 -2
  4. rucio/cli/rse.py +2 -3
  5. rucio/cli/subscription.py +1 -1
  6. rucio/client/baseclient.py +5 -1
  7. rucio/client/didclient.py +16 -16
  8. rucio/client/downloadclient.py +15 -15
  9. rucio/client/lockclient.py +3 -3
  10. rucio/client/replicaclient.py +2 -2
  11. rucio/client/requestclient.py +6 -5
  12. rucio/client/touchclient.py +1 -1
  13. rucio/client/uploadclient.py +725 -181
  14. rucio/common/config.py +1 -2
  15. rucio/common/constants.py +16 -17
  16. rucio/common/didtype.py +2 -2
  17. rucio/common/dumper/__init__.py +1 -1
  18. rucio/common/pcache.py +20 -25
  19. rucio/common/plugins.py +10 -17
  20. rucio/common/schema/__init__.py +7 -5
  21. rucio/common/utils.py +19 -3
  22. rucio/core/authentication.py +1 -1
  23. rucio/core/credential.py +1 -1
  24. rucio/core/did.py +54 -54
  25. rucio/core/did_meta_plugins/__init__.py +10 -10
  26. rucio/core/did_meta_plugins/did_column_meta.py +9 -9
  27. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +3 -3
  28. rucio/core/did_meta_plugins/elasticsearch_meta.py +7 -7
  29. rucio/core/did_meta_plugins/json_meta.py +2 -2
  30. rucio/core/did_meta_plugins/mongo_meta.py +9 -9
  31. rucio/core/did_meta_plugins/postgres_meta.py +7 -7
  32. rucio/core/dirac.py +1 -1
  33. rucio/core/lifetime_exception.py +2 -2
  34. rucio/core/lock.py +7 -7
  35. rucio/core/meta_conventions.py +2 -2
  36. rucio/core/monitor.py +1 -1
  37. rucio/core/naming_convention.py +1 -1
  38. rucio/core/nongrid_trace.py +2 -2
  39. rucio/core/oidc.py +2 -2
  40. rucio/core/permission/__init__.py +7 -5
  41. rucio/core/permission/generic.py +5 -2
  42. rucio/core/permission/generic_multi_vo.py +2 -2
  43. rucio/core/replica.py +18 -18
  44. rucio/core/request.py +2 -2
  45. rucio/core/rule.py +30 -30
  46. rucio/core/rule_grouping.py +2 -3
  47. rucio/core/scope.py +1 -1
  48. rucio/core/trace.py +2 -2
  49. rucio/core/transfer.py +4 -5
  50. rucio/daemons/auditor/__init__.py +1 -1
  51. rucio/daemons/badreplicas/minos.py +9 -3
  52. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -2
  53. rucio/daemons/badreplicas/necromancer.py +9 -3
  54. rucio/daemons/bb8/common.py +1 -1
  55. rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
  56. rucio/daemons/bb8/t2_background_rebalance.py +1 -1
  57. rucio/daemons/cache/consumer.py +1 -1
  58. rucio/daemons/conveyor/finisher.py +13 -4
  59. rucio/daemons/conveyor/poller.py +5 -2
  60. rucio/daemons/conveyor/receiver.py +1 -1
  61. rucio/daemons/conveyor/throttler.py +2 -1
  62. rucio/daemons/follower/follower.py +1 -1
  63. rucio/daemons/hermes/hermes.py +2 -2
  64. rucio/daemons/judge/cleaner.py +2 -2
  65. rucio/daemons/judge/evaluator.py +7 -7
  66. rucio/daemons/judge/injector.py +2 -2
  67. rucio/daemons/judge/repairer.py +2 -2
  68. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +1 -1
  69. rucio/daemons/storage/consistency/actions.py +3 -3
  70. rucio/daemons/transmogrifier/transmogrifier.py +1 -1
  71. rucio/daemons/undertaker/undertaker.py +6 -6
  72. rucio/db/sqla/constants.py +4 -3
  73. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -1
  74. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -1
  75. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +1 -1
  76. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +1 -1
  77. rucio/db/sqla/models.py +2 -2
  78. rucio/db/sqla/session.py +7 -7
  79. rucio/gateway/account.py +65 -90
  80. rucio/gateway/did.py +26 -26
  81. rucio/gateway/dirac.py +1 -1
  82. rucio/gateway/lifetime_exception.py +1 -1
  83. rucio/gateway/replica.py +2 -2
  84. rucio/gateway/request.py +13 -12
  85. rucio/rse/protocols/ngarc.py +2 -2
  86. rucio/rse/protocols/srm.py +1 -1
  87. rucio/rse/protocols/webdav.py +8 -1
  88. rucio/rse/rsemanager.py +2 -2
  89. rucio/tests/common.py +4 -4
  90. rucio/vcsversion.py +3 -3
  91. rucio/web/rest/flaskapi/v1/accountlimits.py +22 -22
  92. rucio/web/rest/flaskapi/v1/accounts.py +177 -177
  93. rucio/web/rest/flaskapi/v1/archives.py +10 -10
  94. rucio/web/rest/flaskapi/v1/auth.py +106 -106
  95. rucio/web/rest/flaskapi/v1/common.py +1 -1
  96. rucio/web/rest/flaskapi/v1/config.py +37 -37
  97. rucio/web/rest/flaskapi/v1/credentials.py +25 -25
  98. rucio/web/rest/flaskapi/v1/dids.py +391 -391
  99. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  100. rucio/web/rest/flaskapi/v1/export.py +6 -6
  101. rucio/web/rest/flaskapi/v1/heartbeats.py +14 -14
  102. rucio/web/rest/flaskapi/v1/identities.py +25 -25
  103. rucio/web/rest/flaskapi/v1/import.py +19 -19
  104. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +54 -54
  105. rucio/web/rest/flaskapi/v1/locks.py +62 -62
  106. rucio/web/rest/flaskapi/v1/main.py +1 -0
  107. rucio/web/rest/flaskapi/v1/meta_conventions.py +29 -29
  108. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  109. rucio/web/rest/flaskapi/v1/ping.py +4 -4
  110. rucio/web/rest/flaskapi/v1/redirect.py +16 -16
  111. rucio/web/rest/flaskapi/v1/replicas.py +282 -282
  112. rucio/web/rest/flaskapi/v1/requests.py +274 -270
  113. rucio/web/rest/flaskapi/v1/rses.py +427 -427
  114. rucio/web/rest/flaskapi/v1/rules.py +129 -129
  115. rucio/web/rest/flaskapi/v1/scopes.py +21 -21
  116. rucio/web/rest/flaskapi/v1/subscriptions.py +120 -120
  117. rucio/web/rest/flaskapi/v1/traces.py +18 -18
  118. rucio/web/rest/flaskapi/v1/vos.py +32 -32
  119. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/requirements.server.txt +1 -1
  120. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-abacus-account +8 -1
  121. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-abacus-rse +8 -1
  122. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-auditor +1 -1
  123. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-throttler +7 -1
  124. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-follower +1 -1
  125. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-judge-cleaner +9 -1
  126. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-necromancer +7 -1
  127. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-replica-recoverer +31 -9
  128. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-undertaker +8 -2
  129. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/METADATA +1 -1
  130. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/RECORD +179 -179
  131. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/WHEEL +1 -1
  132. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  133. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  134. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  135. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
  136. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  137. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  138. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  139. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  140. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  141. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  142. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  143. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  144. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  145. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  146. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/tools/bootstrap.py +0 -0
  147. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  148. {rucio-37.4.0.data → rucio-37.6.0.data}/data/rucio/tools/reset_database.py +0 -0
  149. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio +0 -0
  150. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-abacus-collection-replica +0 -0
  151. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-admin +0 -0
  152. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-atropos +0 -0
  153. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-automatix +0 -0
  154. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-bb8 +0 -0
  155. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-cache-client +0 -0
  156. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-cache-consumer +0 -0
  157. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-finisher +0 -0
  158. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-poller +0 -0
  159. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-preparer +0 -0
  160. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-receiver +0 -0
  161. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-stager +0 -0
  162. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-conveyor-submitter +0 -0
  163. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-dark-reaper +0 -0
  164. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-dumper +0 -0
  165. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-hermes +0 -0
  166. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-judge-evaluator +0 -0
  167. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-judge-injector +0 -0
  168. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-judge-repairer +0 -0
  169. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-kronos +0 -0
  170. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-minos +0 -0
  171. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
  172. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-oauth-manager +0 -0
  173. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-reaper +0 -0
  174. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-rse-decommissioner +0 -0
  175. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-storage-consistency-actions +0 -0
  176. {rucio-37.4.0.data → rucio-37.6.0.data}/scripts/rucio-transmogrifier +0 -0
  177. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/licenses/AUTHORS.rst +0 -0
  178. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/licenses/LICENSE +0 -0
  179. {rucio-37.4.0.dist-info → rucio-37.6.0.dist-info}/top_level.txt +0 -0
@@ -33,7 +33,7 @@ from rucio.common.logging import setup_logging
33
33
  from rucio.core.monitor import MetricManager
34
34
  from rucio.core.rule import get_injected_rules, inject_rule, update_rule
35
35
  from rucio.daemons.common import HeartbeatHandler, run_daemon
36
- from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX
36
+ from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
37
37
 
38
38
  if TYPE_CHECKING:
39
39
  from types import FrameType
@@ -99,7 +99,7 @@ def run_once(
99
99
  inject_rule(rule_id=rule_id, logger=logger)
100
100
  logger(logging.DEBUG, 'injection of %s took %f' % (rule_id, time.time() - start))
101
101
  except (DatabaseException, DatabaseError) as e:
102
- if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])):
102
+ if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
103
103
  paused_rules[rule_id] = datetime.utcnow() + timedelta(seconds=randint(60, 600)) # noqa: S311
104
104
  METRICS.counter('exceptions.{exception}').labels(exception='LocksDetected').inc()
105
105
  logger(logging.WARNING, 'Locks detected for %s' % rule_id)
@@ -35,7 +35,7 @@ from rucio.common.logging import setup_logging
35
35
  from rucio.core.monitor import MetricManager
36
36
  from rucio.core.rule import get_stuck_rules, repair_rule
37
37
  from rucio.daemons.common import HeartbeatHandler, run_daemon
38
- from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX
38
+ from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_CONNECTION_LOST_CONTACT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
39
39
 
40
40
  if TYPE_CHECKING:
41
41
  from types import FrameType
@@ -106,7 +106,7 @@ def run_once(
106
106
  repair_rule(rule_id=rule_id)
107
107
  logger(logging.DEBUG, 'repairing of %s took %f' % (rule_id, time.time() - start))
108
108
  except (DatabaseException, DatabaseError) as e:
109
- if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])):
109
+ if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
110
110
  paused_rules[rule_id] = datetime.utcnow() + timedelta(seconds=randint(600, 2400)) # noqa: S311
111
111
  logger(logging.WARNING, 'Locks detected for %s' % (rule_id))
112
112
  METRICS.counter('exceptions.{exception}').labels(exception='LocksDetected').inc()
@@ -198,7 +198,7 @@ def run_once(heartbeat_handler: Any, younger_than: int, nattempts: int, vos: "Op
198
198
  try:
199
199
  json_file = open(json_file_name, mode="r")
200
200
  logger(logging.INFO, "JSON file has been opened.")
201
- except:
201
+ except Exception:
202
202
  logger(logging.WARNING, "An error occurred while trying to open the JSON file.")
203
203
  must_sleep = True
204
204
  return must_sleep
@@ -166,7 +166,7 @@ class Stats:
166
166
  try:
167
167
  with open(self.path, "r") as f:
168
168
  data = f.read()
169
- except:
169
+ except Exception:
170
170
  data = ""
171
171
  data = json.loads(data or "{}")
172
172
  data.update(self.Data)
@@ -332,7 +332,7 @@ def was_cc_attempted(
332
332
  ) -> Optional[bool]:
333
333
  try:
334
334
  f = open(stats_file, "r")
335
- except:
335
+ except Exception:
336
336
  print("get_data: error ", stats_file)
337
337
  return None
338
338
  stats = json.loads(f.read())
@@ -347,7 +347,7 @@ def was_cc_processed(
347
347
  ) -> Optional[bool]:
348
348
  try:
349
349
  f = open(stats_file, "r")
350
- except:
350
+ except Exception:
351
351
  print("get_data: error ", stats_file)
352
352
  return None
353
353
  stats = json.loads(f.read())
@@ -492,7 +492,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
492
492
  # List all the active subscriptions
493
493
  subscriptions = get_subscriptions(logger=logger)
494
494
 
495
- # Loop over all the new dids
495
+ # Loop over all the new DIDs
496
496
  # Get the new DIDs based on the is_new flag
497
497
  logger(logging.DEBUG, "Listing new dids")
498
498
  for did in list_new_dids(
@@ -13,7 +13,7 @@
13
13
  # limitations under the License.
14
14
 
15
15
  '''
16
- Undertaker is a daemon to manage expired did.
16
+ Undertaker is a daemon to manage expired DID.
17
17
  '''
18
18
 
19
19
  import functools
@@ -36,7 +36,7 @@ from rucio.common.utils import chunks
36
36
  from rucio.core.did import delete_dids, list_expired_dids
37
37
  from rucio.core.monitor import MetricManager
38
38
  from rucio.daemons.common import HeartbeatHandler, run_daemon
39
- from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX
39
+ from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX, PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX
40
40
 
41
41
  if TYPE_CHECKING:
42
42
  from types import FrameType
@@ -51,7 +51,7 @@ DAEMON_NAME = 'undertaker'
51
51
 
52
52
  def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10) -> None:
53
53
  """
54
- Main loop to select and delete dids.
54
+ Main loop to select and delete DIDs.
55
55
  """
56
56
  paused_dids = {} # {(scope, name): datetime}
57
57
  run_daemon(
@@ -72,7 +72,7 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
72
72
  worker_number, total_workers, logger = heartbeat_handler.live()
73
73
 
74
74
  try:
75
- # Refresh paused dids
75
+ # Refresh paused DIDs
76
76
  iter_paused_dids = deepcopy(paused_dids)
77
77
  for key in iter_paused_dids:
78
78
  if datetime.utcnow() > paused_dids[key]:
@@ -96,14 +96,14 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
96
96
  except RuleNotFound as error:
97
97
  logger(logging.ERROR, error)
98
98
  except (DatabaseException, DatabaseError, UnsupportedOperation) as e:
99
- if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
99
+ if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
100
100
  for did in chunk:
101
101
  paused_dids[(did['scope'], did['name'])] = datetime.utcnow() + timedelta(seconds=randint(600, 2400)) # noqa: S311
102
102
  METRICS.counter('delete_dids.exceptions.{exception}').labels(exception='LocksDetected').inc()
103
103
  logger(logging.WARNING, 'Locks detected for chunk')
104
104
  else:
105
105
  logger(logging.ERROR, 'Got database error %s.', str(e))
106
- except:
106
+ except Exception:
107
107
  logging.critical(traceback.format_exc())
108
108
 
109
109
 
@@ -26,6 +26,7 @@ ORACLE_DEADLOCK_DETECTED_REGEX = r".*ORA-00060.*"
26
26
  ORACLE_RESOURCE_BUSY_REGEX = r".*ORA-00054.*"
27
27
  ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX = r".*ORA-00001.*"
28
28
  PSQL_LOCK_NOT_AVAILABLE_REGEX = r".*55P03.*"
29
+ PSQL_PSYCOPG_LOCK_NOT_AVAILABLE_REGEX = r".*psycopg.errors.LockNotAvailable.*"
29
30
  MYSQL_LOCK_NOWAIT_REGEX = r".*3572.*"
30
31
  MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED = "ERROR 1205 (HY000)"
31
32
 
@@ -196,9 +197,9 @@ class SubscriptionState(Enum):
196
197
  BROKEN = 'B'
197
198
 
198
199
 
199
- class TransferLimitDirection(Enum):
200
- SOURCE = 'S'
201
- DESTINATION = 'D'
200
+ # class TransferLimitDirection(Enum):
201
+ # SOURCE = 'S'
202
+ # DESTINATION = 'D'
202
203
 
203
204
 
204
205
  class DatabaseOperationType(Enum):
@@ -19,7 +19,7 @@ import sqlalchemy as sa
19
19
  from alembic import context
20
20
  from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
21
21
 
22
- from rucio.db.sqla.constants import TransferLimitDirection
22
+ from rucio.common.constants import TransferLimitDirection
23
23
  from rucio.db.sqla.types import GUID
24
24
 
25
25
  # Alembic revision identifiers
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- ''' add convention table and closed_at to dids '''
15
+ ''' add convention table and closed_at to DIDs '''
16
16
 
17
17
  import datetime
18
18
 
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- ''' remove temporary dids '''
15
+ ''' remove temporary DIDs '''
16
16
 
17
17
  import sqlalchemy as sa
18
18
  from alembic import context
@@ -12,7 +12,7 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- ''' add lumiblocknr to dids '''
15
+ ''' add lumiblocknr to DIDs '''
16
16
 
17
17
  import sqlalchemy as sa
18
18
  from alembic import context
rucio/db/sqla/models.py CHANGED
@@ -29,6 +29,7 @@ from sqlalchemy.types import LargeBinary
29
29
  # and it must be renamed to avoid conflicts with the policy package schema modules
30
30
  from rucio.common import schema as common_schema
31
31
  from rucio.common import utils
32
+ from rucio.common.constants import TransferLimitDirection
32
33
  from rucio.common.types import InternalAccount, InternalScope # noqa: TCH001 (types are needed by SQLAlchemy)
33
34
  from rucio.db.sqla.constants import (
34
35
  AccountStatus,
@@ -51,7 +52,6 @@ from rucio.db.sqla.constants import (
51
52
  RuleState,
52
53
  ScopeStatus,
53
54
  SubscriptionState,
54
- TransferLimitDirection,
55
55
  )
56
56
  from rucio.db.sqla.session import BASE
57
57
  from rucio.db.sqla.types import GUID, JSON, BooleanString, InternalAccountString, InternalScopeString
@@ -563,7 +563,7 @@ class DeletedDataIdentifier(BASE, ModelBase):
563
563
 
564
564
 
565
565
  class UpdatedDID(BASE, ModelBase):
566
- """Represents the recently updated dids"""
566
+ """Represents the recently updated DIDs"""
567
567
  __tablename__ = 'updated_dids'
568
568
  id: Mapped[str] = mapped_column(GUID(), default=utils.generate_uuid)
569
569
  scope: Mapped[InternalScope] = mapped_column(InternalScopeString(common_schema.get_schema_value('SCOPE_LENGTH')))
rucio/db/sqla/session.py CHANGED
@@ -54,7 +54,7 @@ if TYPE_CHECKING:
54
54
  try:
55
55
  main_script = os.path.basename(sys.argv[0])
56
56
  CURRENT_COMPONENT = main_script.split('-')[1]
57
- except:
57
+ except Exception:
58
58
  CURRENT_COMPONENT = None
59
59
 
60
60
  DATABASE_SECTION = 'database'
@@ -63,7 +63,7 @@ try:
63
63
  sql_connection = config_get('%s-database' % CURRENT_COMPONENT, 'default', check_config_table=False).strip()
64
64
  if sql_connection and len(sql_connection):
65
65
  DATABASE_SECTION = '%s-database' % CURRENT_COMPONENT
66
- except:
66
+ except Exception:
67
67
  pass
68
68
 
69
69
  DEFAULT_SCHEMA_NAME = config_get(DATABASE_SECTION, 'schema',
@@ -221,7 +221,7 @@ def get_engine() -> 'Engine':
221
221
  for param, param_type in config_params:
222
222
  try:
223
223
  params[param] = param_type(config_get(DATABASE_SECTION, param, check_config_table=False))
224
- except:
224
+ except Exception:
225
225
  pass
226
226
  _ENGINE = create_engine(sql_connection, **params)
227
227
  if 'mysql' in sql_connection:
@@ -406,7 +406,7 @@ def read_session(function: "Callable[P, R]"):
406
406
  except DatabaseError as error:
407
407
  session.rollback() # type: ignore
408
408
  raise DatabaseException(str(error))
409
- except:
409
+ except Exception:
410
410
  session.rollback() # type: ignore
411
411
  raise
412
412
  finally:
@@ -451,7 +451,7 @@ def stream_session(function: "Callable[P, R]"):
451
451
  except DatabaseError as error:
452
452
  session.rollback() # type: ignore
453
453
  raise DatabaseException(str(error))
454
- except:
454
+ except Exception:
455
455
  session.rollback() # type: ignore
456
456
  raise
457
457
  finally:
@@ -460,7 +460,7 @@ def stream_session(function: "Callable[P, R]"):
460
460
  try:
461
461
  for row in function(*args, session=session, **kwargs):
462
462
  yield row
463
- except:
463
+ except Exception:
464
464
  raise
465
465
  return _update_session_wrapper(new_funct, function)
466
466
 
@@ -491,7 +491,7 @@ def transactional_session(function: "Callable[P, R]") -> 'Callable':
491
491
  except DatabaseError as error:
492
492
  session.rollback() # type: ignore
493
493
  raise DatabaseException(str(error))
494
- except:
494
+ except Exception:
495
495
  session.rollback() # type: ignore
496
496
  raise
497
497
  finally:
rucio/gateway/account.py CHANGED
@@ -22,27 +22,22 @@ from rucio.common.types import InternalAccount
22
22
  from rucio.common.utils import gateway_update_return_dict
23
23
  from rucio.core import account as account_core
24
24
  from rucio.core.rse import get_rse_id
25
- from rucio.db.sqla.constants import AccountType
26
- from rucio.db.sqla.session import read_session, stream_session, transactional_session
25
+ from rucio.db.sqla.constants import AccountType, DatabaseOperationType
26
+ from rucio.db.sqla.session import db_session
27
27
 
28
28
  if TYPE_CHECKING:
29
29
  from collections.abc import Iterator
30
30
 
31
- from sqlalchemy.orm import Session
32
-
33
31
  from rucio.common.types import AccountAttributesDict, IdentityDict, UsageDict
34
32
  from rucio.db.sqla.models import Account
35
33
 
36
34
 
37
- @transactional_session
38
35
  def add_account(
39
36
  account: str,
40
37
  type_: str,
41
38
  email: str,
42
39
  issuer: str,
43
40
  vo: str = 'def',
44
- *,
45
- session: "Session"
46
41
  ) -> None:
47
42
  """
48
43
  Creates an account with the provided account name, contact information, etc.
@@ -53,29 +48,27 @@ def add_account(
53
48
 
54
49
  :param issuer: The issuer account_core.
55
50
  :param vo: The VO to act on.
56
- :param session: The database session in use.
57
51
 
58
52
  """
59
53
 
60
54
  validate_schema(name='account', obj=account, vo=vo)
61
55
 
62
56
  kwargs = {'account': account, 'type': type_}
63
- auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_account', kwargs=kwargs, session=session)
64
- if not auth_result.allowed:
65
- raise rucio.common.exception.AccessDenied('Account %s can not add account. %s' % (issuer, auth_result.message))
66
57
 
67
- internal_account = InternalAccount(account, vo=vo)
58
+ with db_session(DatabaseOperationType.WRITE) as session:
59
+ auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_account', kwargs=kwargs, session=session)
60
+ if not auth_result.allowed:
61
+ raise rucio.common.exception.AccessDenied('Account %s can not add account. %s' % (issuer, auth_result.message))
62
+
63
+ internal_account = InternalAccount(account, vo=vo)
68
64
 
69
- account_core.add_account(internal_account, AccountType[type_.upper()], email, session=session)
65
+ account_core.add_account(internal_account, AccountType[type_.upper()], email, session=session)
70
66
 
71
67
 
72
- @transactional_session
73
68
  def del_account(
74
69
  account: str,
75
70
  issuer: str,
76
71
  vo: str = 'def',
77
- *,
78
- session: "Session"
79
72
  ) -> None:
80
73
  """
81
74
  Disables an account with the provided account name.
@@ -83,25 +76,22 @@ def del_account(
83
76
  :param account: The account name.
84
77
  :param issuer: The issuer account.
85
78
  :param vo: The VO to act on.
86
- :param session: The database session in use.
87
79
 
88
80
  """
89
81
  kwargs = {'account': account}
90
- auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='del_account', kwargs=kwargs, session=session)
91
- if not auth_result.allowed:
92
- raise rucio.common.exception.AccessDenied('Account %s can not delete account. %s' % (issuer, auth_result.message))
82
+ with db_session(DatabaseOperationType.WRITE) as session:
83
+ auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='del_account', kwargs=kwargs, session=session)
84
+ if not auth_result.allowed:
85
+ raise rucio.common.exception.AccessDenied('Account %s can not delete account. %s' % (issuer, auth_result.message))
93
86
 
94
- internal_account = InternalAccount(account, vo=vo)
87
+ internal_account = InternalAccount(account, vo=vo)
95
88
 
96
- account_core.del_account(internal_account, session=session)
89
+ account_core.del_account(internal_account, session=session)
97
90
 
98
91
 
99
- @read_session
100
92
  def get_account_info(
101
93
  account: str,
102
94
  vo: str = 'def',
103
- *,
104
- session: "Session"
105
95
  ) -> "Account":
106
96
  """
107
97
  Returns the info like the statistics information associated to an account_core.
@@ -109,25 +99,23 @@ def get_account_info(
109
99
  :param account: The account name.
110
100
  :returns: A list with all account information.
111
101
  :param vo: The VO to act on.
112
- :param session: The database session in use.
102
+
113
103
  """
114
104
 
115
105
  internal_account = InternalAccount(account, vo=vo)
116
106
 
117
- acc = account_core.get_account(internal_account, session=session)
118
- acc.account = acc.account.external
119
- return acc
107
+ with db_session(DatabaseOperationType.READ) as session:
108
+ acc = account_core.get_account(internal_account, session=session)
109
+ acc.account = acc.account.external
110
+ return acc
120
111
 
121
112
 
122
- @transactional_session
123
113
  def update_account(
124
114
  account: str,
125
115
  key: str,
126
116
  value: Any,
127
117
  issuer: str = 'root',
128
118
  vo: str = 'def',
129
- *,
130
- session: "Session"
131
119
  ) -> None:
132
120
  """ Update a property of an account_core.
133
121
 
@@ -136,21 +124,21 @@ def update_account(
136
124
  :param value: Property value.
137
125
  :param issuer: The issuer account
138
126
  :param vo: The VO to act on.
139
- :param session: The database session in use.
127
+
140
128
  """
141
129
  validate_schema(name='account', obj=account, vo=vo)
142
130
  kwargs = {}
143
- auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='update_account', kwargs=kwargs, session=session)
144
- if not auth_result.allowed:
145
- raise rucio.common.exception.AccessDenied('Account %s can not change %s of the account. %s' % (issuer, key, auth_result.message))
131
+ with db_session(DatabaseOperationType.WRITE) as session:
132
+ auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='update_account', kwargs=kwargs, session=session)
133
+ if not auth_result.allowed:
134
+ raise rucio.common.exception.AccessDenied('Account %s can not change %s of the account. %s' % (issuer, key, auth_result.message))
146
135
 
147
- internal_account = InternalAccount(account, vo=vo)
136
+ internal_account = InternalAccount(account, vo=vo)
148
137
 
149
- return account_core.update_account(internal_account, key, value, session=session)
138
+ return account_core.update_account(internal_account, key, value, session=session)
150
139
 
151
140
 
152
- @stream_session
153
- def list_accounts(filter_: Optional[dict[str, Any]] = None, vo: str = 'def', *, session: "Session") -> 'Iterator[dict[str, Any]]':
141
+ def list_accounts(filter_: Optional[dict[str, Any]] = None, vo: str = 'def') -> 'Iterator[dict[str, Any]]':
154
142
  """
155
143
  Lists all the Rucio account names.
156
144
 
@@ -158,7 +146,6 @@ def list_accounts(filter_: Optional[dict[str, Any]] = None, vo: str = 'def', *,
158
146
 
159
147
  :param filter_: Dictionary of attributes by which the input data should be filtered
160
148
  :param vo: The VO to act on.
161
- :param session: The database session in use.
162
149
 
163
150
  :returns: List of all accounts.
164
151
  """
@@ -169,80 +156,73 @@ def list_accounts(filter_: Optional[dict[str, Any]] = None, vo: str = 'def', *,
169
156
  filter_['account'] = InternalAccount(filter_['account'], vo=vo)
170
157
  else:
171
158
  filter_['account'] = InternalAccount(account='*', vo=vo)
172
- for result in account_core.list_accounts(filter_=filter_, session=session):
173
- yield gateway_update_return_dict(result, session=session)
159
+
160
+ with db_session(DatabaseOperationType.READ) as session:
161
+ for result in account_core.list_accounts(filter_=filter_, session=session):
162
+ yield gateway_update_return_dict(result, session=session)
174
163
 
175
164
 
176
- @read_session
177
165
  def account_exists(
178
166
  account: str,
179
167
  vo: str = 'def',
180
- *,
181
- session: "Session"
182
168
  ) -> bool:
183
169
  """
184
170
  Checks to see if account exists. This procedure does not check it's status.
185
171
 
186
172
  :param account: Name of the account.
187
173
  :param vo: The VO to act on.
188
- :param session: The database session in use.
174
+
189
175
  :returns: True if found, otherwise false.
190
176
  """
191
177
 
192
178
  internal_account = InternalAccount(account, vo=vo)
193
179
 
194
- return account_core.account_exists(internal_account, session=session)
180
+ with db_session(DatabaseOperationType.READ) as session:
181
+ return account_core.account_exists(internal_account, session=session)
195
182
 
196
183
 
197
- @read_session
198
184
  def list_identities(
199
185
  account: str,
200
186
  vo: str = 'def',
201
- *,
202
- session: "Session"
203
187
  ) -> list["IdentityDict"]:
204
188
  """
205
189
  List all identities on an account_core.
206
190
 
207
191
  :param account: The account name.
208
192
  :param vo: The VO to act on.
209
- :param session: The database session in use.
193
+
210
194
  """
211
195
 
212
196
  internal_account = InternalAccount(account, vo=vo)
213
197
 
214
- return account_core.list_identities(internal_account, session=session)
198
+ with db_session(DatabaseOperationType.READ) as session:
199
+ return account_core.list_identities(internal_account, session=session)
215
200
 
216
201
 
217
- @read_session
218
202
  def list_account_attributes(
219
203
  account: str,
220
204
  vo: str = 'def',
221
- *,
222
- session: "Session"
223
205
  ) -> list["AccountAttributesDict"]:
224
206
  """
225
207
  Returns all the attributes for the given account.
226
208
 
227
209
  :param account: The account name.
228
210
  :param vo: The VO to act on
229
- :param session: The database session in use.
211
+
230
212
  """
231
213
 
232
214
  internal_account = InternalAccount(account, vo=vo)
233
215
 
234
- return account_core.list_account_attributes(internal_account, session=session)
216
+ with db_session(DatabaseOperationType.READ) as session:
217
+ return account_core.list_account_attributes(internal_account, session=session)
235
218
 
236
219
 
237
- @transactional_session
238
220
  def add_account_attribute(
239
221
  key: str,
240
222
  value: Any,
241
223
  account: str,
242
224
  issuer: str,
243
225
  vo: str = 'def',
244
- *,
245
- session: "Session"
246
226
  ) -> None:
247
227
  """
248
228
  Add an attribute to an account.
@@ -252,29 +232,27 @@ def add_account_attribute(
252
232
  :param account: The account name.
253
233
  :param issuer: The issuer account.
254
234
  :param vo: The VO to act on.
255
- :param session: The database session in use.
235
+
256
236
  """
257
237
  validate_schema(name='account_attribute', obj=key, vo=vo)
258
238
  validate_schema(name='account_attribute', obj=value, vo=vo)
259
239
 
260
240
  kwargs = {'account': account, 'key': key, 'value': value}
261
- auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_attribute', kwargs=kwargs, session=session)
262
- if not auth_result.allowed:
263
- raise rucio.common.exception.AccessDenied('Account %s can not add attributes. %s' % (issuer, auth_result.message))
241
+ with db_session(DatabaseOperationType.WRITE) as session:
242
+ auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_attribute', kwargs=kwargs, session=session)
243
+ if not auth_result.allowed:
244
+ raise rucio.common.exception.AccessDenied('Account %s can not add attributes. %s' % (issuer, auth_result.message))
264
245
 
265
- internal_account = InternalAccount(account, vo=vo)
246
+ internal_account = InternalAccount(account, vo=vo)
266
247
 
267
- account_core.add_account_attribute(internal_account, key, value, session=session)
248
+ account_core.add_account_attribute(internal_account, key, value, session=session)
268
249
 
269
250
 
270
- @transactional_session
271
251
  def del_account_attribute(
272
252
  key: str,
273
253
  account: str,
274
254
  issuer: str,
275
255
  vo: str = 'def',
276
- *,
277
- session: "Session"
278
256
  ) -> None:
279
257
  """
280
258
  Delete an attribute to an account.
@@ -283,26 +261,24 @@ def del_account_attribute(
283
261
  :param account: The account name.
284
262
  :param issuer: The issuer account.
285
263
  :param vo: The VO to act on.
286
- :param session: The database session in use.
264
+
287
265
  """
288
266
  kwargs = {'account': account, 'key': key}
289
- auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='del_attribute', kwargs=kwargs, session=session)
290
- if not auth_result.allowed:
291
- raise rucio.common.exception.AccessDenied('Account %s can not delete attribute. %s' % (issuer, auth_result.message))
267
+ with db_session(DatabaseOperationType.WRITE) as session:
268
+ auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='del_attribute', kwargs=kwargs, session=session)
269
+ if not auth_result.allowed:
270
+ raise rucio.common.exception.AccessDenied('Account %s can not delete attribute. %s' % (issuer, auth_result.message))
292
271
 
293
- internal_account = InternalAccount(account, vo=vo)
272
+ internal_account = InternalAccount(account, vo=vo)
294
273
 
295
- account_core.del_account_attribute(internal_account, key, session=session)
274
+ account_core.del_account_attribute(internal_account, key, session=session)
296
275
 
297
276
 
298
- @read_session
299
277
  def get_usage(
300
278
  rse: str,
301
279
  account: str,
302
280
  issuer: str,
303
281
  vo: str = 'def',
304
- *,
305
- session: "Session"
306
282
  ) -> "UsageDict":
307
283
  """
308
284
  Returns current values of the specified counter, or raises CounterNotFound if the counter does not exist.
@@ -311,23 +287,21 @@ def get_usage(
311
287
  :param account: The account name.
312
288
  :param issuer: The issuer account.
313
289
  :param vo: The VO to act on.
314
- :param session: The database session in use.
290
+
315
291
  :returns: A dictionary with total and bytes.
316
292
  """
317
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
318
- internal_account = InternalAccount(account, vo=vo)
293
+ with db_session(DatabaseOperationType.READ) as session:
294
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
295
+ internal_account = InternalAccount(account, vo=vo)
319
296
 
320
- return account_core.get_usage(rse_id, internal_account, session=session)
297
+ return account_core.get_usage(rse_id, internal_account, session=session)
321
298
 
322
299
 
323
- @read_session
324
300
  def get_usage_history(
325
301
  rse: str,
326
302
  account: str,
327
303
  issuer: str,
328
304
  vo: str = 'def',
329
- *,
330
- session: "Session"
331
305
  ) -> list["UsageDict"]:
332
306
  """
333
307
  Returns historical values of the specified counter, or raises CounterNotFound if the counter does not exist.
@@ -336,10 +310,11 @@ def get_usage_history(
336
310
  :param account: The account name.
337
311
  :param issuer: The issuer account.
338
312
  :param vo: The VO to act on.
339
- :param session: The database session in use.
313
+
340
314
  :returns: A dictionary with total and bytes.
341
315
  """
342
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
343
- internal_account = InternalAccount(account, vo=vo)
316
+ with db_session(DatabaseOperationType.READ) as session:
317
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
318
+ internal_account = InternalAccount(account, vo=vo)
344
319
 
345
- return account_core.get_usage_history(rse_id, internal_account, session=session)
320
+ return account_core.get_usage_history(rse_id, internal_account, session=session)