rucio 37.2.0__py3-none-any.whl → 37.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (95) hide show
  1. rucio/common/plugins.py +1 -1
  2. rucio/core/did.py +2 -3
  3. rucio/core/replica.py +1 -1
  4. rucio/db/sqla/util.py +1 -1
  5. rucio/gateway/authentication.py +58 -88
  6. rucio/gateway/config.py +63 -75
  7. rucio/gateway/did.py +245 -329
  8. rucio/gateway/dirac.py +33 -34
  9. rucio/gateway/exporter.py +27 -30
  10. rucio/gateway/importer.py +12 -14
  11. rucio/gateway/lifetime_exception.py +16 -24
  12. rucio/gateway/lock.py +27 -40
  13. rucio/gateway/replica.py +223 -226
  14. rucio/gateway/rse.py +191 -218
  15. rucio/gateway/rule.py +115 -146
  16. rucio/gateway/scope.py +18 -25
  17. rucio/gateway/trace.py +48 -0
  18. rucio/vcsversion.py +3 -3
  19. rucio/web/rest/flaskapi/v1/accounts.py +2 -2
  20. rucio/web/rest/flaskapi/v1/auth.py +15 -0
  21. rucio/web/rest/flaskapi/v1/common.py +3 -0
  22. rucio/web/rest/flaskapi/v1/config.py +7 -7
  23. rucio/web/rest/flaskapi/v1/dids.py +55 -55
  24. rucio/web/rest/flaskapi/v1/dirac.py +2 -2
  25. rucio/web/rest/flaskapi/v1/export.py +1 -1
  26. rucio/web/rest/flaskapi/v1/import.py +1 -1
  27. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +5 -5
  28. rucio/web/rest/flaskapi/v1/locks.py +4 -4
  29. rucio/web/rest/flaskapi/v1/main.py +17 -10
  30. rucio/web/rest/flaskapi/v1/replicas.py +29 -28
  31. rucio/web/rest/flaskapi/v1/rses.py +37 -37
  32. rucio/web/rest/flaskapi/v1/rules.py +15 -15
  33. rucio/web/rest/flaskapi/v1/scopes.py +3 -3
  34. rucio/web/rest/flaskapi/v1/traces.py +75 -77
  35. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/METADATA +1 -1
  36. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/RECORD +95 -94
  37. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/WHEEL +1 -1
  38. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  39. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  40. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  41. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
  42. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  43. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  44. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  45. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  46. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  47. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  48. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  49. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  50. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  51. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  52. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/requirements.server.txt +0 -0
  53. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/tools/bootstrap.py +0 -0
  54. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  55. {rucio-37.2.0.data → rucio-37.3.0.data}/data/rucio/tools/reset_database.py +0 -0
  56. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio +0 -0
  57. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-abacus-account +0 -0
  58. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-abacus-collection-replica +0 -0
  59. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-abacus-rse +0 -0
  60. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-admin +0 -0
  61. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-atropos +0 -0
  62. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-auditor +0 -0
  63. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-automatix +0 -0
  64. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-bb8 +0 -0
  65. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-cache-client +0 -0
  66. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-cache-consumer +0 -0
  67. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-finisher +0 -0
  68. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-poller +0 -0
  69. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-preparer +0 -0
  70. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-receiver +0 -0
  71. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-stager +0 -0
  72. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-submitter +0 -0
  73. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-conveyor-throttler +0 -0
  74. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-dark-reaper +0 -0
  75. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-dumper +0 -0
  76. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-follower +0 -0
  77. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-hermes +0 -0
  78. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-judge-cleaner +0 -0
  79. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-judge-evaluator +0 -0
  80. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-judge-injector +0 -0
  81. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-judge-repairer +0 -0
  82. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-kronos +0 -0
  83. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-minos +0 -0
  84. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
  85. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-necromancer +0 -0
  86. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-oauth-manager +0 -0
  87. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-reaper +0 -0
  88. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-replica-recoverer +0 -0
  89. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-rse-decommissioner +0 -0
  90. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-storage-consistency-actions +0 -0
  91. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-transmogrifier +0 -0
  92. {rucio-37.2.0.data → rucio-37.3.0.data}/scripts/rucio-undertaker +0 -0
  93. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/licenses/AUTHORS.rst +0 -0
  94. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/licenses/LICENSE +0 -0
  95. {rucio-37.2.0.dist-info → rucio-37.3.0.dist-info}/top_level.txt +0 -0
rucio/gateway/replica.py CHANGED
@@ -22,32 +22,35 @@ from rucio.common.types import InternalAccount, InternalScope
22
22
  from rucio.common.utils import gateway_update_return_dict, invert_dict
23
23
  from rucio.core import replica
24
24
  from rucio.core.rse import get_rse_id, get_rse_name
25
- from rucio.db.sqla.constants import BadFilesStatus
26
- from rucio.db.sqla.session import read_session, stream_session, transactional_session
25
+ from rucio.db.sqla.constants import BadFilesStatus, DatabaseOperationType
26
+ from rucio.db.sqla.session import db_session
27
27
  from rucio.gateway import permission
28
28
 
29
29
  if TYPE_CHECKING:
30
30
  from collections.abc import Iterator
31
31
 
32
- from sqlalchemy.orm import Session
33
32
 
34
-
35
- @read_session
36
- def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None, vo='def', *, session: "Session"):
33
+ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None, vo='def'):
37
34
  """
38
35
  List the bad file replicas summary. Method used by the rucio-ui.
39
36
  :param rse_expression: The RSE expression.
40
37
  :param from_date: The start date.
41
38
  :param to_date: The end date.
42
39
  :param vo: the VO to act on.
43
- :param session: The database session in use.
44
40
  """
45
- replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date, filter_={'vo': vo}, session=session)
46
- return [gateway_update_return_dict(r, session=session) for r in replicas]
41
+ with db_session(DatabaseOperationType.READ) as session:
42
+ replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date, filter_={'vo': vo}, session=session)
43
+ return [gateway_update_return_dict(r, session=session) for r in replicas]
47
44
 
48
45
 
49
- @read_session
50
- def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=None, older_than=None, limit=None, list_pfns=False, vo='def', *, session: "Session"):
46
+ def list_bad_replicas_status(
47
+ state: Optional[BadFilesStatus] = BadFilesStatus.BAD,
48
+ rse: Optional[str] = None,
49
+ younger_than: Optional[datetime.datetime] = None,
50
+ older_than: Optional[datetime.datetime] = None,
51
+ limit: Optional[int] = None,
52
+ list_pfns: bool = False,
53
+ vo: str = 'def'):
51
54
  """
52
55
  List the bad file replicas history states. Method used by the rucio-ui.
53
56
  :param state: The state of the file (SUSPICIOUS or BAD).
@@ -56,19 +59,19 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=No
56
59
  :param older_than: datetime object to select bad replicas older than this date.
57
60
  :param limit: The maximum number of replicas returned.
58
61
  :param vo: The VO to act on.
59
- :param session: The database session in use.
60
62
  """
61
63
  rse_id = None
62
- if rse is not None:
63
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
64
64
 
65
- replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than,
66
- older_than=older_than, limit=limit, list_pfns=list_pfns, vo=vo, session=session)
67
- return [gateway_update_return_dict(r, session=session) for r in replicas]
65
+ with db_session(DatabaseOperationType.READ) as session:
66
+ if rse is not None:
67
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
68
68
 
69
+ replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than,
70
+ older_than=older_than, limit=limit, list_pfns=list_pfns, vo=vo, session=session)
71
+ return [gateway_update_return_dict(r, session=session) for r in replicas]
69
72
 
70
- @transactional_session
71
- def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *, session: "Session"):
73
+
74
+ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False):
72
75
  """
73
76
  Declare a list of bad replicas.
74
77
 
@@ -78,7 +81,6 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
78
81
  :param issuer: The issuer account.
79
82
  :param vo: The VO to act on.
80
83
  :param force: boolean, ignore existing replica status in the bad_replicas table. Default: False
81
- :param session: The database session in use.
82
84
  :returns: Dictionary {rse_name -> [list of replicas failed to declare]}
83
85
  """
84
86
 
@@ -95,62 +97,63 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
95
97
 
96
98
  replicas_lst = replicas
97
99
  rse_ids_to_check = set() # to check for permission to declare bad replicas
98
- if as_pfns:
99
- scheme, rses_for_replicas, unknowns = replica.get_pfn_to_rse(replicas, vo=vo, session=session)
100
- if unknowns:
101
- raise exception.ReplicaNotFound("Not all replicas found")
102
- rse_ids_to_check = set(rses_for_replicas.keys())
103
- else:
104
- replicas_lst = []
105
- for r in replicas:
106
- if "name" not in r or "scope" not in r or ("rse" not in r and "rse_id" not in r):
107
- raise exception.InvalidType('The replica dictionary must include scope and either rse (name) or rse_id')
108
- scope = InternalScope(r['scope'], vo=vo)
109
- rse_id = r.get("rse_id") or rse_map.get(r['rse'])
110
- if rse_id is None:
111
- rse = r["rse"]
112
- rse_map[rse] = rse_id = get_rse_id(rse=rse, vo=vo, session=session)
113
- replicas_lst.append({
114
- "rse_id": rse_id,
115
- "scope": scope,
116
- "name": r["name"]
117
- })
118
- rse_ids_to_check.add(rse_id)
119
-
120
- rse_id_to_name = invert_dict(rse_map) # RSE id -> RSE name
121
-
122
- for rse_id in rse_ids_to_check:
123
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_bad_file_replicas',
124
- kwargs={"rse_id": rse_id},
125
- session=session)
126
- if not auth_result.allowed:
127
- raise exception.AccessDenied('Account %s can not declare bad replicas in RSE %s. %s' %
128
- (issuer, rse_id_to_name.get(rse_id, rse_id), auth_result.message))
129
-
130
- undeclared = replica.declare_bad_file_replicas(replicas_lst, reason=reason,
131
- issuer=InternalAccount(issuer, vo=vo),
132
- status=BadFilesStatus.BAD,
133
- force=force, session=session)
134
- out = {}
135
- for rse_id, ulist in undeclared.items():
136
- if ulist:
137
- rse_name = None
138
- if rse_id == 'unknown':
139
- rse_name = 'unknown'
140
- elif rse_id in rse_id_to_name:
141
- rse_name = rse_id_to_name[rse_id]
142
- else:
143
- try:
144
- rse_name = get_rse_name(rse_id=rse_id, session=session)
145
- except (ValueError, exception.RSENotFound):
146
- rse_name = str(rse_id)
147
- if rse_name:
148
- out[rse_name] = out.get(rse_name, []) + ulist
149
- return out
150
-
151
-
152
- @transactional_session
153
- def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def', *, session: "Session"):
100
+
101
+ with db_session(DatabaseOperationType.WRITE) as session:
102
+ if as_pfns:
103
+ scheme, rses_for_replicas, unknowns = replica.get_pfn_to_rse(replicas, vo=vo, session=session)
104
+ if unknowns:
105
+ raise exception.ReplicaNotFound("Not all replicas found")
106
+ rse_ids_to_check = set(rses_for_replicas.keys())
107
+ else:
108
+ replicas_lst = []
109
+ for r in replicas:
110
+ if "name" not in r or "scope" not in r or ("rse" not in r and "rse_id" not in r):
111
+ raise exception.InvalidType('The replica dictionary must include scope and either rse (name) or rse_id')
112
+ scope = InternalScope(r['scope'], vo=vo)
113
+ rse_id = r.get("rse_id") or rse_map.get(r['rse'])
114
+ if rse_id is None:
115
+ rse = r["rse"]
116
+ rse_map[rse] = rse_id = get_rse_id(rse=rse, vo=vo, session=session)
117
+ replicas_lst.append({
118
+ "rse_id": rse_id,
119
+ "scope": scope,
120
+ "name": r["name"]
121
+ })
122
+ rse_ids_to_check.add(rse_id)
123
+
124
+ rse_id_to_name = invert_dict(rse_map) # RSE id -> RSE name
125
+
126
+ for rse_id in rse_ids_to_check:
127
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_bad_file_replicas',
128
+ kwargs={"rse_id": rse_id},
129
+ session=session)
130
+ if not auth_result.allowed:
131
+ raise exception.AccessDenied('Account %s can not declare bad replicas in RSE %s. %s' %
132
+ (issuer, rse_id_to_name.get(rse_id, rse_id), auth_result.message))
133
+
134
+ undeclared = replica.declare_bad_file_replicas(replicas_lst, reason=reason,
135
+ issuer=InternalAccount(issuer, vo=vo),
136
+ status=BadFilesStatus.BAD,
137
+ force=force, session=session)
138
+ out = {}
139
+ for rse_id, ulist in undeclared.items():
140
+ if ulist:
141
+ rse_name = None
142
+ if rse_id == 'unknown':
143
+ rse_name = 'unknown'
144
+ elif rse_id in rse_id_to_name:
145
+ rse_name = rse_id_to_name[rse_id]
146
+ else:
147
+ try:
148
+ rse_name = get_rse_name(rse_id=rse_id, session=session)
149
+ except (ValueError, exception.RSENotFound):
150
+ rse_name = str(rse_id)
151
+ if rse_name:
152
+ out[rse_name] = out.get(rse_name, []) + ulist
153
+ return out
154
+
155
+
156
+ def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def'):
154
157
  """
155
158
  Declare a list of bad replicas.
156
159
 
@@ -158,54 +161,53 @@ def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def', *, session:
158
161
  :param reason: The reason of the loss.
159
162
  :param issuer: The issuer account.
160
163
  :param vo: The VO to act on.
161
- :param session: The database session in use.
162
164
  """
163
165
  kwargs = {}
164
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_suspicious_file_replicas', kwargs=kwargs, session=session)
165
- if not auth_result.allowed:
166
- raise exception.AccessDenied('Account %s can not declare suspicious replicas. %s' % (issuer, auth_result.message))
167
166
 
168
- issuer = InternalAccount(issuer, vo=vo)
167
+ with db_session(DatabaseOperationType.WRITE) as session:
168
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_suspicious_file_replicas', kwargs=kwargs, session=session)
169
+ if not auth_result.allowed:
170
+ raise exception.AccessDenied('Account %s can not declare suspicious replicas. %s' % (issuer, auth_result.message))
171
+
172
+ issuer = InternalAccount(issuer, vo=vo)
169
173
 
170
- replicas = replica.declare_bad_file_replicas(pfns, reason=reason, issuer=issuer, status=BadFilesStatus.SUSPICIOUS, session=session)
174
+ replicas = replica.declare_bad_file_replicas(pfns, reason=reason, issuer=issuer, status=BadFilesStatus.SUSPICIOUS, session=session)
171
175
 
172
- for k in list(replicas):
173
- try:
174
- rse = get_rse_name(rse_id=k, session=session)
175
- replicas[rse] = replicas.pop(k)
176
- except exception.RSENotFound:
177
- pass
176
+ for k in list(replicas):
177
+ try:
178
+ rse = get_rse_name(rse_id=k, session=session)
179
+ replicas[rse] = replicas.pop(k)
180
+ except exception.RSENotFound:
181
+ pass
178
182
 
179
183
  return replicas
180
184
 
181
185
 
182
- @stream_session
183
- def get_did_from_pfns(pfns, rse, vo='def', *, session: "Session"):
186
+ def get_did_from_pfns(pfns, rse, vo='def'):
184
187
  """
185
188
  Get the DIDs associated to a PFN on one given RSE
186
189
 
187
190
  :param pfns: The list of PFNs.
188
191
  :param rse: The RSE name.
189
192
  :param vo: The VO to act on.
190
- :param session: The database session in use.
191
193
  :returns: A dictionary {pfn: {'scope': scope, 'name': name}}
192
194
  """
193
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
194
- replicas = replica.get_did_from_pfns(pfns=pfns, rse_id=rse_id, vo=vo, session=session)
195
+ with db_session(DatabaseOperationType.READ) as session:
196
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
197
+ replicas = replica.get_did_from_pfns(pfns=pfns, rse_id=rse_id, vo=vo, session=session)
195
198
 
196
- for r in replicas:
197
- for k in r.keys():
198
- r[k]['scope'] = r[k]['scope'].external
199
- yield r
199
+ for r in replicas:
200
+ for k in r.keys():
201
+ r[k]['scope'] = r[k]['scope'].external
202
+ yield r
200
203
 
201
204
 
202
- @stream_session
203
205
  def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
204
206
  ignore_availability=True, all_states=False, rse_expression=None,
205
207
  client_location=None, domain=None, signature_lifetime=None,
206
208
  resolve_archives=True, resolve_parents=False,
207
209
  nrandom=None, updated_after=None,
208
- issuer=None, vo='def', *, session: "Session"):
210
+ issuer=None, vo='def'):
209
211
  """
210
212
  List file replicas for a list of data identifiers.
211
213
 
@@ -223,43 +225,43 @@ def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
223
225
  :param updated_after: datetime object (UTC time), only return replicas updated after this time
224
226
  :param issuer: The issuer account.
225
227
  :param vo: The VO to act on.
226
- :param session: The database session in use.
227
228
  """
228
229
  validate_schema(name='r_dids', obj=dids, vo=vo)
229
230
 
230
231
  # Allow selected authenticated users to retrieve signed URLs.
231
232
  # Unauthenticated users, or permission-less users will get the raw URL without the signature.
232
233
  sign_urls = False
233
- if permission.has_permission(issuer=issuer, vo=vo, action='get_signed_url', kwargs={}, session=session):
234
- sign_urls = True
235
234
 
236
- for d in dids:
237
- d['scope'] = InternalScope(d['scope'], vo=vo)
235
+ with db_session(DatabaseOperationType.READ) as session:
236
+ if permission.has_permission(issuer=issuer, vo=vo, action='get_signed_url', kwargs={}, session=session):
237
+ sign_urls = True
238
238
 
239
- replicas = replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
240
- request_id=request_id,
241
- ignore_availability=ignore_availability,
242
- all_states=all_states, rse_expression=rse_expression,
243
- client_location=client_location, domain=domain,
244
- sign_urls=sign_urls, signature_lifetime=signature_lifetime,
245
- resolve_archives=resolve_archives, resolve_parents=resolve_parents,
246
- nrandom=nrandom, updated_after=updated_after, by_rse_name=True, session=session)
239
+ for d in dids:
240
+ d['scope'] = InternalScope(d['scope'], vo=vo)
247
241
 
248
- for rep in replicas:
249
- rep['scope'] = rep['scope'].external
250
- if 'parents' in rep:
251
- new_parents = []
252
- for p in rep['parents']:
253
- scope, name = p.split(':')
254
- scope = InternalScope(scope, from_external=False).external
255
- new_parents.append('{}:{}'.format(scope, name))
256
- rep['parents'] = new_parents
242
+ replicas = replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
243
+ request_id=request_id,
244
+ ignore_availability=ignore_availability,
245
+ all_states=all_states, rse_expression=rse_expression,
246
+ client_location=client_location, domain=domain,
247
+ sign_urls=sign_urls, signature_lifetime=signature_lifetime,
248
+ resolve_archives=resolve_archives, resolve_parents=resolve_parents,
249
+ nrandom=nrandom, updated_after=updated_after, by_rse_name=True, session=session)
257
250
 
258
- yield rep
251
+ for rep in replicas:
252
+ rep['scope'] = rep['scope'].external
253
+ if 'parents' in rep:
254
+ new_parents = []
255
+ for p in rep['parents']:
256
+ scope, name = p.split(':')
257
+ scope = InternalScope(scope, from_external=False).external
258
+ new_parents.append('{}:{}'.format(scope, name))
259
+ rep['parents'] = new_parents
259
260
 
261
+ yield rep
260
262
 
261
- @transactional_session
262
- def add_replicas(rse, files, issuer, ignore_availability=False, vo='def', *, session: "Session"):
263
+
264
+ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
263
265
  """
264
266
  Bulk add file replicas.
265
267
 
@@ -268,7 +270,6 @@ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def', *, ses
268
270
  :param issuer: The issuer account.
269
271
  :param ignore_availability: Ignore blocked RSEs.
270
272
  :param vo: The VO to act on.
271
- :param session: The database session in use.
272
273
 
273
274
  :returns: True is successful, False otherwise
274
275
  """
@@ -276,26 +277,26 @@ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def', *, ses
276
277
  v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
277
278
  validate_schema(name='dids', obj=files, vo=vo)
278
279
 
279
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
280
+ with db_session(DatabaseOperationType.WRITE) as session:
281
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
280
282
 
281
- kwargs = {'rse': rse, 'rse_id': rse_id}
282
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_replicas', kwargs=kwargs, session=session)
283
- if not auth_result.allowed:
284
- raise exception.AccessDenied('Account %s can not add file replicas on %s. %s' % (issuer, rse, auth_result.message))
285
- if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
286
- ignore_availability = False
283
+ kwargs = {'rse': rse, 'rse_id': rse_id}
284
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_replicas', kwargs=kwargs, session=session)
285
+ if not auth_result.allowed:
286
+ raise exception.AccessDenied('Account %s can not add file replicas on %s. %s' % (issuer, rse, auth_result.message))
287
+ if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
288
+ ignore_availability = False
287
289
 
288
- issuer = InternalAccount(issuer, vo=vo)
289
- for f in files:
290
- f['scope'] = InternalScope(f['scope'], vo=vo)
291
- if 'account' in f:
292
- f['account'] = InternalAccount(f['account'], vo=vo)
290
+ issuer = InternalAccount(issuer, vo=vo)
291
+ for f in files:
292
+ f['scope'] = InternalScope(f['scope'], vo=vo)
293
+ if 'account' in f:
294
+ f['account'] = InternalAccount(f['account'], vo=vo)
293
295
 
294
- replica.add_replicas(rse_id=rse_id, files=files, account=issuer, ignore_availability=ignore_availability, session=session)
296
+ replica.add_replicas(rse_id=rse_id, files=files, account=issuer, ignore_availability=ignore_availability, session=session)
295
297
 
296
298
 
297
- @transactional_session
298
- def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def', *, session: "Session"):
299
+ def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
299
300
  """
300
301
  Bulk delete file replicas.
301
302
 
@@ -304,29 +305,28 @@ def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def', *,
304
305
  :param issuer: The issuer account.
305
306
  :param ignore_availability: Ignore blocked RSEs.
306
307
  :param vo: The VO to act on.
307
- :param session: The database session in use.
308
308
 
309
309
  :returns: True is successful, False otherwise
310
310
  """
311
311
  validate_schema(name='r_dids', obj=files, vo=vo)
312
312
 
313
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
313
+ with db_session(DatabaseOperationType.WRITE) as session:
314
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
314
315
 
315
- kwargs = {'rse': rse, 'rse_id': rse_id}
316
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='delete_replicas', kwargs=kwargs, session=session)
317
- if not auth_result.allowed:
318
- raise exception.AccessDenied('Account %s can not delete file replicas on %s. %s' % (issuer, rse, auth_result.message))
319
- if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
320
- ignore_availability = False
316
+ kwargs = {'rse': rse, 'rse_id': rse_id}
317
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='delete_replicas', kwargs=kwargs, session=session)
318
+ if not auth_result.allowed:
319
+ raise exception.AccessDenied('Account %s can not delete file replicas on %s. %s' % (issuer, rse, auth_result.message))
320
+ if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
321
+ ignore_availability = False
321
322
 
322
- for f in files:
323
- f['scope'] = InternalScope(f['scope'], vo=vo)
323
+ for f in files:
324
+ f['scope'] = InternalScope(f['scope'], vo=vo)
324
325
 
325
- replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability, session=session)
326
+ replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability, session=session)
326
327
 
327
328
 
328
- @transactional_session
329
- def update_replicas_states(rse, files, issuer, vo='def', *, session: "Session"):
329
+ def update_replicas_states(rse, files, issuer, vo='def'):
330
330
  """
331
331
  Update File replica information and state.
332
332
 
@@ -334,54 +334,51 @@ def update_replicas_states(rse, files, issuer, vo='def', *, session: "Session"):
334
334
  :param files: The list of files.
335
335
  :param issuer: The issuer account.
336
336
  :param vo: The VO to act on.
337
- :param session: The database session in use.
338
337
  """
339
338
  for v_file in files:
340
339
  v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
341
340
  validate_schema(name='dids', obj=files, vo=vo)
342
341
 
343
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
342
+ with db_session(DatabaseOperationType.WRITE) as session:
343
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
344
344
 
345
- kwargs = {'rse': rse, 'rse_id': rse_id}
346
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_replicas_states', kwargs=kwargs, session=session)
347
- if not auth_result.allowed:
348
- raise exception.AccessDenied('Account %s can not update file replicas state on %s. %s' % (issuer, rse, auth_result.message))
349
- replicas = []
350
- for file in files:
351
- rep = file
352
- rep['rse_id'] = rse_id
353
- rep['scope'] = InternalScope(rep['scope'], vo=vo)
354
- replicas.append(rep)
355
- replica.update_replicas_states(replicas=replicas, session=session)
345
+ kwargs = {'rse': rse, 'rse_id': rse_id}
346
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_replicas_states', kwargs=kwargs, session=session)
347
+ if not auth_result.allowed:
348
+ raise exception.AccessDenied('Account %s can not update file replicas state on %s. %s' % (issuer, rse, auth_result.message))
349
+ replicas = []
350
+ for file in files:
351
+ rep = file
352
+ rep['rse_id'] = rse_id
353
+ rep['scope'] = InternalScope(rep['scope'], vo=vo)
354
+ replicas.append(rep)
355
+ replica.update_replicas_states(replicas=replicas, session=session)
356
356
 
357
357
 
358
- @stream_session
359
- def list_dataset_replicas(scope, name, deep=False, vo='def', *, session: "Session"):
358
+ def list_dataset_replicas(scope, name, deep=False, vo='def'):
360
359
  """
361
360
  :param scope: The scope of the dataset.
362
361
  :param name: The name of the dataset.
363
362
  :param deep: Lookup at the file level.
364
363
  :param vo: The VO to act on.
365
- :param session: The database session in use.
366
364
 
367
365
  :returns: A list of dict dataset replicas
368
366
  """
369
367
 
370
368
  scope = InternalScope(scope, vo=vo)
371
369
 
372
- replicas = replica.list_dataset_replicas(scope=scope, name=name, deep=deep, session=session)
370
+ with db_session(DatabaseOperationType.READ) as session:
371
+ replicas = replica.list_dataset_replicas(scope=scope, name=name, deep=deep, session=session)
373
372
 
374
- for r in replicas:
375
- r['scope'] = r['scope'].external
376
- yield r
373
+ for r in replicas:
374
+ r['scope'] = r['scope'].external
375
+ yield r
377
376
 
378
377
 
379
- @stream_session
380
- def list_dataset_replicas_bulk(dids, vo='def', *, session: "Session"):
378
+ def list_dataset_replicas_bulk(dids, vo='def'):
381
379
  """
382
380
  :param dids: The list of did dictionaries with scope and name.
383
381
  :param vo: The VO to act on.
384
- :param session: The database session in use.
385
382
 
386
383
  :returns: A list of dict dataset replicas
387
384
  """
@@ -399,20 +396,19 @@ def list_dataset_replicas_bulk(dids, vo='def', *, session: "Session"):
399
396
  internal_scope = InternalScope(scope, vo=vo)
400
397
  names_by_intscope[internal_scope] = names_by_scope[scope]
401
398
 
402
- replicas = replica.list_dataset_replicas_bulk(names_by_intscope, session=session)
399
+ with db_session(DatabaseOperationType.READ) as session:
400
+ replicas = replica.list_dataset_replicas_bulk(names_by_intscope, session=session)
403
401
 
404
- for r in replicas:
405
- yield gateway_update_return_dict(r, session=session)
402
+ for r in replicas:
403
+ yield gateway_update_return_dict(r, session=session)
406
404
 
407
405
 
408
- @stream_session
409
- def list_dataset_replicas_vp(scope, name, deep=False, vo='def', *, session: "Session"):
406
+ def list_dataset_replicas_vp(scope, name, deep=False, vo='def'):
410
407
  """
411
408
  :param scope: The scope of the dataset.
412
409
  :param name: The name of the dataset.
413
410
  :param deep: Lookup at the file level.
414
411
  :param vo: The vo to act on.
415
- :param session: The database session in use.
416
412
 
417
413
  :returns: If VP exists a list of dicts of sites, otherwise nothing
418
414
 
@@ -420,33 +416,34 @@ def list_dataset_replicas_vp(scope, name, deep=False, vo='def', *, session: "Ses
420
416
  """
421
417
 
422
418
  scope = InternalScope(scope, vo=vo)
423
- for r in replica.list_dataset_replicas_vp(scope=scope, name=name, deep=deep, session=session):
424
- yield gateway_update_return_dict(r, session=session)
419
+
420
+ with db_session(DatabaseOperationType.READ) as session:
421
+ for r in replica.list_dataset_replicas_vp(scope=scope, name=name, deep=deep, session=session):
422
+ yield gateway_update_return_dict(r, session=session)
425
423
 
426
424
 
427
- @stream_session
428
- def list_datasets_per_rse(rse: str, filters: Optional[dict[str, Any]] = None, limit: Optional[int] = None, vo: str = 'def', *, session: "Session") -> 'Iterator[dict[str, Any]]':
425
+ def list_datasets_per_rse(rse: str, filters: Optional[dict[str, Any]] = None, limit: Optional[int] = None, vo: str = 'def') -> 'Iterator[dict[str, Any]]':
429
426
  """
430
427
  :param scope: The scope of the dataset.
431
428
  :param name: The name of the dataset.
432
429
  :param filters: dictionary of attributes by which the results should be filtered.
433
430
  :param limit: limit number.
434
431
  :param vo: The VO to act on.
435
- :param session: The database session in use.
436
432
 
437
433
  :returns: A list of dict dataset replicas
438
434
  """
439
435
 
440
436
  filters = filters or {}
441
- rse_id = get_rse_id(rse=rse, vo=vo, session=session)
442
- if 'scope' in filters:
443
- filters['scope'] = InternalScope(filters['scope'], vo=vo)
444
- for r in replica.list_datasets_per_rse(rse_id, filters=filters, limit=limit, session=session):
445
- yield gateway_update_return_dict(r, session=session)
437
+
438
+ with db_session(DatabaseOperationType.READ) as session:
439
+ rse_id = get_rse_id(rse=rse, vo=vo, session=session)
440
+ if 'scope' in filters:
441
+ filters['scope'] = InternalScope(filters['scope'], vo=vo)
442
+ for r in replica.list_datasets_per_rse(rse_id, filters=filters, limit=limit, session=session):
443
+ yield gateway_update_return_dict(r, session=session)
446
444
 
447
445
 
448
- @transactional_session
449
- def add_bad_pfns(pfns, issuer, state, reason=None, expires_at=None, vo='def', *, session: "Session"):
446
+ def add_bad_pfns(pfns, issuer, state, reason=None, expires_at=None, vo='def'):
450
447
  """
451
448
  Add bad PFNs.
452
449
 
@@ -456,25 +453,25 @@ def add_bad_pfns(pfns, issuer, state, reason=None, expires_at=None, vo='def', *,
456
453
  :param reason: A string describing the reason of the loss.
457
454
  :param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files.
458
455
  :param vo: The VO to act on.
459
- :param session: The database session in use.
460
456
 
461
457
  :returns: True is successful.
462
458
  """
463
459
  kwargs = {'state': state}
464
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
465
- if not auth_result.allowed:
466
- raise exception.AccessDenied('Account %s can not declare bad PFNs. %s' % (issuer, auth_result.message))
467
460
 
468
- if expires_at and datetime.datetime.utcnow() <= expires_at and expires_at > datetime.datetime.utcnow() + datetime.timedelta(days=30):
469
- raise exception.InputValidationError('The given duration of %s days exceeds the maximum duration of 30 days.' % (expires_at - datetime.datetime.utcnow()).days)
461
+ with db_session(DatabaseOperationType.WRITE) as session:
462
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
463
+ if not auth_result.allowed:
464
+ raise exception.AccessDenied('Account %s can not declare bad PFNs. %s' % (issuer, auth_result.message))
465
+
466
+ if expires_at and datetime.datetime.utcnow() <= expires_at and expires_at > datetime.datetime.utcnow() + datetime.timedelta(days=30):
467
+ raise exception.InputValidationError('The given duration of %s days exceeds the maximum duration of 30 days.' % (expires_at - datetime.datetime.utcnow()).days)
470
468
 
471
- issuer = InternalAccount(issuer, vo=vo)
469
+ issuer = InternalAccount(issuer, vo=vo)
472
470
 
473
- return replica.add_bad_pfns(pfns=pfns, account=issuer, state=state, reason=reason, expires_at=expires_at, session=session)
471
+ return replica.add_bad_pfns(pfns=pfns, account=issuer, state=state, reason=reason, expires_at=expires_at, session=session)
474
472
 
475
473
 
476
- @transactional_session
477
- def add_bad_dids(dids, rse, issuer, state, reason=None, expires_at=None, vo='def', *, session: "Session"):
474
+ def add_bad_dids(dids, rse, issuer, state, reason=None, expires_at=None, vo='def'):
478
475
  """
479
476
  Add bad replica entries for DIDs.
480
477
 
@@ -485,38 +482,38 @@ def add_bad_dids(dids, rse, issuer, state, reason=None, expires_at=None, vo='def
485
482
  :param reason: A string describing the reason of the loss.
486
483
  :param expires_at: None
487
484
  :param vo: The VO to act on.
488
- :param session: The database session in use.
489
485
 
490
486
  :returns: The list of replicas not declared bad
491
487
  """
492
488
  kwargs = {'state': state}
493
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
494
- if not auth_result.allowed:
495
- raise exception.AccessDenied('Account %s can not declare bad PFN or DIDs. %s' % (issuer, auth_result.message))
496
489
 
497
- issuer = InternalAccount(issuer, vo=vo)
498
- rse_id = get_rse_id(rse=rse, session=session)
490
+ with db_session(DatabaseOperationType.WRITE) as session:
491
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
492
+ if not auth_result.allowed:
493
+ raise exception.AccessDenied('Account %s can not declare bad PFN or DIDs. %s' % (issuer, auth_result.message))
494
+
495
+ issuer = InternalAccount(issuer, vo=vo)
496
+ rse_id = get_rse_id(rse=rse, session=session)
499
497
 
500
- return replica.add_bad_dids(dids=dids, rse_id=rse_id, reason=reason, issuer=issuer, state=state, session=session)
498
+ return replica.add_bad_dids(dids=dids, rse_id=rse_id, reason=reason, issuer=issuer, state=state, session=session)
501
499
 
502
500
 
503
- @read_session
504
- def get_suspicious_files(rse_expression, younger_than=None, nattempts=None, vo='def', *, session: "Session"):
501
+ def get_suspicious_files(rse_expression, younger_than=None, nattempts=None, vo='def'):
505
502
  """
506
503
  List the list of suspicious files on a list of RSEs
507
504
  :param rse_expression: The RSE expression where the suspicious files are located
508
505
  :param younger_than: datetime object to select the suspicious replicas younger than this date.
509
506
  :param nattempts: The number of time the replicas have been declared suspicious
510
507
  :param vo: The VO to act on.
511
- :param session: The database session in use.
512
508
  """
513
- replicas = replica.get_suspicious_files(rse_expression=rse_expression, available_elsewhere=SuspiciousAvailability["ALL"].value,
514
- younger_than=younger_than, nattempts=nattempts, filter_={'vo': vo}, session=session)
515
- return [gateway_update_return_dict(r, session=session) for r in replicas]
516
509
 
510
+ with db_session(DatabaseOperationType.READ) as session:
511
+ replicas = replica.get_suspicious_files(rse_expression=rse_expression, available_elsewhere=SuspiciousAvailability["ALL"].value,
512
+ younger_than=younger_than, nattempts=nattempts, filter_={'vo': vo}, session=session)
513
+ return [gateway_update_return_dict(r, session=session) for r in replicas]
517
514
 
518
- @transactional_session
519
- def set_tombstone(rse, scope, name, issuer, vo='def', *, session: "Session"):
515
+
516
+ def set_tombstone(rse, scope, name, issuer, vo='def'):
520
517
  """
521
518
  Sets a tombstone on one replica.
522
519
 
@@ -525,14 +522,14 @@ def set_tombstone(rse, scope, name, issuer, vo='def', *, session: "Session"):
525
522
  :param name: name of the replica DID.
526
523
  :param issuer: The issuer account
527
524
  :param vo: The VO to act on.
528
- :param session: The database session in use.
529
525
  """
530
526
 
531
- rse_id = get_rse_id(rse, vo=vo, session=session)
527
+ with db_session(DatabaseOperationType.WRITE) as session:
528
+ rse_id = get_rse_id(rse, vo=vo, session=session)
532
529
 
533
- auth_result = permission.has_permission(issuer=issuer, vo=vo, action='set_tombstone', kwargs={}, session=session)
534
- if not auth_result.allowed:
535
- raise exception.AccessDenied('Account %s can not set tombstones. %s' % (issuer, auth_result.message))
530
+ auth_result = permission.has_permission(issuer=issuer, vo=vo, action='set_tombstone', kwargs={}, session=session)
531
+ if not auth_result.allowed:
532
+ raise exception.AccessDenied('Account %s can not set tombstones. %s' % (issuer, auth_result.message))
536
533
 
537
- scope = InternalScope(scope, vo=vo)
538
- replica.set_tombstone(rse_id, scope, name, session=session)
534
+ scope = InternalScope(scope, vo=vo)
535
+ replica.set_tombstone(rse_id, scope, name, session=session)