rucio 37.2.0__py3-none-any.whl → 37.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/cli/rule.py +1 -1
- rucio/client/accountclient.py +205 -60
- rucio/client/accountlimitclient.py +84 -25
- rucio/client/baseclient.py +85 -48
- rucio/client/client.py +49 -41
- rucio/client/configclient.py +36 -13
- rucio/client/credentialclient.py +16 -6
- rucio/client/didclient.py +321 -133
- rucio/client/diracclient.py +13 -6
- rucio/client/downloadclient.py +435 -165
- rucio/client/exportclient.py +8 -2
- rucio/client/fileclient.py +10 -3
- rucio/client/importclient.py +4 -1
- rucio/client/lifetimeclient.py +48 -31
- rucio/client/lockclient.py +22 -7
- rucio/client/metaconventionsclient.py +59 -21
- rucio/client/pingclient.py +3 -1
- rucio/client/replicaclient.py +213 -96
- rucio/client/requestclient.py +123 -16
- rucio/client/rseclient.py +385 -160
- rucio/client/ruleclient.py +147 -51
- rucio/client/scopeclient.py +35 -10
- rucio/client/subscriptionclient.py +60 -27
- rucio/client/touchclient.py +16 -7
- rucio/common/plugins.py +1 -1
- rucio/core/did.py +2 -3
- rucio/core/permission/generic.py +37 -1
- rucio/core/replica.py +6 -6
- rucio/core/rule.py +5 -3
- rucio/daemons/judge/evaluator.py +1 -1
- rucio/db/sqla/util.py +1 -1
- rucio/gateway/authentication.py +58 -88
- rucio/gateway/config.py +63 -75
- rucio/gateway/did.py +245 -329
- rucio/gateway/dirac.py +33 -34
- rucio/gateway/exporter.py +27 -30
- rucio/gateway/importer.py +12 -14
- rucio/gateway/lifetime_exception.py +16 -24
- rucio/gateway/lock.py +27 -40
- rucio/gateway/replica.py +334 -249
- rucio/gateway/request.py +176 -103
- rucio/gateway/rse.py +191 -218
- rucio/gateway/rule.py +115 -146
- rucio/gateway/scope.py +18 -25
- rucio/gateway/subscription.py +90 -108
- rucio/gateway/trace.py +48 -0
- rucio/vcsversion.py +3 -3
- rucio/web/rest/flaskapi/v1/accounts.py +2 -2
- rucio/web/rest/flaskapi/v1/auth.py +15 -0
- rucio/web/rest/flaskapi/v1/common.py +3 -0
- rucio/web/rest/flaskapi/v1/config.py +7 -7
- rucio/web/rest/flaskapi/v1/dids.py +55 -55
- rucio/web/rest/flaskapi/v1/dirac.py +2 -2
- rucio/web/rest/flaskapi/v1/export.py +1 -1
- rucio/web/rest/flaskapi/v1/import.py +1 -1
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +5 -5
- rucio/web/rest/flaskapi/v1/locks.py +4 -4
- rucio/web/rest/flaskapi/v1/main.py +17 -10
- rucio/web/rest/flaskapi/v1/redirect.py +1 -1
- rucio/web/rest/flaskapi/v1/replicas.py +30 -29
- rucio/web/rest/flaskapi/v1/requests.py +211 -20
- rucio/web/rest/flaskapi/v1/rses.py +37 -37
- rucio/web/rest/flaskapi/v1/rules.py +15 -15
- rucio/web/rest/flaskapi/v1/scopes.py +3 -3
- rucio/web/rest/flaskapi/v1/subscriptions.py +9 -9
- rucio/web/rest/flaskapi/v1/traces.py +75 -77
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio.cfg.template +0 -1
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -1
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/METADATA +1 -1
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/RECORD +127 -126
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/WHEEL +1 -1
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/requirements.server.txt +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-account +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-admin +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-atropos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-auditor +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-automatix +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-bb8 +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-cache-client +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-receiver +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-dumper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-follower +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-hermes +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-injector +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-kronos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-minos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-necromancer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-reaper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-transmogrifier +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-undertaker +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/top_level.txt +0 -0
rucio/gateway/replica.py
CHANGED
|
@@ -13,41 +13,49 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import datetime
|
|
16
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
|
|
17
17
|
|
|
18
18
|
from rucio.common import exception
|
|
19
19
|
from rucio.common.constants import SuspiciousAvailability
|
|
20
20
|
from rucio.common.schema import validate_schema
|
|
21
|
-
from rucio.common.types import InternalAccount, InternalScope
|
|
21
|
+
from rucio.common.types import InternalAccount, InternalScope, IPDict
|
|
22
22
|
from rucio.common.utils import gateway_update_return_dict, invert_dict
|
|
23
23
|
from rucio.core import replica
|
|
24
24
|
from rucio.core.rse import get_rse_id, get_rse_name
|
|
25
|
-
from rucio.db.sqla.constants import BadFilesStatus
|
|
26
|
-
from rucio.db.sqla.session import
|
|
25
|
+
from rucio.db.sqla.constants import BadFilesStatus, DatabaseOperationType
|
|
26
|
+
from rucio.db.sqla.session import db_session
|
|
27
27
|
from rucio.gateway import permission
|
|
28
28
|
|
|
29
29
|
if TYPE_CHECKING:
|
|
30
|
-
from collections.abc import Iterator
|
|
30
|
+
from collections.abc import Iterable, Iterator
|
|
31
31
|
|
|
32
|
-
from sqlalchemy.orm import Session
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
def get_bad_replicas_summary(
|
|
34
|
+
rse_expression: Optional[str] = None,
|
|
35
|
+
from_date: Optional[datetime.datetime] = None,
|
|
36
|
+
to_date: Optional[datetime.date] = None,
|
|
37
|
+
vo: str = 'def'
|
|
38
|
+
) -> list[dict[str, Any]]:
|
|
37
39
|
"""
|
|
38
40
|
List the bad file replicas summary. Method used by the rucio-ui.
|
|
39
41
|
:param rse_expression: The RSE expression.
|
|
40
42
|
:param from_date: The start date.
|
|
41
43
|
:param to_date: The end date.
|
|
42
44
|
:param vo: the VO to act on.
|
|
43
|
-
:param session: The database session in use.
|
|
44
45
|
"""
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
47
|
+
replicas = replica.get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date, to_date=to_date, filter_={'vo': vo}, session=session)
|
|
48
|
+
return [gateway_update_return_dict(r, session=session) for r in replicas]
|
|
47
49
|
|
|
48
50
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
+
def list_bad_replicas_status(
|
|
52
|
+
state: Optional[BadFilesStatus] = BadFilesStatus.BAD,
|
|
53
|
+
rse: Optional[str] = None,
|
|
54
|
+
younger_than: Optional[datetime.datetime] = None,
|
|
55
|
+
older_than: Optional[datetime.datetime] = None,
|
|
56
|
+
limit: Optional[int] = None,
|
|
57
|
+
list_pfns: bool = False,
|
|
58
|
+
vo: str = 'def'):
|
|
51
59
|
"""
|
|
52
60
|
List the bad file replicas history states. Method used by the rucio-ui.
|
|
53
61
|
:param state: The state of the file (SUSPICIOUS or BAD).
|
|
@@ -56,19 +64,25 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse=None, younger_than=No
|
|
|
56
64
|
:param older_than: datetime object to select bad replicas older than this date.
|
|
57
65
|
:param limit: The maximum number of replicas returned.
|
|
58
66
|
:param vo: The VO to act on.
|
|
59
|
-
:param session: The database session in use.
|
|
60
67
|
"""
|
|
61
68
|
rse_id = None
|
|
62
|
-
if rse is not None:
|
|
63
|
-
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
64
69
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
70
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
71
|
+
if rse is not None:
|
|
72
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
73
|
+
|
|
74
|
+
replicas = replica.list_bad_replicas_status(state=state, rse_id=rse_id, younger_than=younger_than,
|
|
75
|
+
older_than=older_than, limit=limit, list_pfns=list_pfns, vo=vo, session=session)
|
|
76
|
+
return [gateway_update_return_dict(r, session=session) for r in replicas]
|
|
68
77
|
|
|
69
78
|
|
|
70
|
-
|
|
71
|
-
|
|
79
|
+
def declare_bad_file_replicas(
|
|
80
|
+
replicas: Union[list[str], list[dict[str, Any]]],
|
|
81
|
+
reason: str,
|
|
82
|
+
issuer: str,
|
|
83
|
+
vo: str = 'def',
|
|
84
|
+
force: bool = False
|
|
85
|
+
) -> dict[str, Any]:
|
|
72
86
|
"""
|
|
73
87
|
Declare a list of bad replicas.
|
|
74
88
|
|
|
@@ -78,7 +92,6 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
|
|
|
78
92
|
:param issuer: The issuer account.
|
|
79
93
|
:param vo: The VO to act on.
|
|
80
94
|
:param force: boolean, ignore existing replica status in the bad_replicas table. Default: False
|
|
81
|
-
:param session: The database session in use.
|
|
82
95
|
:returns: Dictionary {rse_name -> [list of replicas failed to declare]}
|
|
83
96
|
"""
|
|
84
97
|
|
|
@@ -95,62 +108,72 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False, *
|
|
|
95
108
|
|
|
96
109
|
replicas_lst = replicas
|
|
97
110
|
rse_ids_to_check = set() # to check for permission to declare bad replicas
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
"
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
111
|
+
|
|
112
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
113
|
+
if as_pfns:
|
|
114
|
+
scheme, rses_for_replicas, unknowns = replica.get_pfn_to_rse(replicas, vo=vo, session=session)
|
|
115
|
+
if unknowns:
|
|
116
|
+
raise exception.ReplicaNotFound("Not all replicas found")
|
|
117
|
+
rse_ids_to_check = set(rses_for_replicas.keys())
|
|
118
|
+
else:
|
|
119
|
+
# replicas is a list[dict] in this path,
|
|
120
|
+
# but the static code analyzer does not see it due to as_pfns logic above,
|
|
121
|
+
# so cast is used instead
|
|
122
|
+
replicas = cast("list[dict[str, Any]]", replicas)
|
|
123
|
+
replicas_lst = []
|
|
124
|
+
for r in replicas:
|
|
125
|
+
if "name" not in r or "scope" not in r or ("rse" not in r and "rse_id" not in r):
|
|
126
|
+
raise exception.InvalidType('The replica dictionary must include scope and either rse (name) or rse_id')
|
|
127
|
+
scope = InternalScope(r['scope'], vo=vo)
|
|
128
|
+
rse_id = r.get("rse_id") or rse_map.get(r['rse'])
|
|
129
|
+
if rse_id is None:
|
|
130
|
+
rse = r["rse"]
|
|
131
|
+
rse_map[rse] = rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
132
|
+
replicas_lst.append({
|
|
133
|
+
"rse_id": rse_id,
|
|
134
|
+
"scope": scope,
|
|
135
|
+
"name": r["name"]
|
|
136
|
+
})
|
|
137
|
+
rse_ids_to_check.add(rse_id)
|
|
138
|
+
|
|
139
|
+
rse_id_to_name = invert_dict(rse_map) # RSE id -> RSE name
|
|
140
|
+
|
|
141
|
+
for rse_id in rse_ids_to_check:
|
|
142
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_bad_file_replicas',
|
|
143
|
+
kwargs={"rse_id": rse_id},
|
|
144
|
+
session=session)
|
|
145
|
+
if not auth_result.allowed:
|
|
146
|
+
raise exception.AccessDenied('Account %s can not declare bad replicas in RSE %s. %s' %
|
|
147
|
+
(issuer, rse_id_to_name.get(rse_id, rse_id), auth_result.message))
|
|
148
|
+
|
|
149
|
+
undeclared = replica.declare_bad_file_replicas(replicas_lst, reason=reason,
|
|
150
|
+
issuer=InternalAccount(issuer, vo=vo),
|
|
151
|
+
status=BadFilesStatus.BAD,
|
|
152
|
+
force=force, session=session)
|
|
153
|
+
out = {}
|
|
154
|
+
for rse_id, ulist in undeclared.items():
|
|
155
|
+
if ulist:
|
|
156
|
+
rse_name = None
|
|
157
|
+
if rse_id == 'unknown':
|
|
158
|
+
rse_name = 'unknown'
|
|
159
|
+
elif rse_id in rse_id_to_name:
|
|
160
|
+
rse_name = rse_id_to_name[rse_id]
|
|
161
|
+
else:
|
|
162
|
+
try:
|
|
163
|
+
rse_name = get_rse_name(rse_id=rse_id, session=session)
|
|
164
|
+
except (ValueError, exception.RSENotFound):
|
|
165
|
+
rse_name = str(rse_id)
|
|
166
|
+
if rse_name:
|
|
167
|
+
out[rse_name] = out.get(rse_name, []) + ulist
|
|
168
|
+
return out
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def declare_suspicious_file_replicas(
|
|
172
|
+
pfns: list[Union[str, dict[str, Any]]],
|
|
173
|
+
reason: str,
|
|
174
|
+
issuer: str,
|
|
175
|
+
vo: str = 'def'
|
|
176
|
+
) -> dict[str, list[str]]:
|
|
154
177
|
"""
|
|
155
178
|
Declare a list of bad replicas.
|
|
156
179
|
|
|
@@ -158,54 +181,69 @@ def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def', *, session:
|
|
|
158
181
|
:param reason: The reason of the loss.
|
|
159
182
|
:param issuer: The issuer account.
|
|
160
183
|
:param vo: The VO to act on.
|
|
161
|
-
:param session: The database session in use.
|
|
162
184
|
"""
|
|
163
185
|
kwargs = {}
|
|
164
|
-
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_suspicious_file_replicas', kwargs=kwargs, session=session)
|
|
165
|
-
if not auth_result.allowed:
|
|
166
|
-
raise exception.AccessDenied('Account %s can not declare suspicious replicas. %s' % (issuer, auth_result.message))
|
|
167
186
|
|
|
168
|
-
|
|
187
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
188
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='declare_suspicious_file_replicas', kwargs=kwargs, session=session)
|
|
189
|
+
if not auth_result.allowed:
|
|
190
|
+
raise exception.AccessDenied('Account %s can not declare suspicious replicas. %s' % (issuer, auth_result.message))
|
|
191
|
+
|
|
192
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
169
193
|
|
|
170
|
-
|
|
194
|
+
replicas = replica.declare_bad_file_replicas(pfns, reason=reason, issuer=issuer_account, status=BadFilesStatus.SUSPICIOUS, session=session)
|
|
171
195
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
196
|
+
for k in list(replicas):
|
|
197
|
+
try:
|
|
198
|
+
rse = get_rse_name(rse_id=k, session=session)
|
|
199
|
+
replicas[rse] = replicas.pop(k)
|
|
200
|
+
except exception.RSENotFound:
|
|
201
|
+
pass
|
|
178
202
|
|
|
179
203
|
return replicas
|
|
180
204
|
|
|
181
205
|
|
|
182
|
-
|
|
183
|
-
|
|
206
|
+
def get_did_from_pfns(
|
|
207
|
+
pfns: "Iterable[str]",
|
|
208
|
+
rse: str,
|
|
209
|
+
vo: str = 'def'
|
|
210
|
+
) -> 'Iterator[dict[str, dict[str, Any]]]':
|
|
184
211
|
"""
|
|
185
212
|
Get the DIDs associated to a PFN on one given RSE
|
|
186
213
|
|
|
187
214
|
:param pfns: The list of PFNs.
|
|
188
215
|
:param rse: The RSE name.
|
|
189
216
|
:param vo: The VO to act on.
|
|
190
|
-
:param session: The database session in use.
|
|
191
217
|
:returns: A dictionary {pfn: {'scope': scope, 'name': name}}
|
|
192
218
|
"""
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
for r in replicas:
|
|
197
|
-
for k in r.keys():
|
|
198
|
-
r[k]['scope'] = r[k]['scope'].external
|
|
199
|
-
yield r
|
|
200
|
-
|
|
219
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
220
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
221
|
+
replicas = replica.get_did_from_pfns(pfns=pfns, rse_id=rse_id, vo=vo, session=session)
|
|
201
222
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
223
|
+
for r in replicas:
|
|
224
|
+
for k in r.keys():
|
|
225
|
+
r[k]['scope'] = r[k]['scope'].external
|
|
226
|
+
yield r
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def list_replicas(
|
|
230
|
+
dids: "Iterable[dict[str, Any]]",
|
|
231
|
+
schemes: Optional[list[str]] = None,
|
|
232
|
+
unavailable: bool = False,
|
|
233
|
+
request_id: Optional[str] = None,
|
|
234
|
+
ignore_availability: bool = True,
|
|
235
|
+
all_states: bool = False,
|
|
236
|
+
rse_expression: Optional[str] = None,
|
|
237
|
+
client_location: Optional[IPDict] = None,
|
|
238
|
+
domain: Optional[str] = None,
|
|
239
|
+
signature_lifetime: Optional[int] = None,
|
|
240
|
+
resolve_archives: bool = True,
|
|
241
|
+
resolve_parents: bool = False,
|
|
242
|
+
nrandom: Optional[int] = None,
|
|
243
|
+
updated_after: Optional[datetime.datetime] = None,
|
|
244
|
+
issuer: Optional[str] = None,
|
|
245
|
+
vo: str = 'def'
|
|
246
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
209
247
|
"""
|
|
210
248
|
List file replicas for a list of data identifiers.
|
|
211
249
|
|
|
@@ -223,43 +261,49 @@ def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
|
|
|
223
261
|
:param updated_after: datetime object (UTC time), only return replicas updated after this time
|
|
224
262
|
:param issuer: The issuer account.
|
|
225
263
|
:param vo: The VO to act on.
|
|
226
|
-
:param session: The database session in use.
|
|
227
264
|
"""
|
|
228
265
|
validate_schema(name='r_dids', obj=dids, vo=vo)
|
|
229
266
|
|
|
230
267
|
# Allow selected authenticated users to retrieve signed URLs.
|
|
231
268
|
# Unauthenticated users, or permission-less users will get the raw URL without the signature.
|
|
232
269
|
sign_urls = False
|
|
233
|
-
if permission.has_permission(issuer=issuer, vo=vo, action='get_signed_url', kwargs={}, session=session):
|
|
234
|
-
sign_urls = True
|
|
235
|
-
|
|
236
|
-
for d in dids:
|
|
237
|
-
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
238
|
-
|
|
239
|
-
replicas = replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
|
|
240
|
-
request_id=request_id,
|
|
241
|
-
ignore_availability=ignore_availability,
|
|
242
|
-
all_states=all_states, rse_expression=rse_expression,
|
|
243
|
-
client_location=client_location, domain=domain,
|
|
244
|
-
sign_urls=sign_urls, signature_lifetime=signature_lifetime,
|
|
245
|
-
resolve_archives=resolve_archives, resolve_parents=resolve_parents,
|
|
246
|
-
nrandom=nrandom, updated_after=updated_after, by_rse_name=True, session=session)
|
|
247
270
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
271
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
272
|
+
if permission.has_permission(issuer=issuer, vo=vo, action='get_signed_url', kwargs={}, session=session):
|
|
273
|
+
sign_urls = True
|
|
274
|
+
|
|
275
|
+
for d in dids:
|
|
276
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
277
|
+
|
|
278
|
+
replicas = replica.list_replicas(dids=dids, schemes=schemes, unavailable=unavailable,
|
|
279
|
+
request_id=request_id,
|
|
280
|
+
ignore_availability=ignore_availability,
|
|
281
|
+
all_states=all_states, rse_expression=rse_expression,
|
|
282
|
+
client_location=client_location, domain=domain,
|
|
283
|
+
sign_urls=sign_urls, signature_lifetime=signature_lifetime,
|
|
284
|
+
resolve_archives=resolve_archives, resolve_parents=resolve_parents,
|
|
285
|
+
nrandom=nrandom, updated_after=updated_after, by_rse_name=True, session=session)
|
|
286
|
+
|
|
287
|
+
for rep in replicas:
|
|
288
|
+
rep['scope'] = rep['scope'].external
|
|
289
|
+
if 'parents' in rep:
|
|
290
|
+
new_parents = []
|
|
291
|
+
for p in rep['parents']:
|
|
292
|
+
scope, name = p.split(':')
|
|
293
|
+
scope = InternalScope(scope, from_external=False).external
|
|
294
|
+
new_parents.append('{}:{}'.format(scope, name))
|
|
295
|
+
rep['parents'] = new_parents
|
|
296
|
+
|
|
297
|
+
yield rep
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def add_replicas(
|
|
301
|
+
rse: str,
|
|
302
|
+
files: "Iterable[dict[str, Any]]",
|
|
303
|
+
issuer: str,
|
|
304
|
+
ignore_availability: bool = False,
|
|
305
|
+
vo: str = 'def'
|
|
306
|
+
) -> None:
|
|
263
307
|
"""
|
|
264
308
|
Bulk add file replicas.
|
|
265
309
|
|
|
@@ -268,34 +312,37 @@ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def', *, ses
|
|
|
268
312
|
:param issuer: The issuer account.
|
|
269
313
|
:param ignore_availability: Ignore blocked RSEs.
|
|
270
314
|
:param vo: The VO to act on.
|
|
271
|
-
:param session: The database session in use.
|
|
272
|
-
|
|
273
|
-
:returns: True is successful, False otherwise
|
|
274
315
|
"""
|
|
275
316
|
for v_file in files:
|
|
276
317
|
v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
|
|
277
318
|
validate_schema(name='dids', obj=files, vo=vo)
|
|
278
319
|
|
|
279
|
-
|
|
320
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
321
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
280
322
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
323
|
+
kwargs = {'rse': rse, 'rse_id': rse_id}
|
|
324
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_replicas', kwargs=kwargs, session=session)
|
|
325
|
+
if not auth_result.allowed:
|
|
326
|
+
raise exception.AccessDenied('Account %s can not add file replicas on %s. %s' % (issuer, rse, auth_result.message))
|
|
327
|
+
if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
|
|
328
|
+
ignore_availability = False
|
|
287
329
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
330
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
331
|
+
for f in files:
|
|
332
|
+
f['scope'] = InternalScope(f['scope'], vo=vo)
|
|
333
|
+
if 'account' in f:
|
|
334
|
+
f['account'] = InternalAccount(f['account'], vo=vo)
|
|
293
335
|
|
|
294
|
-
|
|
336
|
+
replica.add_replicas(rse_id=rse_id, files=files, account=issuer_account, ignore_availability=ignore_availability, session=session)
|
|
295
337
|
|
|
296
338
|
|
|
297
|
-
|
|
298
|
-
|
|
339
|
+
def delete_replicas(
|
|
340
|
+
rse: str,
|
|
341
|
+
files: "Iterable[dict[str, Any]]",
|
|
342
|
+
issuer: str,
|
|
343
|
+
ignore_availability: bool = False,
|
|
344
|
+
vo: str = 'def'
|
|
345
|
+
) -> None:
|
|
299
346
|
"""
|
|
300
347
|
Bulk delete file replicas.
|
|
301
348
|
|
|
@@ -304,29 +351,31 @@ def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def', *,
|
|
|
304
351
|
:param issuer: The issuer account.
|
|
305
352
|
:param ignore_availability: Ignore blocked RSEs.
|
|
306
353
|
:param vo: The VO to act on.
|
|
307
|
-
:param session: The database session in use.
|
|
308
|
-
|
|
309
|
-
:returns: True is successful, False otherwise
|
|
310
354
|
"""
|
|
311
355
|
validate_schema(name='r_dids', obj=files, vo=vo)
|
|
312
356
|
|
|
313
|
-
|
|
357
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
358
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
314
359
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
360
|
+
kwargs = {'rse': rse, 'rse_id': rse_id}
|
|
361
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='delete_replicas', kwargs=kwargs, session=session)
|
|
362
|
+
if not auth_result.allowed:
|
|
363
|
+
raise exception.AccessDenied('Account %s can not delete file replicas on %s. %s' % (issuer, rse, auth_result.message))
|
|
364
|
+
if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
|
|
365
|
+
ignore_availability = False
|
|
321
366
|
|
|
322
|
-
|
|
323
|
-
|
|
367
|
+
for f in files:
|
|
368
|
+
f['scope'] = InternalScope(f['scope'], vo=vo)
|
|
324
369
|
|
|
325
|
-
|
|
370
|
+
replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability, session=session)
|
|
326
371
|
|
|
327
372
|
|
|
328
|
-
|
|
329
|
-
|
|
373
|
+
def update_replicas_states(
|
|
374
|
+
rse: str,
|
|
375
|
+
files: "Iterable[dict[str, Any]]",
|
|
376
|
+
issuer: str,
|
|
377
|
+
vo: str = 'def'
|
|
378
|
+
) -> None:
|
|
330
379
|
"""
|
|
331
380
|
Update File replica information and state.
|
|
332
381
|
|
|
@@ -334,54 +383,59 @@ def update_replicas_states(rse, files, issuer, vo='def', *, session: "Session"):
|
|
|
334
383
|
:param files: The list of files.
|
|
335
384
|
:param issuer: The issuer account.
|
|
336
385
|
:param vo: The VO to act on.
|
|
337
|
-
:param session: The database session in use.
|
|
338
386
|
"""
|
|
339
387
|
for v_file in files:
|
|
340
388
|
v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
|
|
341
389
|
validate_schema(name='dids', obj=files, vo=vo)
|
|
342
390
|
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
kwargs = {'rse': rse, 'rse_id': rse_id}
|
|
346
|
-
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_replicas_states', kwargs=kwargs, session=session)
|
|
347
|
-
if not auth_result.allowed:
|
|
348
|
-
raise exception.AccessDenied('Account %s can not update file replicas state on %s. %s' % (issuer, rse, auth_result.message))
|
|
349
|
-
replicas = []
|
|
350
|
-
for file in files:
|
|
351
|
-
rep = file
|
|
352
|
-
rep['rse_id'] = rse_id
|
|
353
|
-
rep['scope'] = InternalScope(rep['scope'], vo=vo)
|
|
354
|
-
replicas.append(rep)
|
|
355
|
-
replica.update_replicas_states(replicas=replicas, session=session)
|
|
356
|
-
|
|
391
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
392
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
357
393
|
|
|
358
|
-
|
|
359
|
-
|
|
394
|
+
kwargs = {'rse': rse, 'rse_id': rse_id}
|
|
395
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='update_replicas_states', kwargs=kwargs, session=session)
|
|
396
|
+
if not auth_result.allowed:
|
|
397
|
+
raise exception.AccessDenied('Account %s can not update file replicas state on %s. %s' % (issuer, rse, auth_result.message))
|
|
398
|
+
replicas = []
|
|
399
|
+
for file in files:
|
|
400
|
+
rep = file
|
|
401
|
+
rep['rse_id'] = rse_id
|
|
402
|
+
rep['scope'] = InternalScope(rep['scope'], vo=vo)
|
|
403
|
+
replicas.append(rep)
|
|
404
|
+
replica.update_replicas_states(replicas=replicas, session=session)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def list_dataset_replicas(
|
|
408
|
+
scope: str,
|
|
409
|
+
name: str,
|
|
410
|
+
deep: bool = False,
|
|
411
|
+
vo: str = 'def'
|
|
412
|
+
) -> "Iterator[dict[str, Any]]":
|
|
360
413
|
"""
|
|
361
414
|
:param scope: The scope of the dataset.
|
|
362
415
|
:param name: The name of the dataset.
|
|
363
416
|
:param deep: Lookup at the file level.
|
|
364
417
|
:param vo: The VO to act on.
|
|
365
|
-
:param session: The database session in use.
|
|
366
418
|
|
|
367
419
|
:returns: A list of dict dataset replicas
|
|
368
420
|
"""
|
|
369
421
|
|
|
370
|
-
|
|
422
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
371
423
|
|
|
372
|
-
|
|
424
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
425
|
+
replicas = replica.list_dataset_replicas(scope=internal_scope, name=name, deep=deep, session=session)
|
|
373
426
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
427
|
+
for r in replicas:
|
|
428
|
+
r['scope'] = r['scope'].external
|
|
429
|
+
yield r
|
|
377
430
|
|
|
378
431
|
|
|
379
|
-
|
|
380
|
-
|
|
432
|
+
def list_dataset_replicas_bulk(
|
|
433
|
+
dids: 'Iterable[dict[str, Any]]',
|
|
434
|
+
vo: str = 'def'
|
|
435
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
381
436
|
"""
|
|
382
437
|
:param dids: The list of did dictionaries with scope and name.
|
|
383
438
|
:param vo: The VO to act on.
|
|
384
|
-
:param session: The database session in use.
|
|
385
439
|
|
|
386
440
|
:returns: A list of dict dataset replicas
|
|
387
441
|
"""
|
|
@@ -399,54 +453,66 @@ def list_dataset_replicas_bulk(dids, vo='def', *, session: "Session"):
|
|
|
399
453
|
internal_scope = InternalScope(scope, vo=vo)
|
|
400
454
|
names_by_intscope[internal_scope] = names_by_scope[scope]
|
|
401
455
|
|
|
402
|
-
|
|
456
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
457
|
+
replicas = replica.list_dataset_replicas_bulk(names_by_intscope, session=session)
|
|
403
458
|
|
|
404
|
-
|
|
405
|
-
|
|
459
|
+
for r in replicas:
|
|
460
|
+
yield gateway_update_return_dict(r, session=session)
|
|
406
461
|
|
|
407
462
|
|
|
408
|
-
|
|
409
|
-
|
|
463
|
+
def list_dataset_replicas_vp(
|
|
464
|
+
scope: str,
|
|
465
|
+
name: str,
|
|
466
|
+
deep: bool = False,
|
|
467
|
+
vo: str = 'def'
|
|
468
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
410
469
|
"""
|
|
411
470
|
:param scope: The scope of the dataset.
|
|
412
471
|
:param name: The name of the dataset.
|
|
413
472
|
:param deep: Lookup at the file level.
|
|
414
473
|
:param vo: The vo to act on.
|
|
415
|
-
:param session: The database session in use.
|
|
416
474
|
|
|
417
475
|
:returns: If VP exists a list of dicts of sites, otherwise nothing
|
|
418
476
|
|
|
419
477
|
NOTICE: This is an RnD function and might change or go away at any time.
|
|
420
478
|
"""
|
|
421
479
|
|
|
422
|
-
|
|
423
|
-
for r in replica.list_dataset_replicas_vp(scope=scope, name=name, deep=deep, session=session):
|
|
424
|
-
yield gateway_update_return_dict(r, session=session)
|
|
480
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
425
481
|
|
|
482
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
483
|
+
for r in replica.list_dataset_replicas_vp(scope=internal_scope, name=name, deep=deep, session=session):
|
|
484
|
+
yield gateway_update_return_dict(r, session=session)
|
|
426
485
|
|
|
427
|
-
|
|
428
|
-
def list_datasets_per_rse(rse: str, filters: Optional[dict[str, Any]] = None, limit: Optional[int] = None, vo: str = 'def'
|
|
486
|
+
|
|
487
|
+
def list_datasets_per_rse(rse: str, filters: Optional[dict[str, Any]] = None, limit: Optional[int] = None, vo: str = 'def') -> 'Iterator[dict[str, Any]]':
|
|
429
488
|
"""
|
|
430
489
|
:param scope: The scope of the dataset.
|
|
431
490
|
:param name: The name of the dataset.
|
|
432
491
|
:param filters: dictionary of attributes by which the results should be filtered.
|
|
433
492
|
:param limit: limit number.
|
|
434
493
|
:param vo: The VO to act on.
|
|
435
|
-
:param session: The database session in use.
|
|
436
494
|
|
|
437
495
|
:returns: A list of dict dataset replicas
|
|
438
496
|
"""
|
|
439
497
|
|
|
440
498
|
filters = filters or {}
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
499
|
+
|
|
500
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
501
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
502
|
+
if 'scope' in filters:
|
|
503
|
+
filters['scope'] = InternalScope(filters['scope'], vo=vo)
|
|
504
|
+
for r in replica.list_datasets_per_rse(rse_id, filters=filters, limit=limit, session=session):
|
|
505
|
+
yield gateway_update_return_dict(r, session=session)
|
|
446
506
|
|
|
447
507
|
|
|
448
|
-
|
|
449
|
-
|
|
508
|
+
def add_bad_pfns(
|
|
509
|
+
pfns: "Iterable[str]",
|
|
510
|
+
issuer: str,
|
|
511
|
+
state: BadFilesStatus,
|
|
512
|
+
reason: Optional[str] = None,
|
|
513
|
+
expires_at: Optional[datetime.datetime] = None,
|
|
514
|
+
vo: str = 'def'
|
|
515
|
+
) -> Literal[True]:
|
|
450
516
|
"""
|
|
451
517
|
Add bad PFNs.
|
|
452
518
|
|
|
@@ -456,25 +522,33 @@ def add_bad_pfns(pfns, issuer, state, reason=None, expires_at=None, vo='def', *,
|
|
|
456
522
|
:param reason: A string describing the reason of the loss.
|
|
457
523
|
:param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files.
|
|
458
524
|
:param vo: The VO to act on.
|
|
459
|
-
:param session: The database session in use.
|
|
460
525
|
|
|
461
526
|
:returns: True is successful.
|
|
462
527
|
"""
|
|
463
528
|
kwargs = {'state': state}
|
|
464
|
-
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
|
|
465
|
-
if not auth_result.allowed:
|
|
466
|
-
raise exception.AccessDenied('Account %s can not declare bad PFNs. %s' % (issuer, auth_result.message))
|
|
467
529
|
|
|
468
|
-
|
|
469
|
-
|
|
530
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
531
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
|
|
532
|
+
if not auth_result.allowed:
|
|
533
|
+
raise exception.AccessDenied('Account %s can not declare bad PFNs. %s' % (issuer, auth_result.message))
|
|
534
|
+
|
|
535
|
+
if expires_at and datetime.datetime.utcnow() <= expires_at and expires_at > datetime.datetime.utcnow() + datetime.timedelta(days=30):
|
|
536
|
+
raise exception.InputValidationError('The given duration of %s days exceeds the maximum duration of 30 days.' % (expires_at - datetime.datetime.utcnow()).days)
|
|
470
537
|
|
|
471
|
-
|
|
538
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
472
539
|
|
|
473
|
-
|
|
540
|
+
return replica.add_bad_pfns(pfns=pfns, account=issuer_account, state=state, reason=reason, expires_at=expires_at, session=session)
|
|
474
541
|
|
|
475
542
|
|
|
476
|
-
|
|
477
|
-
|
|
543
|
+
def add_bad_dids(
|
|
544
|
+
dids: "Iterable[dict[str, Any]]",
|
|
545
|
+
rse: str,
|
|
546
|
+
issuer: str,
|
|
547
|
+
state: BadFilesStatus,
|
|
548
|
+
reason: Optional[str] = None,
|
|
549
|
+
expires_at: Optional[datetime.datetime] = None,
|
|
550
|
+
vo: str = 'def'
|
|
551
|
+
) -> list[str]:
|
|
478
552
|
"""
|
|
479
553
|
Add bad replica entries for DIDs.
|
|
480
554
|
|
|
@@ -485,38 +559,49 @@ def add_bad_dids(dids, rse, issuer, state, reason=None, expires_at=None, vo='def
|
|
|
485
559
|
:param reason: A string describing the reason of the loss.
|
|
486
560
|
:param expires_at: None
|
|
487
561
|
:param vo: The VO to act on.
|
|
488
|
-
:param session: The database session in use.
|
|
489
562
|
|
|
490
563
|
:returns: The list of replicas not declared bad
|
|
491
564
|
"""
|
|
492
565
|
kwargs = {'state': state}
|
|
493
|
-
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
|
|
494
|
-
if not auth_result.allowed:
|
|
495
|
-
raise exception.AccessDenied('Account %s can not declare bad PFN or DIDs. %s' % (issuer, auth_result.message))
|
|
496
566
|
|
|
497
|
-
|
|
498
|
-
|
|
567
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
568
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='add_bad_pfns', kwargs=kwargs, session=session)
|
|
569
|
+
if not auth_result.allowed:
|
|
570
|
+
raise exception.AccessDenied('Account %s can not declare bad PFN or DIDs. %s' % (issuer, auth_result.message))
|
|
571
|
+
|
|
572
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
573
|
+
rse_id = get_rse_id(rse=rse, session=session)
|
|
499
574
|
|
|
500
|
-
|
|
575
|
+
return replica.add_bad_dids(dids=dids, rse_id=rse_id, reason=reason, issuer=issuer_account, state=state, session=session)
|
|
501
576
|
|
|
502
577
|
|
|
503
|
-
|
|
504
|
-
|
|
578
|
+
def get_suspicious_files(
|
|
579
|
+
rse_expression: Optional[str],
|
|
580
|
+
younger_than: Optional[datetime.datetime] = None,
|
|
581
|
+
nattempts: Optional[int] = None,
|
|
582
|
+
vo: str = 'def'
|
|
583
|
+
) -> list[dict[str, Any]]:
|
|
505
584
|
"""
|
|
506
585
|
List the list of suspicious files on a list of RSEs
|
|
507
586
|
:param rse_expression: The RSE expression where the suspicious files are located
|
|
508
587
|
:param younger_than: datetime object to select the suspicious replicas younger than this date.
|
|
509
588
|
:param nattempts: The number of time the replicas have been declared suspicious
|
|
510
589
|
:param vo: The VO to act on.
|
|
511
|
-
:param session: The database session in use.
|
|
512
590
|
"""
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
591
|
+
|
|
592
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
593
|
+
replicas = replica.get_suspicious_files(rse_expression=rse_expression, available_elsewhere=SuspiciousAvailability["ALL"].value,
|
|
594
|
+
younger_than=younger_than, nattempts=nattempts, filter_={'vo': vo}, session=session)
|
|
595
|
+
return [gateway_update_return_dict(r, session=session) for r in replicas]
|
|
516
596
|
|
|
517
597
|
|
|
518
|
-
|
|
519
|
-
|
|
598
|
+
def set_tombstone(
|
|
599
|
+
rse: str,
|
|
600
|
+
scope: str,
|
|
601
|
+
name: str,
|
|
602
|
+
issuer: str,
|
|
603
|
+
vo: str = 'def'
|
|
604
|
+
) -> None:
|
|
520
605
|
"""
|
|
521
606
|
Sets a tombstone on one replica.
|
|
522
607
|
|
|
@@ -525,14 +610,14 @@ def set_tombstone(rse, scope, name, issuer, vo='def', *, session: "Session"):
|
|
|
525
610
|
:param name: name of the replica DID.
|
|
526
611
|
:param issuer: The issuer account
|
|
527
612
|
:param vo: The VO to act on.
|
|
528
|
-
:param session: The database session in use.
|
|
529
613
|
"""
|
|
530
614
|
|
|
531
|
-
|
|
615
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
616
|
+
rse_id = get_rse_id(rse, vo=vo, session=session)
|
|
532
617
|
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
618
|
+
auth_result = permission.has_permission(issuer=issuer, vo=vo, action='set_tombstone', kwargs={}, session=session)
|
|
619
|
+
if not auth_result.allowed:
|
|
620
|
+
raise exception.AccessDenied('Account %s can not set tombstones. %s' % (issuer, auth_result.message))
|
|
536
621
|
|
|
537
|
-
|
|
538
|
-
|
|
622
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
623
|
+
replica.set_tombstone(rse_id, internal_scope, name, session=session)
|