rucio 37.2.0__py3-none-any.whl → 37.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/cli/rule.py +1 -1
- rucio/client/accountclient.py +205 -60
- rucio/client/accountlimitclient.py +84 -25
- rucio/client/baseclient.py +85 -48
- rucio/client/client.py +49 -41
- rucio/client/configclient.py +36 -13
- rucio/client/credentialclient.py +16 -6
- rucio/client/didclient.py +321 -133
- rucio/client/diracclient.py +13 -6
- rucio/client/downloadclient.py +435 -165
- rucio/client/exportclient.py +8 -2
- rucio/client/fileclient.py +10 -3
- rucio/client/importclient.py +4 -1
- rucio/client/lifetimeclient.py +48 -31
- rucio/client/lockclient.py +22 -7
- rucio/client/metaconventionsclient.py +59 -21
- rucio/client/pingclient.py +3 -1
- rucio/client/replicaclient.py +213 -96
- rucio/client/requestclient.py +123 -16
- rucio/client/rseclient.py +385 -160
- rucio/client/ruleclient.py +147 -51
- rucio/client/scopeclient.py +35 -10
- rucio/client/subscriptionclient.py +60 -27
- rucio/client/touchclient.py +16 -7
- rucio/common/plugins.py +1 -1
- rucio/core/did.py +2 -3
- rucio/core/permission/generic.py +37 -1
- rucio/core/replica.py +6 -6
- rucio/core/rule.py +5 -3
- rucio/daemons/judge/evaluator.py +1 -1
- rucio/db/sqla/util.py +1 -1
- rucio/gateway/authentication.py +58 -88
- rucio/gateway/config.py +63 -75
- rucio/gateway/did.py +245 -329
- rucio/gateway/dirac.py +33 -34
- rucio/gateway/exporter.py +27 -30
- rucio/gateway/importer.py +12 -14
- rucio/gateway/lifetime_exception.py +16 -24
- rucio/gateway/lock.py +27 -40
- rucio/gateway/replica.py +334 -249
- rucio/gateway/request.py +176 -103
- rucio/gateway/rse.py +191 -218
- rucio/gateway/rule.py +115 -146
- rucio/gateway/scope.py +18 -25
- rucio/gateway/subscription.py +90 -108
- rucio/gateway/trace.py +48 -0
- rucio/vcsversion.py +3 -3
- rucio/web/rest/flaskapi/v1/accounts.py +2 -2
- rucio/web/rest/flaskapi/v1/auth.py +15 -0
- rucio/web/rest/flaskapi/v1/common.py +3 -0
- rucio/web/rest/flaskapi/v1/config.py +7 -7
- rucio/web/rest/flaskapi/v1/dids.py +55 -55
- rucio/web/rest/flaskapi/v1/dirac.py +2 -2
- rucio/web/rest/flaskapi/v1/export.py +1 -1
- rucio/web/rest/flaskapi/v1/import.py +1 -1
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +5 -5
- rucio/web/rest/flaskapi/v1/locks.py +4 -4
- rucio/web/rest/flaskapi/v1/main.py +17 -10
- rucio/web/rest/flaskapi/v1/redirect.py +1 -1
- rucio/web/rest/flaskapi/v1/replicas.py +30 -29
- rucio/web/rest/flaskapi/v1/requests.py +211 -20
- rucio/web/rest/flaskapi/v1/rses.py +37 -37
- rucio/web/rest/flaskapi/v1/rules.py +15 -15
- rucio/web/rest/flaskapi/v1/scopes.py +3 -3
- rucio/web/rest/flaskapi/v1/subscriptions.py +9 -9
- rucio/web/rest/flaskapi/v1/traces.py +75 -77
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio.cfg.template +0 -1
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -1
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/METADATA +1 -1
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/RECORD +127 -126
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/WHEEL +1 -1
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/requirements.server.txt +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-account +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-admin +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-atropos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-auditor +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-automatix +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-bb8 +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-cache-client +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-receiver +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-dumper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-follower +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-hermes +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-injector +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-kronos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-minos +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-necromancer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-reaper +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-transmogrifier +0 -0
- {rucio-37.2.0.data → rucio-37.4.0.data}/scripts/rucio-undertaker +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-37.2.0.dist-info → rucio-37.4.0.dist-info}/top_level.txt +0 -0
rucio/gateway/did.py
CHANGED
|
@@ -24,19 +24,16 @@ from rucio.common.utils import gateway_update_return_dict
|
|
|
24
24
|
from rucio.core import did, naming_convention
|
|
25
25
|
from rucio.core import meta_conventions as meta_convention_core
|
|
26
26
|
from rucio.core.rse import get_rse_id
|
|
27
|
-
from rucio.db.sqla.constants import DIDType
|
|
28
|
-
from rucio.db.sqla.session import
|
|
27
|
+
from rucio.db.sqla.constants import DatabaseOperationType, DIDType
|
|
28
|
+
from rucio.db.sqla.session import db_session
|
|
29
29
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
31
31
|
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
32
32
|
|
|
33
|
-
from sqlalchemy.orm import Session
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
@stream_session
|
|
37
34
|
def list_dids(
|
|
38
35
|
scope: str,
|
|
39
|
-
filters: '
|
|
36
|
+
filters: 'Iterable[dict[Any, Any]]',
|
|
40
37
|
did_type: str = 'collection',
|
|
41
38
|
ignore_case: bool = False,
|
|
42
39
|
limit: Optional[int] = None,
|
|
@@ -44,8 +41,6 @@ def list_dids(
|
|
|
44
41
|
long: bool = False,
|
|
45
42
|
recursive: bool = False,
|
|
46
43
|
vo: str = 'def',
|
|
47
|
-
*,
|
|
48
|
-
session: "Session"
|
|
49
44
|
) -> 'Iterator[dict[str, Any]]':
|
|
50
45
|
"""
|
|
51
46
|
List dids in a scope.
|
|
@@ -59,7 +54,6 @@ def list_dids(
|
|
|
59
54
|
:param long: Long format option to display more information for each DID.
|
|
60
55
|
:param recursive: Recursively list DIDs content.
|
|
61
56
|
:param vo: The VO to act on.
|
|
62
|
-
:param session: The database session in use.
|
|
63
57
|
"""
|
|
64
58
|
internal_scope = InternalScope(scope, vo=vo)
|
|
65
59
|
|
|
@@ -70,14 +64,14 @@ def list_dids(
|
|
|
70
64
|
if 'scope' in or_group:
|
|
71
65
|
or_group['account'] = InternalScope(or_group['scope'], vo=vo)
|
|
72
66
|
|
|
73
|
-
|
|
74
|
-
|
|
67
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
68
|
+
result = did.list_dids(scope=internal_scope, filters=filters, did_type=did_type, ignore_case=ignore_case,
|
|
69
|
+
limit=limit, offset=offset, long=long, recursive=recursive, session=session)
|
|
75
70
|
|
|
76
|
-
|
|
77
|
-
|
|
71
|
+
for d in result:
|
|
72
|
+
yield gateway_update_return_dict(d, session=session)
|
|
78
73
|
|
|
79
74
|
|
|
80
|
-
@transactional_session
|
|
81
75
|
def add_did(
|
|
82
76
|
scope: str,
|
|
83
77
|
name: str,
|
|
@@ -91,8 +85,6 @@ def add_did(
|
|
|
91
85
|
dids: Optional['Sequence[dict[str, Any]]'] = None,
|
|
92
86
|
rse: Optional[str] = None,
|
|
93
87
|
vo: str = 'def',
|
|
94
|
-
*,
|
|
95
|
-
session: "Session"
|
|
96
88
|
) -> None:
|
|
97
89
|
"""
|
|
98
90
|
Add data did.
|
|
@@ -109,7 +101,6 @@ def add_did(
|
|
|
109
101
|
:param dids: The content.
|
|
110
102
|
:param rse: The RSE name when registering replicas.
|
|
111
103
|
:param vo: The VO to act on.
|
|
112
|
-
:param session: The database session in use.
|
|
113
104
|
"""
|
|
114
105
|
statuses = statuses or {}
|
|
115
106
|
meta = meta or {}
|
|
@@ -120,49 +111,48 @@ def add_did(
|
|
|
120
111
|
validate_schema(name='dids', obj=dids, vo=vo)
|
|
121
112
|
validate_schema(name='rse', obj=rse, vo=vo)
|
|
122
113
|
kwargs = {'scope': scope, 'name': name, 'type': did_type, 'issuer': issuer, 'account': account, 'statuses': statuses, 'meta': meta, 'rules': rules, 'lifetime': lifetime}
|
|
123
|
-
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_did', kwargs=kwargs, session=session)
|
|
124
|
-
if not auth_result.allowed:
|
|
125
|
-
raise AccessDenied('Account %s can not add data identifier to scope %s. %s' % (issuer, scope, auth_result.message))
|
|
126
114
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
115
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
116
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_did', kwargs=kwargs, session=session)
|
|
117
|
+
if not auth_result.allowed:
|
|
118
|
+
raise AccessDenied('Account %s can not add data identifier to scope %s. %s' % (issuer, scope, auth_result.message))
|
|
119
|
+
|
|
120
|
+
owner_account = None if account is None else InternalAccount(account, vo=vo)
|
|
121
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
122
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
123
|
+
for d in dids:
|
|
124
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
125
|
+
for r in rules:
|
|
126
|
+
r['account'] = InternalAccount(r['account'], vo=vo)
|
|
134
127
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
128
|
+
rse_id = None
|
|
129
|
+
if rse is not None:
|
|
130
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
138
131
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
132
|
+
if did_type == 'DATASET':
|
|
133
|
+
# naming_convention validation
|
|
134
|
+
extra_meta = naming_convention.validate_name(scope=internal_scope, name=name, did_type='D', session=session)
|
|
142
135
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
136
|
+
# merge extra_meta with meta
|
|
137
|
+
for k in extra_meta or {}:
|
|
138
|
+
if k not in meta:
|
|
139
|
+
meta[k] = extra_meta[k]
|
|
140
|
+
elif meta[k] != extra_meta[k]:
|
|
141
|
+
print("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
|
|
142
|
+
raise InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
|
|
150
143
|
|
|
151
|
-
|
|
152
|
-
|
|
144
|
+
# Validate metadata
|
|
145
|
+
meta_convention_core.validate_meta(meta=meta, did_type=DIDType[did_type.upper()], session=session)
|
|
153
146
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
147
|
+
return did.add_did(scope=internal_scope, name=name, did_type=DIDType[did_type.upper()], account=owner_account or issuer_account,
|
|
148
|
+
statuses=statuses, meta=meta, rules=rules, lifetime=lifetime,
|
|
149
|
+
dids=dids, rse_id=rse_id, session=session)
|
|
157
150
|
|
|
158
151
|
|
|
159
|
-
@transactional_session
|
|
160
152
|
def add_dids(
|
|
161
153
|
dids: 'Sequence[dict[str, Any]]',
|
|
162
154
|
issuer: str,
|
|
163
155
|
vo: str = 'def',
|
|
164
|
-
*,
|
|
165
|
-
session: "Session"
|
|
166
156
|
) -> None:
|
|
167
157
|
"""
|
|
168
158
|
Bulk Add did.
|
|
@@ -170,40 +160,37 @@ def add_dids(
|
|
|
170
160
|
:param dids: A list of dids.
|
|
171
161
|
:param issuer: The issuer account.
|
|
172
162
|
:param vo: The VO to act on.
|
|
173
|
-
:param session: The database session in use.
|
|
174
163
|
"""
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
d
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
child
|
|
195
|
-
|
|
164
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
165
|
+
for d in dids:
|
|
166
|
+
if 'rse' in d:
|
|
167
|
+
rse_id = None
|
|
168
|
+
if d['rse'] is not None:
|
|
169
|
+
rse_id = get_rse_id(rse=d['rse'], vo=vo, session=session)
|
|
170
|
+
d['rse_id'] = rse_id
|
|
171
|
+
|
|
172
|
+
kwargs = {'issuer': issuer, 'dids': dids}
|
|
173
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='add_dids', kwargs=kwargs, session=session)
|
|
174
|
+
if not auth_result.allowed:
|
|
175
|
+
raise AccessDenied('Account %s can not bulk add data identifier. %s' % (issuer, auth_result.message))
|
|
176
|
+
|
|
177
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
178
|
+
for d in dids:
|
|
179
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
180
|
+
if 'account' in d.keys():
|
|
181
|
+
d['account'] = InternalAccount(d['account'], vo=vo)
|
|
182
|
+
if 'dids' in d.keys():
|
|
183
|
+
for child in d['dids']:
|
|
184
|
+
child['scope'] = InternalScope(child['scope'], vo=vo)
|
|
185
|
+
return did.add_dids(dids, account=issuer_account, session=session)
|
|
196
186
|
|
|
197
187
|
|
|
198
|
-
@transactional_session
|
|
199
188
|
def attach_dids(
|
|
200
189
|
scope: str,
|
|
201
190
|
name: str,
|
|
202
191
|
attachment: dict[str, Any],
|
|
203
192
|
issuer: str,
|
|
204
193
|
vo='def',
|
|
205
|
-
*,
|
|
206
|
-
session: "Session"
|
|
207
194
|
) -> None:
|
|
208
195
|
"""
|
|
209
196
|
Append content to data did.
|
|
@@ -211,48 +198,46 @@ def attach_dids(
|
|
|
211
198
|
:param attachment: The attachment.
|
|
212
199
|
:param issuer: The issuer account.
|
|
213
200
|
:param vo: The VO to act on.
|
|
214
|
-
:param session: The database session in use.
|
|
215
201
|
"""
|
|
216
202
|
validate_schema(name='attachment', obj=attachment, vo=vo)
|
|
217
203
|
|
|
218
|
-
|
|
219
|
-
if 'rse' in attachment:
|
|
220
|
-
if attachment['rse'] is not None:
|
|
221
|
-
rse_id = get_rse_id(rse=attachment['rse'], vo=vo, session=session)
|
|
222
|
-
attachment['rse_id'] = rse_id
|
|
204
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
223
205
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
206
|
+
rse_id = None
|
|
207
|
+
if 'rse' in attachment:
|
|
208
|
+
if attachment['rse'] is not None:
|
|
209
|
+
rse_id = get_rse_id(rse=attachment['rse'], vo=vo, session=session)
|
|
210
|
+
attachment['rse_id'] = rse_id
|
|
228
211
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
for d in attachment['dids']:
|
|
234
|
-
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
235
|
-
if 'account' in d.keys():
|
|
236
|
-
d['account'] = InternalAccount(d['account'], vo=vo)
|
|
212
|
+
kwargs = {'scope': scope, 'name': name, 'attachment': attachment}
|
|
213
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='attach_dids', kwargs=kwargs, session=session)
|
|
214
|
+
if not auth_result.allowed:
|
|
215
|
+
raise AccessDenied('Account %s can not add data identifiers to %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
237
216
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
217
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
218
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
219
|
+
if 'account' in attachment.keys():
|
|
220
|
+
attachment['account'] = InternalAccount(attachment['account'], vo=vo)
|
|
221
|
+
for d in attachment['dids']:
|
|
222
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
223
|
+
if 'account' in d.keys():
|
|
224
|
+
d['account'] = InternalAccount(d['account'], vo=vo)
|
|
225
|
+
|
|
226
|
+
if rse_id is not None:
|
|
227
|
+
dids = did.attach_dids(scope=internal_scope, name=name, dids=attachment['dids'],
|
|
228
|
+
account=attachment.get('account', issuer_account), rse_id=rse_id, session=session)
|
|
229
|
+
else:
|
|
230
|
+
dids = did.attach_dids(scope=internal_scope, name=name, dids=attachment['dids'],
|
|
231
|
+
account=attachment.get('account', issuer_account), session=session)
|
|
244
232
|
|
|
245
233
|
return dids
|
|
246
234
|
|
|
247
235
|
|
|
248
|
-
@transactional_session
|
|
249
236
|
def attach_dids_to_dids(
|
|
250
237
|
attachments: 'Sequence[dict[str, Any]]',
|
|
251
238
|
issuer: str,
|
|
252
239
|
ignore_duplicate: bool = False,
|
|
253
240
|
vo: str = 'def',
|
|
254
|
-
*,
|
|
255
|
-
session: "Session"
|
|
256
241
|
) -> None:
|
|
257
242
|
"""
|
|
258
243
|
Append content to dids.
|
|
@@ -261,42 +246,40 @@ def attach_dids_to_dids(
|
|
|
261
246
|
:param issuer: The issuer account.
|
|
262
247
|
:param ignore_duplicate: If True, ignore duplicate entries.
|
|
263
248
|
:param vo: The VO to act on.
|
|
264
|
-
:param session: The database session in use.
|
|
265
249
|
"""
|
|
266
250
|
validate_schema(name='attachments', obj=attachments, vo=vo)
|
|
267
251
|
|
|
268
|
-
|
|
269
|
-
if 'rse' in a:
|
|
270
|
-
rse_id = None
|
|
271
|
-
if a['rse'] is not None:
|
|
272
|
-
rse_id = get_rse_id(rse=a['rse'], vo=vo, session=session)
|
|
273
|
-
a['rse_id'] = rse_id
|
|
252
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
274
253
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
254
|
+
for a in attachments:
|
|
255
|
+
if 'rse' in a:
|
|
256
|
+
rse_id = None
|
|
257
|
+
if a['rse'] is not None:
|
|
258
|
+
rse_id = get_rse_id(rse=a['rse'], vo=vo, session=session)
|
|
259
|
+
a['rse_id'] = rse_id
|
|
278
260
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
for d in attachment['dids']:
|
|
283
|
-
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
284
|
-
if 'account' in d.keys():
|
|
285
|
-
d['account'] = InternalAccount(d['account'], vo=vo)
|
|
261
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='attach_dids_to_dids', kwargs={'attachments': attachments}, session=session)
|
|
262
|
+
if not auth_result.allowed:
|
|
263
|
+
raise AccessDenied('Account %s can not add data identifiers. %s' % (issuer, auth_result.message))
|
|
286
264
|
|
|
287
|
-
|
|
288
|
-
|
|
265
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
266
|
+
for attachment in attachments:
|
|
267
|
+
attachment['scope'] = InternalScope(attachment['scope'], vo=vo)
|
|
268
|
+
for d in attachment['dids']:
|
|
269
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
270
|
+
if 'account' in d.keys():
|
|
271
|
+
d['account'] = InternalAccount(d['account'], vo=vo)
|
|
272
|
+
|
|
273
|
+
return did.attach_dids_to_dids(attachments=attachments, account=issuer_account,
|
|
274
|
+
ignore_duplicate=ignore_duplicate, session=session)
|
|
289
275
|
|
|
290
276
|
|
|
291
|
-
@transactional_session
|
|
292
277
|
def detach_dids(
|
|
293
278
|
scope: str,
|
|
294
279
|
name: str,
|
|
295
280
|
dids: 'Sequence[dict[str, Any]]',
|
|
296
281
|
issuer: str,
|
|
297
282
|
vo: str = 'def',
|
|
298
|
-
*,
|
|
299
|
-
session: "Session"
|
|
300
283
|
) -> None:
|
|
301
284
|
"""
|
|
302
285
|
Detach data identifier
|
|
@@ -306,29 +289,27 @@ def detach_dids(
|
|
|
306
289
|
:param dids: The content.
|
|
307
290
|
:param issuer: The issuer account.
|
|
308
291
|
:param vo: The VO to act on.
|
|
309
|
-
:param session: The database session in use.
|
|
310
292
|
"""
|
|
311
293
|
kwargs = {'scope': scope, 'name': name, 'dids': dids, 'issuer': issuer}
|
|
312
|
-
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='detach_dids', kwargs=kwargs, session=session)
|
|
313
|
-
if not auth_result.allowed:
|
|
314
|
-
raise AccessDenied('Account %s can not detach data identifiers from %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
315
294
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
295
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
296
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='detach_dids', kwargs=kwargs, session=session)
|
|
297
|
+
if not auth_result.allowed:
|
|
298
|
+
raise AccessDenied('Account %s can not detach data identifiers from %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
319
299
|
|
|
320
|
-
|
|
300
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
301
|
+
for d in dids:
|
|
302
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
303
|
+
|
|
304
|
+
return did.detach_dids(scope=internal_scope, name=name, dids=dids, session=session)
|
|
321
305
|
|
|
322
306
|
|
|
323
|
-
@stream_session
|
|
324
307
|
def list_new_dids(
|
|
325
308
|
did_type: Optional[str] = None,
|
|
326
309
|
thread: Optional[int] = None,
|
|
327
310
|
total_threads: Optional[int] = None,
|
|
328
311
|
chunk_size: int = 1000,
|
|
329
312
|
vo: str = 'def',
|
|
330
|
-
*,
|
|
331
|
-
session: "Session"
|
|
332
313
|
) -> 'Iterator[dict[str, Any]]':
|
|
333
314
|
"""
|
|
334
315
|
List recent identifiers.
|
|
@@ -338,21 +319,21 @@ def list_new_dids(
|
|
|
338
319
|
:param total_threads: The total number of threads of all necromancers.
|
|
339
320
|
:param chunk_size: Number of requests to return per yield.
|
|
340
321
|
:param vo: The VO to act on.
|
|
341
|
-
:param session: The database session in use.
|
|
342
322
|
"""
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
323
|
+
|
|
324
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
325
|
+
dids = did.list_new_dids(did_type=did_type and DIDType[did_type.upper()], thread=thread, total_threads=total_threads, chunk_size=chunk_size, session=session)
|
|
326
|
+
for d in dids:
|
|
327
|
+
if d['scope'].vo == vo:
|
|
328
|
+
d = gateway_update_return_dict(d, session=session)
|
|
329
|
+
|
|
330
|
+
yield from dids
|
|
347
331
|
|
|
348
332
|
|
|
349
|
-
@transactional_session
|
|
350
333
|
def set_new_dids(
|
|
351
334
|
dids: 'Sequence[dict[str, Any]]',
|
|
352
335
|
new_flag: bool = True,
|
|
353
336
|
vo: str = 'def',
|
|
354
|
-
*,
|
|
355
|
-
session: "Session"
|
|
356
337
|
) -> bool:
|
|
357
338
|
"""
|
|
358
339
|
Set/reset the flag new
|
|
@@ -361,21 +342,18 @@ def set_new_dids(
|
|
|
361
342
|
:param name: The data identifier name.
|
|
362
343
|
:param new_flag: A boolean to flag new DIDs.
|
|
363
344
|
:param vo: The VO to act on.
|
|
364
|
-
:param session: The database session in use.
|
|
365
345
|
"""
|
|
366
346
|
for d in dids:
|
|
367
347
|
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
368
348
|
|
|
369
|
-
|
|
349
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
350
|
+
return did.set_new_dids(dids, new_flag, session=session)
|
|
370
351
|
|
|
371
352
|
|
|
372
|
-
@stream_session
|
|
373
353
|
def list_content(
|
|
374
354
|
scope: str,
|
|
375
355
|
name: str,
|
|
376
356
|
vo: str = 'def',
|
|
377
|
-
*,
|
|
378
|
-
session: "Session"
|
|
379
357
|
) -> 'Iterator[dict[str, Any]]':
|
|
380
358
|
"""
|
|
381
359
|
List data identifier contents.
|
|
@@ -383,23 +361,20 @@ def list_content(
|
|
|
383
361
|
:param scope: The scope name.
|
|
384
362
|
:param name: The data identifier name.
|
|
385
363
|
:param vo: The VO to act on.
|
|
386
|
-
:param session: The database session in use.
|
|
387
364
|
"""
|
|
388
365
|
|
|
389
366
|
internal_scope = InternalScope(scope, vo=vo)
|
|
390
367
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
368
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
369
|
+
dids = did.list_content(scope=internal_scope, name=name, session=session)
|
|
370
|
+
for d in dids:
|
|
371
|
+
yield gateway_update_return_dict(d, session=session)
|
|
394
372
|
|
|
395
373
|
|
|
396
|
-
@stream_session
|
|
397
374
|
def list_content_history(
|
|
398
375
|
scope: str,
|
|
399
376
|
name: str,
|
|
400
377
|
vo='def',
|
|
401
|
-
*,
|
|
402
|
-
session: "Session"
|
|
403
378
|
) -> 'Iterator[dict[str, Any]]':
|
|
404
379
|
"""
|
|
405
380
|
List data identifier contents history.
|
|
@@ -407,24 +382,21 @@ def list_content_history(
|
|
|
407
382
|
:param scope: The scope name.
|
|
408
383
|
:param name: The data identifier name.
|
|
409
384
|
:param vo: The VO to act on.
|
|
410
|
-
:param session: The database session in use.
|
|
411
385
|
"""
|
|
412
386
|
|
|
413
387
|
internal_scope = InternalScope(scope, vo=vo)
|
|
414
388
|
|
|
415
|
-
|
|
389
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
390
|
+
dids = did.list_content_history(scope=internal_scope, name=name, session=session)
|
|
416
391
|
|
|
417
|
-
|
|
418
|
-
|
|
392
|
+
for d in dids:
|
|
393
|
+
yield gateway_update_return_dict(d, session=session)
|
|
419
394
|
|
|
420
395
|
|
|
421
|
-
@stream_session
|
|
422
396
|
def bulk_list_files(
|
|
423
397
|
dids: 'Iterable[dict[str, Any]]',
|
|
424
398
|
long: bool = False,
|
|
425
399
|
vo: str = 'def',
|
|
426
|
-
*,
|
|
427
|
-
session: "Session"
|
|
428
400
|
) -> 'Iterator[dict[str, Any]]':
|
|
429
401
|
"""
|
|
430
402
|
List file contents of a list of data identifiers.
|
|
@@ -432,24 +404,21 @@ def bulk_list_files(
|
|
|
432
404
|
:param dids: A list of DIDs.
|
|
433
405
|
:param long: A boolean to choose if more metadata are returned or not.
|
|
434
406
|
:param vo: The VO to act on.
|
|
435
|
-
:param session: The database session in use.
|
|
436
407
|
"""
|
|
437
408
|
|
|
438
409
|
for did_ in dids:
|
|
439
410
|
did_['scope'] = InternalScope(did_['scope'], vo=vo)
|
|
440
411
|
|
|
441
|
-
|
|
442
|
-
|
|
412
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
413
|
+
for file_ in did.bulk_list_files(dids=dids, long=long, session=session):
|
|
414
|
+
yield gateway_update_return_dict(file_, session=session)
|
|
443
415
|
|
|
444
416
|
|
|
445
|
-
@stream_session
|
|
446
417
|
def list_files(
|
|
447
418
|
scope: str,
|
|
448
419
|
name: str,
|
|
449
420
|
long: bool,
|
|
450
421
|
vo: str = 'def',
|
|
451
|
-
*,
|
|
452
|
-
session: "Session"
|
|
453
422
|
) -> 'Iterator[dict[str, Any]]':
|
|
454
423
|
"""
|
|
455
424
|
List data identifier file contents.
|
|
@@ -458,25 +427,22 @@ def list_files(
|
|
|
458
427
|
:param name: The data identifier name.
|
|
459
428
|
:param long: A boolean to choose if GUID is returned or not.
|
|
460
429
|
:param vo: The VO to act on.
|
|
461
|
-
:param session: The database session in use.
|
|
462
430
|
"""
|
|
463
431
|
|
|
464
432
|
internal_scope = InternalScope(scope, vo=vo)
|
|
465
433
|
|
|
466
|
-
|
|
434
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
435
|
+
dids = did.list_files(scope=internal_scope, name=name, long=long, session=session)
|
|
467
436
|
|
|
468
|
-
|
|
469
|
-
|
|
437
|
+
for d in dids:
|
|
438
|
+
yield gateway_update_return_dict(d, session=session)
|
|
470
439
|
|
|
471
440
|
|
|
472
|
-
@stream_session
|
|
473
441
|
def scope_list(
|
|
474
442
|
scope: str,
|
|
475
443
|
name: Optional[str] = None,
|
|
476
444
|
recursive: bool = False,
|
|
477
445
|
vo: str = 'def',
|
|
478
|
-
*,
|
|
479
|
-
session: "Session"
|
|
480
446
|
) -> 'Iterator[dict[str, Any]]':
|
|
481
447
|
"""
|
|
482
448
|
List data identifiers in a scope.
|
|
@@ -485,23 +451,22 @@ def scope_list(
|
|
|
485
451
|
:param name: The data identifier name.
|
|
486
452
|
:param recursive: boolean, True or False.
|
|
487
453
|
:param vo: The VO to act on.
|
|
488
|
-
:param session: The database session in use.
|
|
489
454
|
"""
|
|
490
455
|
|
|
491
456
|
internal_scope = InternalScope(scope, vo=vo)
|
|
492
457
|
|
|
493
|
-
|
|
458
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
459
|
+
dids = did.scope_list(internal_scope, name=name, recursive=recursive, session=session)
|
|
494
460
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
461
|
+
for d in dids:
|
|
462
|
+
ret_did = deepcopy(d)
|
|
463
|
+
ret_did['scope'] = ret_did['scope'].external
|
|
464
|
+
if ret_did['parent'] is not None:
|
|
465
|
+
ret_did['parent']['scope'] = ret_did['parent']['scope'].external
|
|
466
|
+
yield ret_did
|
|
501
467
|
|
|
502
468
|
|
|
503
|
-
|
|
504
|
-
def get_did(scope: str, name: str, dynamic_depth: Optional[DIDType] = None, vo: str = 'def', *, session: "Session") -> "dict[str, Any]":
|
|
469
|
+
def get_did(scope: str, name: str, dynamic_depth: Optional[DIDType] = None, vo: str = 'def') -> "dict[str, Any]":
|
|
505
470
|
"""
|
|
506
471
|
Retrieve a single data did.
|
|
507
472
|
|
|
@@ -512,16 +477,15 @@ def get_did(scope: str, name: str, dynamic_depth: Optional[DIDType] = None, vo:
|
|
|
512
477
|
will not compute the size dynamically.
|
|
513
478
|
:param vo: The VO to act on.
|
|
514
479
|
:return did: Dictionary containing {'name', 'scope', 'type'}, Exception otherwise
|
|
515
|
-
:param session: The database session in use.
|
|
516
480
|
"""
|
|
517
481
|
|
|
518
482
|
internal_scope = InternalScope(scope, vo=vo)
|
|
519
483
|
|
|
520
|
-
|
|
521
|
-
|
|
484
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
485
|
+
d = did.get_did(scope=internal_scope, name=name, dynamic_depth=dynamic_depth, session=session)
|
|
486
|
+
return gateway_update_return_dict(d, session=session)
|
|
522
487
|
|
|
523
488
|
|
|
524
|
-
@transactional_session
|
|
525
489
|
def set_metadata(
|
|
526
490
|
scope: str,
|
|
527
491
|
name: str,
|
|
@@ -530,8 +494,6 @@ def set_metadata(
|
|
|
530
494
|
issuer: str,
|
|
531
495
|
recursive: bool = False,
|
|
532
496
|
vo: str = 'def',
|
|
533
|
-
*,
|
|
534
|
-
session: "Session"
|
|
535
497
|
) -> None:
|
|
536
498
|
"""
|
|
537
499
|
Add metadata to data did.
|
|
@@ -543,22 +505,21 @@ def set_metadata(
|
|
|
543
505
|
:param issuer: The issuer account.
|
|
544
506
|
:param recursive: Option to propagate the metadata update to content.
|
|
545
507
|
:param vo: The VO to act on.
|
|
546
|
-
:param session: The database session in use.
|
|
547
508
|
"""
|
|
548
509
|
kwargs = {'scope': scope, 'name': name, 'key': key, 'value': value, 'issuer': issuer}
|
|
549
510
|
|
|
550
511
|
if key in RESERVED_KEYS:
|
|
551
512
|
raise AccessDenied('Account %s can not change this metadata value to data identifier %s:%s' % (issuer, scope, name))
|
|
552
513
|
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
514
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
515
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='set_metadata', kwargs=kwargs, session=session)
|
|
516
|
+
if not auth_result.allowed:
|
|
517
|
+
raise AccessDenied('Account %s can not add metadata to data identifier %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
556
518
|
|
|
557
|
-
|
|
558
|
-
|
|
519
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
520
|
+
return did.set_metadata(scope=internal_scope, name=name, key=key, value=value, recursive=recursive, session=session)
|
|
559
521
|
|
|
560
522
|
|
|
561
|
-
@transactional_session
|
|
562
523
|
def set_metadata_bulk(
|
|
563
524
|
scope: str,
|
|
564
525
|
name: str,
|
|
@@ -566,8 +527,6 @@ def set_metadata_bulk(
|
|
|
566
527
|
issuer: str,
|
|
567
528
|
recursive: bool = False,
|
|
568
529
|
vo: str = 'def',
|
|
569
|
-
*,
|
|
570
|
-
session: "Session"
|
|
571
530
|
) -> None:
|
|
572
531
|
"""
|
|
573
532
|
Add metadata to data did.
|
|
@@ -578,7 +537,6 @@ def set_metadata_bulk(
|
|
|
578
537
|
:param issuer: The issuer account.
|
|
579
538
|
:param recursive: Option to propagate the metadata update to content.
|
|
580
539
|
:param vo: The VO to act on.
|
|
581
|
-
:param session: The database session in use.
|
|
582
540
|
"""
|
|
583
541
|
kwargs = {'scope': scope, 'name': name, 'meta': meta, 'issuer': issuer}
|
|
584
542
|
|
|
@@ -586,22 +544,20 @@ def set_metadata_bulk(
|
|
|
586
544
|
if key in RESERVED_KEYS:
|
|
587
545
|
raise AccessDenied('Account %s can not change the value of the metadata key %s to data identifier %s:%s' % (issuer, key, scope, name))
|
|
588
546
|
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
547
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
548
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='set_metadata_bulk', kwargs=kwargs, session=session)
|
|
549
|
+
if not auth_result.allowed:
|
|
550
|
+
raise AccessDenied('Account %s can not add metadata to data identifier %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
592
551
|
|
|
593
|
-
|
|
594
|
-
|
|
552
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
553
|
+
return did.set_metadata_bulk(scope=internal_scope, name=name, meta=meta, recursive=recursive, session=session)
|
|
595
554
|
|
|
596
555
|
|
|
597
|
-
@transactional_session
|
|
598
556
|
def set_dids_metadata_bulk(
|
|
599
557
|
dids: 'Iterable[dict[str, Any]]',
|
|
600
558
|
issuer: str,
|
|
601
559
|
recursive: bool = False,
|
|
602
560
|
vo: str = 'def',
|
|
603
|
-
*,
|
|
604
|
-
session: "Session"
|
|
605
561
|
) -> None:
|
|
606
562
|
"""
|
|
607
563
|
Add metadata to a list of data identifiers.
|
|
@@ -610,31 +566,28 @@ def set_dids_metadata_bulk(
|
|
|
610
566
|
:param dids: A list of dids including metadata.
|
|
611
567
|
:param recursive: Option to propagate the metadata update to content.
|
|
612
568
|
:param vo: The VO to act on.
|
|
613
|
-
:param session: The database session in use.
|
|
614
569
|
"""
|
|
615
570
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
571
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
572
|
+
for entry in dids:
|
|
573
|
+
kwargs = {'scope': entry['scope'], 'name': entry['name'], 'meta': entry['meta'], 'issuer': issuer}
|
|
574
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='set_metadata_bulk', kwargs=kwargs, session=session)
|
|
575
|
+
if not auth_result.allowed:
|
|
576
|
+
raise AccessDenied('Account %s can not add metadata to data identifier %s:%s. %s' % (issuer, entry['scope'], entry['name'], auth_result.message))
|
|
577
|
+
entry['scope'] = InternalScope(entry['scope'], vo=vo)
|
|
578
|
+
meta = entry['meta']
|
|
579
|
+
for key in meta:
|
|
580
|
+
if key in RESERVED_KEYS:
|
|
581
|
+
raise AccessDenied('Account %s can not change the value of the metadata key %s to data identifier %s:%s' % (issuer, key, entry['scope'], entry['name']))
|
|
626
582
|
|
|
627
|
-
|
|
583
|
+
return did.set_dids_metadata_bulk(dids=dids, recursive=recursive, session=session)
|
|
628
584
|
|
|
629
585
|
|
|
630
|
-
@read_session
|
|
631
586
|
def get_metadata(
|
|
632
587
|
scope: str,
|
|
633
588
|
name: str,
|
|
634
589
|
plugin: str = 'DID_COLUMN',
|
|
635
590
|
vo: str = 'def',
|
|
636
|
-
*,
|
|
637
|
-
session: "Session"
|
|
638
591
|
) -> dict[str, Any]:
|
|
639
592
|
"""
|
|
640
593
|
Get data identifier metadata
|
|
@@ -643,23 +596,20 @@ def get_metadata(
|
|
|
643
596
|
:param name: The data identifier name.
|
|
644
597
|
:param vo: The VO to act on.
|
|
645
598
|
:param plugin: The metadata plugin to query, 'ALL' for all available plugins
|
|
646
|
-
:param session: The database session in use.
|
|
647
599
|
"""
|
|
648
600
|
|
|
649
601
|
internal_scope = InternalScope(scope, vo=vo)
|
|
650
602
|
|
|
651
|
-
|
|
652
|
-
|
|
603
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
604
|
+
d = did.get_metadata(scope=internal_scope, name=name, plugin=plugin, session=session)
|
|
605
|
+
return gateway_update_return_dict(d, session=session)
|
|
653
606
|
|
|
654
607
|
|
|
655
|
-
@stream_session
|
|
656
608
|
def get_metadata_bulk(
|
|
657
609
|
dids: 'Iterable[dict[str, Any]]',
|
|
658
610
|
inherit: bool = False,
|
|
659
611
|
plugin: str = 'DID_COLUMN',
|
|
660
612
|
vo: str = 'def',
|
|
661
|
-
*,
|
|
662
|
-
session: "Session"
|
|
663
613
|
) -> 'Iterator[dict[str, Any]]':
|
|
664
614
|
"""
|
|
665
615
|
Get metadata for a list of dids
|
|
@@ -667,25 +617,23 @@ def get_metadata_bulk(
|
|
|
667
617
|
:param inherit: A boolean. If set to true, the metadata of the parent are concatenated.
|
|
668
618
|
:param plugin: The metadata plugin to query, 'ALL' for all available plugins
|
|
669
619
|
:param vo: The VO to act on.
|
|
670
|
-
:param session: The database session in use.
|
|
671
620
|
"""
|
|
672
621
|
|
|
673
622
|
validate_schema(name='dids', obj=dids, vo=vo)
|
|
674
623
|
for entry in dids:
|
|
675
624
|
entry['scope'] = InternalScope(entry['scope'], vo=vo)
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
625
|
+
|
|
626
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
627
|
+
meta = did.get_metadata_bulk(dids, inherit=inherit, plugin=plugin, session=session)
|
|
628
|
+
for met in meta:
|
|
629
|
+
yield gateway_update_return_dict(met, session=session)
|
|
679
630
|
|
|
680
631
|
|
|
681
|
-
@transactional_session
|
|
682
632
|
def delete_metadata(
|
|
683
633
|
scope: str,
|
|
684
634
|
name: str,
|
|
685
635
|
key: str,
|
|
686
636
|
vo: str = 'def',
|
|
687
|
-
*,
|
|
688
|
-
session: "Session"
|
|
689
637
|
) -> None:
|
|
690
638
|
"""
|
|
691
639
|
Delete a key from the metadata column
|
|
@@ -694,21 +642,18 @@ def delete_metadata(
|
|
|
694
642
|
:param name: the name of the did
|
|
695
643
|
:param key: the key to be deleted
|
|
696
644
|
:param vo: The VO to act on.
|
|
697
|
-
:param session: The database session in use.
|
|
698
645
|
"""
|
|
699
646
|
|
|
700
647
|
internal_scope = InternalScope(scope, vo=vo)
|
|
701
|
-
|
|
648
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
649
|
+
return did.delete_metadata(scope=internal_scope, name=name, key=key, session=session)
|
|
702
650
|
|
|
703
651
|
|
|
704
|
-
@transactional_session
|
|
705
652
|
def set_status(
|
|
706
653
|
scope: str,
|
|
707
654
|
name: str,
|
|
708
655
|
issuer: str,
|
|
709
656
|
vo: str = 'def',
|
|
710
|
-
*,
|
|
711
|
-
session: "Session",
|
|
712
657
|
**kwargs
|
|
713
658
|
) -> None:
|
|
714
659
|
"""
|
|
@@ -719,48 +664,42 @@ def set_status(
|
|
|
719
664
|
:param issuer: The issuer account.
|
|
720
665
|
:param kwargs: Keyword arguments of the form status_name=value.
|
|
721
666
|
:param vo: The VO to act on.
|
|
722
|
-
:param session: The database session in use.
|
|
723
667
|
"""
|
|
724
668
|
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
669
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
670
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='set_status', kwargs={'scope': scope, 'name': name, 'issuer': issuer}, session=session)
|
|
671
|
+
if not auth_result.allowed:
|
|
672
|
+
raise AccessDenied('Account %s can not set status on data identifier %s:%s. %s' % (issuer, scope, name, auth_result.message))
|
|
728
673
|
|
|
729
|
-
|
|
674
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
730
675
|
|
|
731
|
-
|
|
676
|
+
return did.set_status(scope=internal_scope, name=name, session=session, **kwargs)
|
|
732
677
|
|
|
733
678
|
|
|
734
|
-
@stream_session
|
|
735
679
|
def get_dataset_by_guid(
|
|
736
680
|
guid: str,
|
|
737
681
|
vo: str = 'def',
|
|
738
|
-
*,
|
|
739
|
-
session: "Session"
|
|
740
682
|
) -> 'Iterator[dict[str, Any]]':
|
|
741
683
|
"""
|
|
742
684
|
Get the parent datasets for a given GUID.
|
|
743
685
|
:param guid: The GUID.
|
|
744
686
|
:param vo: The VO to act on.
|
|
745
|
-
:param session: The database session in use.
|
|
746
687
|
|
|
747
688
|
:returns: A did
|
|
748
689
|
"""
|
|
749
|
-
|
|
690
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
691
|
+
dids = did.get_dataset_by_guid(guid=guid, session=session)
|
|
750
692
|
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
693
|
+
for d in dids:
|
|
694
|
+
if d['scope'].vo != vo:
|
|
695
|
+
raise RucioException('GUID unavailable on VO {}'.format(vo))
|
|
696
|
+
yield gateway_update_return_dict(d, session=session)
|
|
755
697
|
|
|
756
698
|
|
|
757
|
-
@stream_session
|
|
758
699
|
def list_parent_dids(
|
|
759
700
|
scope: str,
|
|
760
701
|
name: str,
|
|
761
702
|
vo: str = 'def',
|
|
762
|
-
*,
|
|
763
|
-
session: "Session"
|
|
764
703
|
) -> 'Iterator[dict[str, Any]]':
|
|
765
704
|
"""
|
|
766
705
|
List parent datasets and containers of a did.
|
|
@@ -768,18 +707,17 @@ def list_parent_dids(
|
|
|
768
707
|
:param scope: The scope.
|
|
769
708
|
:param name: The name.
|
|
770
709
|
:param vo: The VO to act on.
|
|
771
|
-
:param session: The database session in use.
|
|
772
710
|
"""
|
|
773
711
|
|
|
774
712
|
internal_scope = InternalScope(scope, vo=vo)
|
|
775
713
|
|
|
776
|
-
|
|
714
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
715
|
+
dids = did.list_parent_dids(scope=internal_scope, name=name, session=session)
|
|
777
716
|
|
|
778
|
-
|
|
779
|
-
|
|
717
|
+
for d in dids:
|
|
718
|
+
yield gateway_update_return_dict(d, session=session)
|
|
780
719
|
|
|
781
720
|
|
|
782
|
-
@transactional_session
|
|
783
721
|
def create_did_sample(
|
|
784
722
|
input_scope: str,
|
|
785
723
|
input_name: str,
|
|
@@ -788,8 +726,6 @@ def create_did_sample(
|
|
|
788
726
|
issuer: str,
|
|
789
727
|
nbfiles: str,
|
|
790
728
|
vo: str = 'def',
|
|
791
|
-
*,
|
|
792
|
-
session: "Session"
|
|
793
729
|
):
|
|
794
730
|
"""
|
|
795
731
|
Create a sample from an input collection.
|
|
@@ -802,29 +738,27 @@ def create_did_sample(
|
|
|
802
738
|
:param nbfiles: The number of files to register in the output dataset.
|
|
803
739
|
:param issuer: The issuer account.
|
|
804
740
|
:param vo: The VO to act on.
|
|
805
|
-
:param session: The database session in use.
|
|
806
741
|
"""
|
|
807
742
|
kwargs = {'issuer': issuer, 'scope': output_scope}
|
|
808
|
-
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='create_did_sample', kwargs=kwargs, session=session)
|
|
809
|
-
if not auth_result.allowed:
|
|
810
|
-
raise AccessDenied('Account %s can not bulk add data identifier. %s' % (issuer, auth_result.message))
|
|
811
743
|
|
|
812
|
-
|
|
813
|
-
|
|
744
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
745
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='create_did_sample', kwargs=kwargs, session=session)
|
|
746
|
+
if not auth_result.allowed:
|
|
747
|
+
raise AccessDenied('Account %s can not bulk add data identifier. %s' % (issuer, auth_result.message))
|
|
748
|
+
|
|
749
|
+
input_internal_scope = InternalScope(input_scope, vo=vo)
|
|
750
|
+
output_internal_scope = InternalScope(output_scope, vo=vo)
|
|
814
751
|
|
|
815
|
-
|
|
752
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
816
753
|
|
|
817
|
-
|
|
818
|
-
|
|
754
|
+
return did.create_did_sample(input_scope=input_internal_scope, input_name=input_name, output_scope=output_internal_scope, output_name=output_name,
|
|
755
|
+
account=issuer_account, nbfiles=nbfiles, session=session)
|
|
819
756
|
|
|
820
757
|
|
|
821
|
-
@transactional_session
|
|
822
758
|
def resurrect(
|
|
823
759
|
dids: 'Iterable[dict[str, Any]]',
|
|
824
760
|
issuer: str,
|
|
825
761
|
vo: str = 'def',
|
|
826
|
-
*,
|
|
827
|
-
session: "Session"
|
|
828
762
|
) -> None:
|
|
829
763
|
"""
|
|
830
764
|
Resurrect DIDs.
|
|
@@ -832,27 +766,24 @@ def resurrect(
|
|
|
832
766
|
:param dids: A list of dids.
|
|
833
767
|
:param issuer: The issuer account.
|
|
834
768
|
:param vo: The VO to act on.
|
|
835
|
-
:param session: The database session in use.
|
|
836
769
|
"""
|
|
837
770
|
kwargs = {'issuer': issuer}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
771
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
772
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='resurrect', kwargs=kwargs, session=session)
|
|
773
|
+
if not auth_result.allowed:
|
|
774
|
+
raise AccessDenied('Account %s can not resurrect data identifiers. %s' % (issuer, auth_result.message))
|
|
775
|
+
validate_schema(name='dids', obj=dids, vo=vo)
|
|
842
776
|
|
|
843
|
-
|
|
844
|
-
|
|
777
|
+
for d in dids:
|
|
778
|
+
d['scope'] = InternalScope(d['scope'], vo=vo)
|
|
845
779
|
|
|
846
|
-
|
|
780
|
+
return did.resurrect(dids=dids, session=session)
|
|
847
781
|
|
|
848
782
|
|
|
849
|
-
@stream_session
|
|
850
783
|
def list_archive_content(
|
|
851
784
|
scope: str,
|
|
852
785
|
name: str,
|
|
853
786
|
vo: str = 'def',
|
|
854
|
-
*,
|
|
855
|
-
session: "Session"
|
|
856
787
|
) -> 'Iterator[dict[str, Any]]':
|
|
857
788
|
"""
|
|
858
789
|
List archive contents.
|
|
@@ -860,23 +791,20 @@ def list_archive_content(
|
|
|
860
791
|
:param scope: The archive scope name.
|
|
861
792
|
:param name: The archive data identifier name.
|
|
862
793
|
:param vo: The VO to act on.
|
|
863
|
-
:param session: The database session in use.
|
|
864
794
|
"""
|
|
865
795
|
|
|
866
796
|
internal_scope = InternalScope(scope, vo=vo)
|
|
867
797
|
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
798
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
799
|
+
dids = did.list_archive_content(scope=internal_scope, name=name, session=session)
|
|
800
|
+
for d in dids:
|
|
801
|
+
yield gateway_update_return_dict(d, session=session)
|
|
871
802
|
|
|
872
803
|
|
|
873
|
-
@transactional_session
|
|
874
804
|
def add_did_to_followed(
|
|
875
805
|
scope: str,
|
|
876
806
|
name: str,
|
|
877
807
|
account: str,
|
|
878
|
-
*,
|
|
879
|
-
session: "Session",
|
|
880
808
|
vo: str = 'def'
|
|
881
809
|
) -> None:
|
|
882
810
|
"""
|
|
@@ -885,19 +813,16 @@ def add_did_to_followed(
|
|
|
885
813
|
:param scope: The scope name.
|
|
886
814
|
:param name: The data identifier name.
|
|
887
815
|
:param account: The account owner.
|
|
888
|
-
:param session: The database session in use.
|
|
889
816
|
"""
|
|
890
817
|
internal_scope = InternalScope(scope, vo=vo)
|
|
891
818
|
internal_account = InternalAccount(account, vo=vo)
|
|
892
|
-
|
|
819
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
820
|
+
return did.add_did_to_followed(scope=internal_scope, name=name, account=internal_account, session=session)
|
|
893
821
|
|
|
894
822
|
|
|
895
|
-
@transactional_session
|
|
896
823
|
def add_dids_to_followed(
|
|
897
824
|
dids: 'Iterable[Mapping[str, Any]]',
|
|
898
825
|
account: str,
|
|
899
|
-
*,
|
|
900
|
-
session: "Session",
|
|
901
826
|
vo: str = 'def'
|
|
902
827
|
) -> None:
|
|
903
828
|
"""
|
|
@@ -905,18 +830,15 @@ def add_dids_to_followed(
|
|
|
905
830
|
|
|
906
831
|
:param dids: A list of dids.
|
|
907
832
|
:param account: The account owner.
|
|
908
|
-
:param session: The database session in use.
|
|
909
833
|
"""
|
|
910
834
|
internal_account = InternalAccount(account, vo=vo)
|
|
911
|
-
|
|
835
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
836
|
+
return did.add_dids_to_followed(dids=dids, account=internal_account, session=session)
|
|
912
837
|
|
|
913
838
|
|
|
914
|
-
@stream_session
|
|
915
839
|
def get_users_following_did(
|
|
916
840
|
name: str,
|
|
917
841
|
scope: str,
|
|
918
|
-
*,
|
|
919
|
-
session: "Session",
|
|
920
842
|
vo: str = 'def'
|
|
921
843
|
) -> 'Iterator[dict[str, str]]':
|
|
922
844
|
"""
|
|
@@ -924,23 +846,20 @@ def get_users_following_did(
|
|
|
924
846
|
|
|
925
847
|
:param scope: The scope name.
|
|
926
848
|
:param name: The data identifier name.
|
|
927
|
-
:param session: The database session in use.
|
|
928
849
|
"""
|
|
929
850
|
internal_scope = InternalScope(scope, vo=vo)
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
user
|
|
933
|
-
|
|
851
|
+
with db_session(DatabaseOperationType.READ) as session:
|
|
852
|
+
users = did.get_users_following_did(name=name, scope=internal_scope, session=session)
|
|
853
|
+
for user in users:
|
|
854
|
+
user['user'] = user['user'].external
|
|
855
|
+
yield user
|
|
934
856
|
|
|
935
857
|
|
|
936
|
-
@transactional_session
|
|
937
858
|
def remove_did_from_followed(
|
|
938
859
|
scope: str,
|
|
939
860
|
name: str,
|
|
940
861
|
account: str,
|
|
941
862
|
issuer: str,
|
|
942
|
-
*,
|
|
943
|
-
session: "Session",
|
|
944
863
|
vo: str = 'def'
|
|
945
864
|
) -> None:
|
|
946
865
|
"""
|
|
@@ -949,26 +868,23 @@ def remove_did_from_followed(
|
|
|
949
868
|
:param scope: The scope name.
|
|
950
869
|
:param name: The data identifier name.
|
|
951
870
|
:param account: The account owner.
|
|
952
|
-
:param session: The database session in use.
|
|
953
871
|
:param issuer: The issuer account
|
|
954
872
|
"""
|
|
955
873
|
kwargs = {'scope': scope, 'issuer': issuer}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
874
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
875
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='remove_did_from_followed', kwargs=kwargs, session=session)
|
|
876
|
+
if not auth_result.allowed:
|
|
877
|
+
raise AccessDenied('Account %s can not remove data identifiers from followed table. %s' % (issuer, auth_result.message))
|
|
959
878
|
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
879
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
880
|
+
internal_account = InternalAccount(account, vo=vo)
|
|
881
|
+
return did.remove_did_from_followed(scope=internal_scope, name=name, account=internal_account, session=session)
|
|
963
882
|
|
|
964
883
|
|
|
965
|
-
@transactional_session
|
|
966
884
|
def remove_dids_from_followed(
|
|
967
885
|
dids: 'Iterable[Mapping[str, Any]]',
|
|
968
886
|
account: str,
|
|
969
887
|
issuer: str,
|
|
970
|
-
*,
|
|
971
|
-
session: "Session",
|
|
972
888
|
vo: str = 'def'
|
|
973
889
|
) -> None:
|
|
974
890
|
"""
|
|
@@ -976,12 +892,12 @@ def remove_dids_from_followed(
|
|
|
976
892
|
|
|
977
893
|
:param dids: A list of dids.
|
|
978
894
|
:param account: The account owner.
|
|
979
|
-
:param session: The database session in use.
|
|
980
895
|
"""
|
|
981
896
|
kwargs = {'dids': dids, 'issuer': issuer}
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
897
|
+
with db_session(DatabaseOperationType.WRITE) as session:
|
|
898
|
+
auth_result = rucio.gateway.permission.has_permission(issuer=issuer, vo=vo, action='remove_dids_from_followed', kwargs=kwargs, session=session)
|
|
899
|
+
if not auth_result.allowed:
|
|
900
|
+
raise AccessDenied('Account %s can not bulk remove data identifiers from followed table. %s' % (issuer, auth_result.message))
|
|
985
901
|
|
|
986
|
-
|
|
987
|
-
|
|
902
|
+
internal_account = InternalAccount(account, vo=vo)
|
|
903
|
+
return did.remove_dids_from_followed(dids=dids, account=internal_account, session=session)
|