rucio-clients 37.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/__init__.py +17 -0
- rucio/alembicrevision.py +15 -0
- rucio/cli/__init__.py +14 -0
- rucio/cli/account.py +216 -0
- rucio/cli/bin_legacy/__init__.py +13 -0
- rucio/cli/bin_legacy/rucio.py +2825 -0
- rucio/cli/bin_legacy/rucio_admin.py +2500 -0
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/__init__.py +15 -0
- rucio/client/accountclient.py +432 -0
- rucio/client/accountlimitclient.py +183 -0
- rucio/client/baseclient.py +983 -0
- rucio/client/client.py +120 -0
- rucio/client/configclient.py +126 -0
- rucio/client/credentialclient.py +59 -0
- rucio/client/didclient.py +868 -0
- rucio/client/diracclient.py +56 -0
- rucio/client/downloadclient.py +1783 -0
- rucio/client/exportclient.py +44 -0
- rucio/client/fileclient.py +50 -0
- rucio/client/importclient.py +42 -0
- rucio/client/lifetimeclient.py +90 -0
- rucio/client/lockclient.py +109 -0
- rucio/client/metaconventionsclient.py +140 -0
- rucio/client/pingclient.py +44 -0
- rucio/client/replicaclient.py +452 -0
- rucio/client/requestclient.py +125 -0
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +746 -0
- rucio/client/ruleclient.py +294 -0
- rucio/client/scopeclient.py +90 -0
- rucio/client/subscriptionclient.py +173 -0
- rucio/client/touchclient.py +82 -0
- rucio/client/uploadclient.py +969 -0
- rucio/common/__init__.py +13 -0
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +111 -0
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +788 -0
- rucio/common/constants.py +217 -0
- rucio/common/constraints.py +17 -0
- rucio/common/didtype.py +237 -0
- rucio/common/exception.py +1208 -0
- rucio/common/extra.py +31 -0
- rucio/common/logging.py +420 -0
- rucio/common/pcache.py +1409 -0
- rucio/common/plugins.py +185 -0
- rucio/common/policy.py +93 -0
- rucio/common/schema/__init__.py +200 -0
- rucio/common/schema/generic.py +416 -0
- rucio/common/schema/generic_multi_vo.py +395 -0
- rucio/common/stomp_utils.py +423 -0
- rucio/common/stopwatch.py +55 -0
- rucio/common/test_rucio_server.py +154 -0
- rucio/common/types.py +483 -0
- rucio/common/utils.py +1688 -0
- rucio/rse/__init__.py +96 -0
- rucio/rse/protocols/__init__.py +13 -0
- rucio/rse/protocols/bittorrent.py +194 -0
- rucio/rse/protocols/cache.py +111 -0
- rucio/rse/protocols/dummy.py +100 -0
- rucio/rse/protocols/gfal.py +708 -0
- rucio/rse/protocols/globus.py +243 -0
- rucio/rse/protocols/http_cache.py +82 -0
- rucio/rse/protocols/mock.py +123 -0
- rucio/rse/protocols/ngarc.py +209 -0
- rucio/rse/protocols/posix.py +250 -0
- rucio/rse/protocols/protocol.py +361 -0
- rucio/rse/protocols/rclone.py +365 -0
- rucio/rse/protocols/rfio.py +145 -0
- rucio/rse/protocols/srm.py +338 -0
- rucio/rse/protocols/ssh.py +414 -0
- rucio/rse/protocols/storm.py +195 -0
- rucio/rse/protocols/webdav.py +594 -0
- rucio/rse/protocols/xrootd.py +302 -0
- rucio/rse/rsemanager.py +881 -0
- rucio/rse/translation.py +260 -0
- rucio/vcsversion.py +11 -0
- rucio/version.py +45 -0
- rucio_clients-37.0.0rc1.data/data/etc/rse-accounts.cfg.template +25 -0
- rucio_clients-37.0.0rc1.data/data/etc/rucio.cfg.atlas.client.template +43 -0
- rucio_clients-37.0.0rc1.data/data/etc/rucio.cfg.template +241 -0
- rucio_clients-37.0.0rc1.data/data/requirements.client.txt +19 -0
- rucio_clients-37.0.0rc1.data/data/rucio_client/merge_rucio_configs.py +144 -0
- rucio_clients-37.0.0rc1.data/scripts/rucio +133 -0
- rucio_clients-37.0.0rc1.data/scripts/rucio-admin +97 -0
- rucio_clients-37.0.0rc1.dist-info/METADATA +54 -0
- rucio_clients-37.0.0rc1.dist-info/RECORD +104 -0
- rucio_clients-37.0.0rc1.dist-info/WHEEL +5 -0
- rucio_clients-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
- rucio_clients-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
- rucio_clients-37.0.0rc1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from json import dumps, loads
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union
|
|
17
|
+
from urllib.parse import quote_plus
|
|
18
|
+
|
|
19
|
+
from requests.status_codes import codes
|
|
20
|
+
|
|
21
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
22
|
+
from rucio.common.utils import build_url
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class RuleClient(BaseClient):
|
|
29
|
+
|
|
30
|
+
"""RuleClient class for working with replication rules"""
|
|
31
|
+
|
|
32
|
+
RULE_BASEURL = 'rules'
|
|
33
|
+
|
|
34
|
+
def add_replication_rule(
|
|
35
|
+
self,
|
|
36
|
+
dids: "Sequence[dict[str, str]]",
|
|
37
|
+
copies: int,
|
|
38
|
+
rse_expression: str,
|
|
39
|
+
priority: int = 3,
|
|
40
|
+
lifetime: Optional[int] = None,
|
|
41
|
+
grouping: str = 'DATASET',
|
|
42
|
+
notify: str = 'N',
|
|
43
|
+
source_replica_expression: Optional[str] = None,
|
|
44
|
+
activity: Optional[str] = None,
|
|
45
|
+
account: Optional[str] = None,
|
|
46
|
+
meta: Optional[str] = None,
|
|
47
|
+
ignore_availability: bool = False,
|
|
48
|
+
purge_replicas: bool = False,
|
|
49
|
+
ask_approval: bool = False,
|
|
50
|
+
asynchronous: bool = False,
|
|
51
|
+
locked: bool = False,
|
|
52
|
+
delay_injection: Optional[int] = None,
|
|
53
|
+
comment: Optional[str] = None,
|
|
54
|
+
weight: Optional[int] = None,
|
|
55
|
+
) -> Any:
|
|
56
|
+
"""
|
|
57
|
+
:param dids: The data identifier set.
|
|
58
|
+
:param copies: The number of replicas.
|
|
59
|
+
:param rse_expression: Boolean string expression to give the list of RSEs.
|
|
60
|
+
:param priority: Priority of the transfers.
|
|
61
|
+
:param lifetime: The lifetime of the replication rules (in seconds).
|
|
62
|
+
:param grouping: ALL - All files will be replicated to the same RSE.
|
|
63
|
+
DATASET - All files in the same dataset will be replicated to the same RSE.
|
|
64
|
+
NONE - Files will be completely spread over all allowed RSEs without any grouping considerations at all.
|
|
65
|
+
:param notify: Notification setting for the rule (Y, N, C).
|
|
66
|
+
:param source_replica_expression: RSE Expression for RSEs to be considered for source replicas.
|
|
67
|
+
:param activity: Transfer Activity to be passed to FTS.
|
|
68
|
+
:param account: The account owning the rule.
|
|
69
|
+
:param meta: Metadata, as dictionary.
|
|
70
|
+
:param ignore_availability: Option to ignore the availability of RSEs.
|
|
71
|
+
:param purge_replicas: When the rule gets deleted purge the associated replicas immediately.
|
|
72
|
+
:param ask_approval: Ask for approval of this replication rule.
|
|
73
|
+
:param asynchronous: Create rule asynchronously by judge-injector.
|
|
74
|
+
:param locked: If the rule is locked, it cannot be deleted.
|
|
75
|
+
:param delay_injection:
|
|
76
|
+
:param comment: Comment about the rule.
|
|
77
|
+
:param weight: If the weighting option of the replication rule is used, the choice of RSEs takes their weight into account.
|
|
78
|
+
"""
|
|
79
|
+
path = self.RULE_BASEURL + '/'
|
|
80
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
81
|
+
# TODO remove the subscription_id from the client; It will only be used by the core;
|
|
82
|
+
data = dumps({'dids': dids, 'copies': copies, 'rse_expression': rse_expression,
|
|
83
|
+
'weight': weight, 'lifetime': lifetime, 'grouping': grouping,
|
|
84
|
+
'account': account, 'locked': locked, 'source_replica_expression': source_replica_expression,
|
|
85
|
+
'activity': activity, 'notify': notify, 'purge_replicas': purge_replicas,
|
|
86
|
+
'ignore_availability': ignore_availability, 'comment': comment, 'ask_approval': ask_approval,
|
|
87
|
+
'asynchronous': asynchronous, 'delay_injection': delay_injection, 'priority': priority, 'meta': meta})
|
|
88
|
+
r = self._send_request(url, type_='POST', data=data)
|
|
89
|
+
if r.status_code == codes.created:
|
|
90
|
+
return loads(r.text)
|
|
91
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
92
|
+
raise exc_cls(exc_msg)
|
|
93
|
+
|
|
94
|
+
def delete_replication_rule(
|
|
95
|
+
self, rule_id: str, purge_replicas: Optional[bool] = None
|
|
96
|
+
) -> Literal[True]:
|
|
97
|
+
"""
|
|
98
|
+
Deletes a replication rule and all associated locks.
|
|
99
|
+
|
|
100
|
+
:param rule_id: The id of the rule to be deleted
|
|
101
|
+
:param purge_replicas: Immediately delete the replicas.
|
|
102
|
+
:raises: RuleNotFound, AccessDenied
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
path = self.RULE_BASEURL + '/' + rule_id
|
|
106
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
107
|
+
|
|
108
|
+
data = dumps({'purge_replicas': purge_replicas})
|
|
109
|
+
|
|
110
|
+
r = self._send_request(url, type_='DEL', data=data)
|
|
111
|
+
|
|
112
|
+
if r.status_code == codes.ok:
|
|
113
|
+
return True
|
|
114
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
115
|
+
raise exc_cls(exc_msg)
|
|
116
|
+
|
|
117
|
+
def get_replication_rule(self, rule_id: str) -> Any:
|
|
118
|
+
"""
|
|
119
|
+
Get a replication rule.
|
|
120
|
+
|
|
121
|
+
:param rule_id: The id of the rule to be retrieved.
|
|
122
|
+
:raises: RuleNotFound
|
|
123
|
+
"""
|
|
124
|
+
path = self.RULE_BASEURL + '/' + rule_id
|
|
125
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
126
|
+
r = self._send_request(url, type_='GET')
|
|
127
|
+
if r.status_code == codes.ok:
|
|
128
|
+
return next(self._load_json_data(r))
|
|
129
|
+
else:
|
|
130
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
131
|
+
raise exc_cls(exc_msg)
|
|
132
|
+
|
|
133
|
+
def update_replication_rule(self, rule_id: str, options: dict[str, Any]) -> Literal[True]:
|
|
134
|
+
"""
|
|
135
|
+
:param rule_id: The id of the rule to be retrieved.
|
|
136
|
+
:param options: Options dictionary.
|
|
137
|
+
:raises: RuleNotFound
|
|
138
|
+
"""
|
|
139
|
+
path = self.RULE_BASEURL + '/' + rule_id
|
|
140
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
141
|
+
data = dumps({'options': options})
|
|
142
|
+
r = self._send_request(url, type_='PUT', data=data)
|
|
143
|
+
if r.status_code == codes.ok:
|
|
144
|
+
return True
|
|
145
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
146
|
+
raise exc_cls(exc_msg)
|
|
147
|
+
|
|
148
|
+
def reduce_replication_rule(
|
|
149
|
+
self,
|
|
150
|
+
rule_id: str,
|
|
151
|
+
copies: int,
|
|
152
|
+
exclude_expression: Optional[str] = None
|
|
153
|
+
) -> Any:
|
|
154
|
+
"""
|
|
155
|
+
:param rule_id: Rule to be reduced.
|
|
156
|
+
:param copies: Number of copies of the new rule.
|
|
157
|
+
:param exclude_expression: RSE Expression of RSEs to exclude.
|
|
158
|
+
:raises: RuleReplaceFailed, RuleNotFound
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
path = self.RULE_BASEURL + '/' + rule_id + '/reduce'
|
|
162
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
163
|
+
data = dumps({'copies': copies, 'exclude_expression': exclude_expression})
|
|
164
|
+
r = self._send_request(url, type_='POST', data=data)
|
|
165
|
+
if r.status_code == codes.ok:
|
|
166
|
+
return loads(r.text)
|
|
167
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
168
|
+
raise exc_cls(exc_msg)
|
|
169
|
+
|
|
170
|
+
def move_replication_rule(
|
|
171
|
+
self,
|
|
172
|
+
rule_id: str,
|
|
173
|
+
rse_expression: str,
|
|
174
|
+
override: "Mapping[str, Any]"
|
|
175
|
+
) -> Any:
|
|
176
|
+
"""
|
|
177
|
+
Move a replication rule to another RSE and, once done, delete the original one.
|
|
178
|
+
|
|
179
|
+
:param rule_id: Rule to be moved.
|
|
180
|
+
:param rse_expression: RSE expression of the new rule.
|
|
181
|
+
:param override: Configurations to update for the new rule.
|
|
182
|
+
:raises: RuleNotFound, RuleReplaceFailed
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
path = self.RULE_BASEURL + '/' + rule_id + '/move'
|
|
186
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
187
|
+
data = dumps({
|
|
188
|
+
'rule_id': rule_id,
|
|
189
|
+
'rse_expression': rse_expression,
|
|
190
|
+
'override': override,
|
|
191
|
+
})
|
|
192
|
+
r = self._send_request(url, type_='POST', data=data)
|
|
193
|
+
if r.status_code == codes.created:
|
|
194
|
+
return loads(r.text)
|
|
195
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
196
|
+
raise exc_cls(exc_msg)
|
|
197
|
+
|
|
198
|
+
def approve_replication_rule(self, rule_id: str) -> Literal[True]:
|
|
199
|
+
"""
|
|
200
|
+
:param rule_id: Rule to be approved.
|
|
201
|
+
:raises: RuleNotFound
|
|
202
|
+
"""
|
|
203
|
+
|
|
204
|
+
path = self.RULE_BASEURL + '/' + rule_id
|
|
205
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
206
|
+
data = dumps({'options': {'approve': True}})
|
|
207
|
+
r = self._send_request(url, type_='PUT', data=data)
|
|
208
|
+
if r.status_code == codes.ok:
|
|
209
|
+
return True
|
|
210
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
211
|
+
raise exc_cls(exc_msg)
|
|
212
|
+
|
|
213
|
+
def deny_replication_rule(self, rule_id: str, reason: Optional[str] = None) -> Literal[True]:
|
|
214
|
+
"""
|
|
215
|
+
:param rule_id: Rule to be denied.
|
|
216
|
+
:param reason: Reason for denying the rule.
|
|
217
|
+
:raises: RuleNotFound
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
path = self.RULE_BASEURL + '/' + rule_id
|
|
221
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
222
|
+
options: dict[str, Union[bool, str]] = {'approve': False}
|
|
223
|
+
if reason:
|
|
224
|
+
options['comment'] = reason
|
|
225
|
+
data = dumps({'options': options})
|
|
226
|
+
r = self._send_request(url, type_='PUT', data=data)
|
|
227
|
+
if r.status_code == codes.ok:
|
|
228
|
+
return True
|
|
229
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
230
|
+
raise exc_cls(exc_msg)
|
|
231
|
+
|
|
232
|
+
def list_replication_rule_full_history(
|
|
233
|
+
self, scope: Union[str, bytes], name: Union[str, bytes]
|
|
234
|
+
) -> "Iterator[dict[str, Any]]":
|
|
235
|
+
"""
|
|
236
|
+
List the rule history of a DID.
|
|
237
|
+
|
|
238
|
+
:param scope: The scope of the DID.
|
|
239
|
+
:param name: The name of the DID.
|
|
240
|
+
"""
|
|
241
|
+
path = '/'.join([self.RULE_BASEURL, quote_plus(scope), quote_plus(name), 'history'])
|
|
242
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
243
|
+
r = self._send_request(url, type_='GET')
|
|
244
|
+
if r.status_code == codes.ok:
|
|
245
|
+
return self._load_json_data(r)
|
|
246
|
+
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
|
|
247
|
+
raise exc_cls(exc_msg)
|
|
248
|
+
|
|
249
|
+
def examine_replication_rule(self, rule_id: str) -> Any:
|
|
250
|
+
"""
|
|
251
|
+
Examine a replication rule for errors during transfer.
|
|
252
|
+
|
|
253
|
+
:param rule_id: Rule to be denied.
|
|
254
|
+
:raises: RuleNotFound
|
|
255
|
+
"""
|
|
256
|
+
path = self.RULE_BASEURL + '/' + rule_id + '/analysis'
|
|
257
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
258
|
+
r = self._send_request(url, type_='GET')
|
|
259
|
+
if r.status_code == codes.ok:
|
|
260
|
+
return next(self._load_json_data(r))
|
|
261
|
+
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
|
|
262
|
+
raise exc_cls(exc_msg)
|
|
263
|
+
|
|
264
|
+
def list_replica_locks(self, rule_id: str) -> Any:
|
|
265
|
+
"""
|
|
266
|
+
List details of all replica locks for a rule.
|
|
267
|
+
|
|
268
|
+
:param rule_id: Rule to be denied.
|
|
269
|
+
:raises: RuleNotFound
|
|
270
|
+
"""
|
|
271
|
+
path = self.RULE_BASEURL + '/' + rule_id + '/locks'
|
|
272
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
273
|
+
r = self._send_request(url, type_='GET')
|
|
274
|
+
if r.status_code == codes.ok:
|
|
275
|
+
return self._load_json_data(r)
|
|
276
|
+
exc_cls, exc_msg = self._get_exception(r.headers, r.status_code)
|
|
277
|
+
raise exc_cls(exc_msg)
|
|
278
|
+
|
|
279
|
+
def list_replication_rules(self, filters: Optional[dict[str, Any]] = None) -> "Iterator[dict[str, Any]]":
|
|
280
|
+
"""
|
|
281
|
+
List all replication rules which match a filter
|
|
282
|
+
:param filters: dictionary of attributes by which the rules should be filtered
|
|
283
|
+
|
|
284
|
+
:returns: True if successful, otherwise false.
|
|
285
|
+
"""
|
|
286
|
+
filters = filters or {}
|
|
287
|
+
path = self.RULE_BASEURL + '/'
|
|
288
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
289
|
+
r = self._send_request(url, type_='GET', params=filters)
|
|
290
|
+
if r.status_code == codes.ok:
|
|
291
|
+
return self._load_json_data(r)
|
|
292
|
+
else:
|
|
293
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
294
|
+
raise exc_cls(exc_msg)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from json import loads
|
|
16
|
+
from urllib.parse import quote_plus
|
|
17
|
+
|
|
18
|
+
from requests.status_codes import codes
|
|
19
|
+
|
|
20
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
21
|
+
from rucio.common.utils import build_url
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ScopeClient(BaseClient):
|
|
25
|
+
|
|
26
|
+
"""Scope client class for working with rucio scopes"""
|
|
27
|
+
|
|
28
|
+
SCOPE_BASEURL = 'accounts'
|
|
29
|
+
|
|
30
|
+
def add_scope(
|
|
31
|
+
self,
|
|
32
|
+
account: str,
|
|
33
|
+
scope: str
|
|
34
|
+
) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Sends the request to add a new scope.
|
|
37
|
+
|
|
38
|
+
:param account: the name of the account to add the scope to.
|
|
39
|
+
:param scope: the name of the new scope.
|
|
40
|
+
:return: True if scope was created successfully.
|
|
41
|
+
:raises Duplicate: if scope already exists.
|
|
42
|
+
:raises AccountNotFound: if account doesn't exist.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
path = '/'.join([self.SCOPE_BASEURL, account, 'scopes', quote_plus(scope)])
|
|
46
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
47
|
+
r = self._send_request(url, type_='POST')
|
|
48
|
+
if r.status_code == codes.created:
|
|
49
|
+
return True
|
|
50
|
+
else:
|
|
51
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
52
|
+
raise exc_cls(exc_msg)
|
|
53
|
+
|
|
54
|
+
def list_scopes(self) -> list[str]:
|
|
55
|
+
"""
|
|
56
|
+
Sends the request to list all scopes.
|
|
57
|
+
|
|
58
|
+
:return: a list containing the names of all scopes.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
path = '/'.join(['scopes/'])
|
|
62
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
63
|
+
r = self._send_request(url)
|
|
64
|
+
if r.status_code == codes.ok:
|
|
65
|
+
scopes = loads(r.text)
|
|
66
|
+
return scopes
|
|
67
|
+
else:
|
|
68
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
69
|
+
raise exc_cls(exc_msg)
|
|
70
|
+
|
|
71
|
+
def list_scopes_for_account(self, account: str) -> list[str]:
|
|
72
|
+
"""
|
|
73
|
+
Sends the request to list all scopes for a rucio account.
|
|
74
|
+
|
|
75
|
+
:param account: the rucio account to list scopes for.
|
|
76
|
+
:return: a list containing the names of all scopes for a rucio account.
|
|
77
|
+
:raises AccountNotFound: if account doesn't exist.
|
|
78
|
+
:raises ScopeNotFound: if no scopes exist for account.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/'])
|
|
82
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
83
|
+
|
|
84
|
+
r = self._send_request(url)
|
|
85
|
+
if r.status_code == codes.ok:
|
|
86
|
+
scopes = loads(r.text)
|
|
87
|
+
return scopes
|
|
88
|
+
else:
|
|
89
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
90
|
+
raise exc_cls(exc_msg)
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from json import dumps
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union
|
|
17
|
+
|
|
18
|
+
from requests.status_codes import codes
|
|
19
|
+
|
|
20
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
21
|
+
from rucio.common.utils import build_url
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from collections.abc import Iterator
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SubscriptionClient(BaseClient):
|
|
28
|
+
|
|
29
|
+
"""SubscriptionClient class for working with subscriptions"""
|
|
30
|
+
|
|
31
|
+
SUB_BASEURL = 'subscriptions'
|
|
32
|
+
|
|
33
|
+
def add_subscription(
|
|
34
|
+
self,
|
|
35
|
+
name: str,
|
|
36
|
+
account: str,
|
|
37
|
+
filter_: dict[str, Any],
|
|
38
|
+
replication_rules: dict[str, Any],
|
|
39
|
+
comments: str,
|
|
40
|
+
lifetime: Union[int, Literal[False]],
|
|
41
|
+
retroactive: bool,
|
|
42
|
+
dry_run: bool,
|
|
43
|
+
priority: int = 3
|
|
44
|
+
) -> str:
|
|
45
|
+
"""
|
|
46
|
+
Adds a new subscription which will be verified against every new added file and dataset
|
|
47
|
+
|
|
48
|
+
:param name: Name of the subscription
|
|
49
|
+
:param account: Account identifier
|
|
50
|
+
:param filter_: Dictionary of attributes by which the input data should be filtered
|
|
51
|
+
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
|
|
52
|
+
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
|
|
53
|
+
:param comments: Comments for the subscription
|
|
54
|
+
:param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
|
|
55
|
+
:param retroactive: Flag to know if the subscription should be applied on previous data
|
|
56
|
+
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
|
|
57
|
+
:param priority: The priority of the subscription (3 by default)
|
|
58
|
+
"""
|
|
59
|
+
path = self.SUB_BASEURL + '/' + account + '/' + name
|
|
60
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
61
|
+
if retroactive:
|
|
62
|
+
raise NotImplementedError('Retroactive mode is not implemented')
|
|
63
|
+
if filter_ and not isinstance(filter_, dict):
|
|
64
|
+
raise TypeError('filter should be a dict')
|
|
65
|
+
if replication_rules and not isinstance(replication_rules, list):
|
|
66
|
+
raise TypeError('replication_rules should be a list')
|
|
67
|
+
data = dumps({'options': {'filter': filter_, 'replication_rules': replication_rules, 'comments': comments,
|
|
68
|
+
'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
|
|
69
|
+
result = self._send_request(url, type_='POST', data=data)
|
|
70
|
+
if result.status_code == codes.created: # pylint: disable=no-member
|
|
71
|
+
return result.text
|
|
72
|
+
else:
|
|
73
|
+
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
74
|
+
raise exc_cls(exc_msg)
|
|
75
|
+
|
|
76
|
+
def list_subscriptions(
|
|
77
|
+
self,
|
|
78
|
+
name: Optional[str] = None,
|
|
79
|
+
account: Optional[dict[str, Any]] = None
|
|
80
|
+
) -> Union["Iterator[dict[str, Any]]", list]:
|
|
81
|
+
"""
|
|
82
|
+
Returns a dictionary with the subscription information :
|
|
83
|
+
Examples: ``{'status': 'INACTIVE/ACTIVE/BROKEN', 'last_modified_date': ...}``
|
|
84
|
+
|
|
85
|
+
:param name: Name of the subscription
|
|
86
|
+
:param account: Account identifier
|
|
87
|
+
:returns: Dictionary containing subscription parameter
|
|
88
|
+
:raises: exception.NotFound if subscription is not found
|
|
89
|
+
"""
|
|
90
|
+
path = self.SUB_BASEURL
|
|
91
|
+
if account:
|
|
92
|
+
path += '/%s' % (account)
|
|
93
|
+
if name:
|
|
94
|
+
path += '/%s' % (name)
|
|
95
|
+
elif name:
|
|
96
|
+
path += '/name/%s' % (name)
|
|
97
|
+
else:
|
|
98
|
+
path += '/'
|
|
99
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
100
|
+
result = self._send_request(url, type_='GET')
|
|
101
|
+
if result.status_code == codes.ok: # pylint: disable=no-member
|
|
102
|
+
return self._load_json_data(result)
|
|
103
|
+
if result.status_code == codes.not_found:
|
|
104
|
+
return []
|
|
105
|
+
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
106
|
+
raise exc_cls(exc_msg)
|
|
107
|
+
|
|
108
|
+
def update_subscription(
|
|
109
|
+
self,
|
|
110
|
+
name,
|
|
111
|
+
account: Optional[str] = None,
|
|
112
|
+
filter_: Optional[dict[str, Any]] = None,
|
|
113
|
+
replication_rules: Optional[dict[str, Any]] = None,
|
|
114
|
+
comments: Optional[str] = None,
|
|
115
|
+
lifetime: Optional[Union[int, Literal[False]]] = None,
|
|
116
|
+
retroactive: Optional[bool] = None,
|
|
117
|
+
dry_run: Optional[bool] = None,
|
|
118
|
+
priority: Optional[int] = None
|
|
119
|
+
) -> Literal[True]:
|
|
120
|
+
"""
|
|
121
|
+
Updates a subscription
|
|
122
|
+
|
|
123
|
+
:param name: Name of the subscription
|
|
124
|
+
:param account: Account identifier
|
|
125
|
+
:param filter_: Dictionary of attributes by which the input data should be filtered
|
|
126
|
+
**Example**: ``{'dsn': 'data11_hi*.express_express.*,data11_hi*physics_MinBiasOverlay*', 'account': 'tzero'}``
|
|
127
|
+
:param replication_rules: Replication rules to be set : Dictionary with keys copies, rse_expression, weight, rse_expression
|
|
128
|
+
:param comments: Comments for the subscription
|
|
129
|
+
:param lifetime: Subscription's lifetime (days); False if subscription has no lifetime
|
|
130
|
+
:param retroactive: Flag to know if the subscription should be applied on previous data
|
|
131
|
+
:param dry_run: Just print the subscriptions actions without actually executing them (Useful if retroactive flag is set)
|
|
132
|
+
:param priority: The priority of the subscription
|
|
133
|
+
:raises: exception.NotFound if subscription is not found
|
|
134
|
+
"""
|
|
135
|
+
if not account:
|
|
136
|
+
account = self.account
|
|
137
|
+
if retroactive:
|
|
138
|
+
raise NotImplementedError('Retroactive mode is not implemented')
|
|
139
|
+
path = self.SUB_BASEURL + '/' + account + '/' + name # type: ignore
|
|
140
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
141
|
+
if filter_ and not isinstance(filter_, dict):
|
|
142
|
+
raise TypeError('filter should be a dict')
|
|
143
|
+
if replication_rules and not isinstance(replication_rules, list):
|
|
144
|
+
raise TypeError('replication_rules should be a list')
|
|
145
|
+
data = dumps({'options': {'filter': filter_, 'replication_rules': replication_rules, 'comments': comments,
|
|
146
|
+
'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
|
|
147
|
+
result = self._send_request(url, type_='PUT', data=data)
|
|
148
|
+
if result.status_code == codes.created: # pylint: disable=no-member
|
|
149
|
+
return True
|
|
150
|
+
else:
|
|
151
|
+
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
152
|
+
raise exc_cls(exc_msg)
|
|
153
|
+
|
|
154
|
+
def list_subscription_rules(
|
|
155
|
+
self,
|
|
156
|
+
account: str,
|
|
157
|
+
name: str
|
|
158
|
+
) -> "Iterator[dict[str, Any]]":
|
|
159
|
+
"""
|
|
160
|
+
List the associated rules of a subscription.
|
|
161
|
+
|
|
162
|
+
:param account: Account of the subscription.
|
|
163
|
+
:param name: Name of the subscription.
|
|
164
|
+
"""
|
|
165
|
+
|
|
166
|
+
path = '/'.join([self.SUB_BASEURL, account, name, 'rules'])
|
|
167
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
168
|
+
result = self._send_request(url, type_='GET')
|
|
169
|
+
if result.status_code == codes.ok: # pylint: disable=no-member
|
|
170
|
+
return self._load_json_data(result)
|
|
171
|
+
else:
|
|
172
|
+
exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)
|
|
173
|
+
raise exc_cls(exc_msg)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from json import dumps
|
|
16
|
+
from typing import Optional
|
|
17
|
+
|
|
18
|
+
from requests import post
|
|
19
|
+
|
|
20
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
21
|
+
from rucio.common.exception import RucioException, UnsupportedDIDType
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TouchClient(BaseClient):
|
|
25
|
+
|
|
26
|
+
"""
|
|
27
|
+
Touch client class to send a trace that can be used to
|
|
28
|
+
update accessed_at for file or dataset DIDs
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
DIDS_BASEURL = 'dids'
|
|
32
|
+
TRACES_BASEURL = 'traces'
|
|
33
|
+
|
|
34
|
+
def touch(
|
|
35
|
+
self,
|
|
36
|
+
scope: str,
|
|
37
|
+
name: str,
|
|
38
|
+
rse: Optional[str] = None
|
|
39
|
+
) -> None:
|
|
40
|
+
"""
|
|
41
|
+
Sends a touch trace for a given file or dataset.
|
|
42
|
+
|
|
43
|
+
:param scope: the scope of the file/dataset to update.
|
|
44
|
+
:param name: the name of file/dataset to update.
|
|
45
|
+
:param rse: optional parameter if a specific replica should be touched.
|
|
46
|
+
:raises DataIdentifierNotFound: if given dids does not exist.
|
|
47
|
+
:raises RSENotFound: if rse is not None and given rse does not exist.
|
|
48
|
+
:raises UnsupportedDIDType: if type of the given DID is not FILE or DATASET.
|
|
49
|
+
:raises RucioException: if trace could not be sent successfully.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
trace = {}
|
|
53
|
+
|
|
54
|
+
trace['eventType'] = 'touch'
|
|
55
|
+
trace['clientState'] = 'DONE'
|
|
56
|
+
trace['account'] = self.account
|
|
57
|
+
if self.vo != 'def':
|
|
58
|
+
trace['vo'] = self.vo
|
|
59
|
+
|
|
60
|
+
if rse:
|
|
61
|
+
self.get_rse(rse) # pylint: disable=no-member
|
|
62
|
+
|
|
63
|
+
trace['localSite'] = trace['remoteSite'] = rse
|
|
64
|
+
|
|
65
|
+
info = self.get_did(scope, name) # pylint: disable=no-member
|
|
66
|
+
|
|
67
|
+
if info['type'] == 'CONTAINER':
|
|
68
|
+
raise UnsupportedDIDType("%s:%s is a container." % (scope, name))
|
|
69
|
+
|
|
70
|
+
if info['type'] == 'FILE':
|
|
71
|
+
trace['scope'] = scope
|
|
72
|
+
trace['filename'] = name
|
|
73
|
+
elif info['type'] == 'DATASET':
|
|
74
|
+
trace['datasetScope'] = scope
|
|
75
|
+
trace['dataset'] = name
|
|
76
|
+
|
|
77
|
+
url = '%s/%s/' % (choice(self.list_hosts), self.TRACES_BASEURL)
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
post(url, verify=False, data=dumps(trace))
|
|
81
|
+
except Exception as error:
|
|
82
|
+
raise RucioException("Could not send trace. " + str(error))
|