rucio-clients 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio_clients-35.8.0.data/data/requirements.client.txt +15 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/rucio_client/merge_rucio_configs.py +2 -5
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio +87 -85
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/METADATA +13 -13
- rucio_clients-35.8.0.dist-info/RECORD +88 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/WHEEL +1 -1
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio_clients-32.8.6.data/data/requirements.txt +0 -55
- rucio_clients-32.8.6.dist-info/RECORD +0 -88
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-32.8.6.data → rucio_clients-35.8.0.data}/data/etc/rucio.cfg.template +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-32.8.6.dist-info → rucio_clients-35.8.0.dist-info}/top_level.txt +0 -0
rucio/client/configclient.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -14,11 +13,11 @@
|
|
|
14
13
|
# limitations under the License.
|
|
15
14
|
|
|
16
15
|
from json import dumps
|
|
16
|
+
from typing import Any, Optional
|
|
17
17
|
|
|
18
18
|
from requests.status_codes import codes
|
|
19
19
|
|
|
20
|
-
from rucio.client.baseclient import BaseClient
|
|
21
|
-
from rucio.client.baseclient import choice
|
|
20
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
22
21
|
from rucio.common.utils import build_url
|
|
23
22
|
|
|
24
23
|
|
|
@@ -28,7 +27,11 @@ class ConfigClient(BaseClient):
|
|
|
28
27
|
|
|
29
28
|
CONFIG_BASEURL = 'config'
|
|
30
29
|
|
|
31
|
-
def get_config(
|
|
30
|
+
def get_config(
|
|
31
|
+
self,
|
|
32
|
+
section: Optional[str] = None,
|
|
33
|
+
option: Optional[str] = None
|
|
34
|
+
) -> dict[str, Any]:
|
|
32
35
|
"""
|
|
33
36
|
Sends the request to get the matching configuration.
|
|
34
37
|
|
|
@@ -55,7 +58,13 @@ class ConfigClient(BaseClient):
|
|
|
55
58
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
56
59
|
raise exc_cls(exc_msg)
|
|
57
60
|
|
|
58
|
-
def set_config_option(
|
|
61
|
+
def set_config_option(
|
|
62
|
+
self,
|
|
63
|
+
section: str,
|
|
64
|
+
option: str,
|
|
65
|
+
value: Any,
|
|
66
|
+
use_body_for_params: bool = True
|
|
67
|
+
) -> bool:
|
|
59
68
|
"""
|
|
60
69
|
Sends the request to create or set an option within a section. Missing sections will be created.
|
|
61
70
|
|
|
@@ -92,7 +101,11 @@ class ConfigClient(BaseClient):
|
|
|
92
101
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
93
102
|
raise exc_cls(exc_msg)
|
|
94
103
|
|
|
95
|
-
def delete_config_option(
|
|
104
|
+
def delete_config_option(
|
|
105
|
+
self,
|
|
106
|
+
section: str,
|
|
107
|
+
option: str
|
|
108
|
+
) -> bool:
|
|
96
109
|
"""
|
|
97
110
|
Sends the request to remove an option from a section
|
|
98
111
|
|
rucio/client/credentialclient.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -15,8 +14,7 @@
|
|
|
15
14
|
|
|
16
15
|
from requests.status_codes import codes
|
|
17
16
|
|
|
18
|
-
from rucio.client.baseclient import BaseClient
|
|
19
|
-
from rucio.client.baseclient import choice
|
|
17
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
20
18
|
from rucio.common.utils import build_url
|
|
21
19
|
|
|
22
20
|
|
|
@@ -25,7 +23,14 @@ class CredentialClient(BaseClient):
|
|
|
25
23
|
|
|
26
24
|
CREDENTIAL_BASEURL = 'credentials'
|
|
27
25
|
|
|
28
|
-
def get_signed_url(
|
|
26
|
+
def get_signed_url(
|
|
27
|
+
self,
|
|
28
|
+
rse: str,
|
|
29
|
+
service: str,
|
|
30
|
+
operation: str,
|
|
31
|
+
url: str,
|
|
32
|
+
lifetime: int = 3600
|
|
33
|
+
) -> str:
|
|
29
34
|
"""
|
|
30
35
|
Return a signed version of the given URL for the given operation.
|
|
31
36
|
|
rucio/client/didclient.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -15,14 +14,17 @@
|
|
|
15
14
|
|
|
16
15
|
from datetime import datetime
|
|
17
16
|
from json import dumps
|
|
17
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union
|
|
18
18
|
from urllib.parse import quote_plus
|
|
19
19
|
|
|
20
20
|
from requests.status_codes import codes
|
|
21
21
|
|
|
22
|
-
from rucio.client.baseclient import BaseClient
|
|
23
|
-
from rucio.client.baseclient import choice
|
|
22
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
24
23
|
from rucio.common.exception import DeprecationError
|
|
25
|
-
from rucio.common.utils import build_url,
|
|
24
|
+
from rucio.common.utils import build_url, date_to_str, render_json
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
class DIDClient(BaseClient):
|
|
@@ -32,7 +34,14 @@ class DIDClient(BaseClient):
|
|
|
32
34
|
DIDS_BASEURL = 'dids'
|
|
33
35
|
ARCHIVES_BASEURL = 'archives'
|
|
34
36
|
|
|
35
|
-
def list_dids(
|
|
37
|
+
def list_dids(
|
|
38
|
+
self,
|
|
39
|
+
scope: str,
|
|
40
|
+
filters: "Sequence[dict[str, Any]]",
|
|
41
|
+
did_type: Literal['all', 'collection', 'dataset', 'container', 'file'] = 'collection',
|
|
42
|
+
long: bool = False,
|
|
43
|
+
recursive: bool = False
|
|
44
|
+
) -> "Iterator[dict[str, Any]]":
|
|
36
45
|
"""
|
|
37
46
|
List all data identifiers in a scope which match a given pattern.
|
|
38
47
|
|
|
@@ -48,7 +57,7 @@ class DIDClient(BaseClient):
|
|
|
48
57
|
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), 'dids', 'search'])
|
|
49
58
|
|
|
50
59
|
# stringify dates.
|
|
51
|
-
if isinstance(filters, dict): # backwards
|
|
60
|
+
if isinstance(filters, dict): # backwards compatibility for filters as single {}
|
|
52
61
|
filters = [filters]
|
|
53
62
|
for or_group in filters:
|
|
54
63
|
for key, value in or_group.items():
|
|
@@ -79,14 +88,25 @@ class DIDClient(BaseClient):
|
|
|
79
88
|
"""
|
|
80
89
|
raise DeprecationError("Command or function has been deprecated. Please use list_dids instead.")
|
|
81
90
|
|
|
82
|
-
def add_did(
|
|
91
|
+
def add_did(
|
|
92
|
+
self,
|
|
93
|
+
scope: str,
|
|
94
|
+
name: str,
|
|
95
|
+
did_type: Literal['DATASET', 'CONTAINER'],
|
|
96
|
+
statuses: Optional["Mapping[str, Any]"] = None,
|
|
97
|
+
meta: Optional["Mapping[str, Any]"] = None,
|
|
98
|
+
rules: Optional["Sequence[Mapping[str, Any]]"] = None,
|
|
99
|
+
lifetime: Optional[int] = None,
|
|
100
|
+
dids: Optional["Sequence[Mapping[str, Any]]"] = None,
|
|
101
|
+
rse: Optional[str] = None
|
|
102
|
+
) -> bool:
|
|
83
103
|
"""
|
|
84
104
|
Add data identifier for a dataset or container.
|
|
85
105
|
|
|
86
106
|
:param scope: The scope name.
|
|
87
107
|
:param name: The data identifier name.
|
|
88
|
-
:param did_type: The data identifier type (
|
|
89
|
-
:param statuses: Dictionary with statuses, e.g.
|
|
108
|
+
:param did_type: The data identifier type (dataset|container).
|
|
109
|
+
:param statuses: Dictionary with statuses, e.g. {'monotonic':True}.
|
|
90
110
|
:param meta: Meta-data associated with the data identifier is represented using key/value pairs in a dictionary.
|
|
91
111
|
:param rules: Replication rules associated with the data identifier. A list of dictionaries, e.g., [{'copies': 2, 'rse_expression': 'TIERS1'}, ].
|
|
92
112
|
:param lifetime: DID's lifetime (in seconds).
|
|
@@ -96,7 +116,7 @@ class DIDClient(BaseClient):
|
|
|
96
116
|
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name)])
|
|
97
117
|
url = build_url(choice(self.list_hosts), path=path)
|
|
98
118
|
# Build json
|
|
99
|
-
data = {'type': did_type}
|
|
119
|
+
data: dict[str, Any] = {'type': did_type}
|
|
100
120
|
if statuses:
|
|
101
121
|
data['statuses'] = statuses
|
|
102
122
|
if meta:
|
|
@@ -116,20 +136,30 @@ class DIDClient(BaseClient):
|
|
|
116
136
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
117
137
|
raise exc_cls(exc_msg)
|
|
118
138
|
|
|
119
|
-
def add_dids(self, dids):
|
|
139
|
+
def add_dids(self, dids: "Sequence[Mapping[str, Any]]") -> bool:
|
|
120
140
|
"""
|
|
121
141
|
Bulk add datasets/containers.
|
|
122
142
|
"""
|
|
123
143
|
path = '/'.join([self.DIDS_BASEURL])
|
|
124
144
|
url = build_url(choice(self.list_hosts), path=path)
|
|
125
|
-
r = self._send_request(url, type_='POST', data=
|
|
145
|
+
r = self._send_request(url, type_='POST', data=render_json(dids))
|
|
126
146
|
if r.status_code == codes.created:
|
|
127
147
|
return True
|
|
128
148
|
else:
|
|
129
149
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
130
150
|
raise exc_cls(exc_msg)
|
|
131
151
|
|
|
132
|
-
def add_dataset(
|
|
152
|
+
def add_dataset(
|
|
153
|
+
self,
|
|
154
|
+
scope: str,
|
|
155
|
+
name: str,
|
|
156
|
+
statuses: Optional["Mapping[str, Any]"] = None,
|
|
157
|
+
meta: Optional["Mapping[str, Any]"] = None,
|
|
158
|
+
rules: Optional["Sequence[Mapping[str, Any]]"] = None,
|
|
159
|
+
lifetime: Optional[int] = None,
|
|
160
|
+
files: Optional["Sequence[Mapping[str, Any]]"] = None,
|
|
161
|
+
rse: Optional[str] = None
|
|
162
|
+
) -> bool:
|
|
133
163
|
"""
|
|
134
164
|
Add data identifier for a dataset.
|
|
135
165
|
|
|
@@ -146,7 +176,7 @@ class DIDClient(BaseClient):
|
|
|
146
176
|
statuses=statuses, meta=meta, rules=rules,
|
|
147
177
|
lifetime=lifetime, dids=files, rse=rse)
|
|
148
178
|
|
|
149
|
-
def add_datasets(self, dsns):
|
|
179
|
+
def add_datasets(self, dsns: "Iterable[dict[str, Any]]") -> bool:
|
|
150
180
|
"""
|
|
151
181
|
Bulk add datasets.
|
|
152
182
|
|
|
@@ -154,7 +184,15 @@ class DIDClient(BaseClient):
|
|
|
154
184
|
"""
|
|
155
185
|
return self.add_dids(dids=[dict(list(dsn.items()) + [('type', 'DATASET')]) for dsn in dsns])
|
|
156
186
|
|
|
157
|
-
def add_container(
|
|
187
|
+
def add_container(
|
|
188
|
+
self,
|
|
189
|
+
scope: str,
|
|
190
|
+
name: str,
|
|
191
|
+
statuses: Optional["Mapping[str, Any]"] = None,
|
|
192
|
+
meta: Optional["Mapping[str, Any]"] = None,
|
|
193
|
+
rules: Optional["Sequence[Mapping[str, Any]]"] = None,
|
|
194
|
+
lifetime: Optional[int] = None
|
|
195
|
+
) -> bool:
|
|
158
196
|
"""
|
|
159
197
|
Add data identifier for a container.
|
|
160
198
|
|
|
@@ -167,15 +205,21 @@ class DIDClient(BaseClient):
|
|
|
167
205
|
"""
|
|
168
206
|
return self.add_did(scope=scope, name=name, did_type='CONTAINER', statuses=statuses, meta=meta, rules=rules, lifetime=lifetime)
|
|
169
207
|
|
|
170
|
-
def add_containers(self, cnts):
|
|
208
|
+
def add_containers(self, cnts: "Sequence[dict[str, Any]]") -> bool:
|
|
171
209
|
"""
|
|
172
210
|
Bulk add containers.
|
|
173
211
|
|
|
174
212
|
:param cnts: A list of containers.
|
|
175
213
|
"""
|
|
176
|
-
return self.add_dids(dids=[dict(list(
|
|
214
|
+
return self.add_dids(dids=[dict(list(cnt.items()) + [('type', 'CONTAINER')]) for cnt in cnts])
|
|
177
215
|
|
|
178
|
-
def attach_dids(
|
|
216
|
+
def attach_dids(
|
|
217
|
+
self,
|
|
218
|
+
scope: str,
|
|
219
|
+
name: str,
|
|
220
|
+
dids: "Sequence[Mapping[str, Any]]",
|
|
221
|
+
rse: Optional[str] = None
|
|
222
|
+
) -> bool:
|
|
179
223
|
"""
|
|
180
224
|
Attach data identifier.
|
|
181
225
|
|
|
@@ -186,7 +230,7 @@ class DIDClient(BaseClient):
|
|
|
186
230
|
"""
|
|
187
231
|
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'dids'])
|
|
188
232
|
url = build_url(choice(self.list_hosts), path=path)
|
|
189
|
-
data = {'dids': dids}
|
|
233
|
+
data: dict[str, Any] = {'dids': dids}
|
|
190
234
|
if rse:
|
|
191
235
|
data['rse'] = rse
|
|
192
236
|
r = self._send_request(url, type_='POST', data=render_json(**data))
|
|
@@ -196,7 +240,12 @@ class DIDClient(BaseClient):
|
|
|
196
240
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
197
241
|
raise exc_cls(exc_msg)
|
|
198
242
|
|
|
199
|
-
def detach_dids(
|
|
243
|
+
def detach_dids(
|
|
244
|
+
self,
|
|
245
|
+
scope: str,
|
|
246
|
+
name: str,
|
|
247
|
+
dids: Optional["Sequence[Mapping[str, Any]]"] = None
|
|
248
|
+
) -> bool:
|
|
200
249
|
"""
|
|
201
250
|
Detach data identifier
|
|
202
251
|
|
|
@@ -214,7 +263,11 @@ class DIDClient(BaseClient):
|
|
|
214
263
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
215
264
|
raise exc_cls(exc_msg)
|
|
216
265
|
|
|
217
|
-
def attach_dids_to_dids(
|
|
266
|
+
def attach_dids_to_dids(
|
|
267
|
+
self,
|
|
268
|
+
attachments: "Sequence[dict[str, Union[str, Sequence[dict[str, Any]]]]]",
|
|
269
|
+
ignore_duplicate: bool = False
|
|
270
|
+
) -> bool:
|
|
218
271
|
"""
|
|
219
272
|
Add dids to dids.
|
|
220
273
|
|
|
@@ -234,7 +287,11 @@ class DIDClient(BaseClient):
|
|
|
234
287
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
235
288
|
raise exc_cls(exc_msg)
|
|
236
289
|
|
|
237
|
-
def add_files_to_datasets(
|
|
290
|
+
def add_files_to_datasets(
|
|
291
|
+
self,
|
|
292
|
+
attachments: "Sequence[dict[str, Union[str, Sequence[dict[str, Any]]]]]",
|
|
293
|
+
ignore_duplicate: bool = False
|
|
294
|
+
) -> bool:
|
|
238
295
|
"""
|
|
239
296
|
Add files to datasets.
|
|
240
297
|
|
|
@@ -247,7 +304,10 @@ class DIDClient(BaseClient):
|
|
|
247
304
|
return self.attach_dids_to_dids(attachments=attachments,
|
|
248
305
|
ignore_duplicate=ignore_duplicate)
|
|
249
306
|
|
|
250
|
-
def add_datasets_to_containers(
|
|
307
|
+
def add_datasets_to_containers(
|
|
308
|
+
self,
|
|
309
|
+
attachments: "Sequence[dict[str, Union[str, Sequence[dict[str, Any]]]]]"
|
|
310
|
+
) -> bool:
|
|
251
311
|
"""
|
|
252
312
|
Add datasets_to_containers.
|
|
253
313
|
|
|
@@ -258,7 +318,10 @@ class DIDClient(BaseClient):
|
|
|
258
318
|
"""
|
|
259
319
|
return self.attach_dids_to_dids(attachments=attachments)
|
|
260
320
|
|
|
261
|
-
def add_containers_to_containers(
|
|
321
|
+
def add_containers_to_containers(
|
|
322
|
+
self,
|
|
323
|
+
attachments: "Sequence[dict[str, Union[str, Sequence[dict[str, Any]]]]]"
|
|
324
|
+
) -> bool:
|
|
262
325
|
"""
|
|
263
326
|
Add containers_to_containers.
|
|
264
327
|
|
|
@@ -269,7 +332,13 @@ class DIDClient(BaseClient):
|
|
|
269
332
|
"""
|
|
270
333
|
return self.attach_dids_to_dids(attachments=attachments)
|
|
271
334
|
|
|
272
|
-
def add_files_to_dataset(
|
|
335
|
+
def add_files_to_dataset(
|
|
336
|
+
self,
|
|
337
|
+
scope: str,
|
|
338
|
+
name: str,
|
|
339
|
+
files: "Sequence[Mapping[str, Any]]",
|
|
340
|
+
rse: Optional[str] = None
|
|
341
|
+
) -> bool:
|
|
273
342
|
"""
|
|
274
343
|
Add files to datasets.
|
|
275
344
|
|
|
@@ -280,7 +349,12 @@ class DIDClient(BaseClient):
|
|
|
280
349
|
"""
|
|
281
350
|
return self.attach_dids(scope=scope, name=name, dids=files, rse=rse)
|
|
282
351
|
|
|
283
|
-
def add_files_to_archive(
|
|
352
|
+
def add_files_to_archive(
|
|
353
|
+
self,
|
|
354
|
+
scope: str,
|
|
355
|
+
name: str,
|
|
356
|
+
files: "Sequence[Mapping[str, Any]]"
|
|
357
|
+
) -> bool:
|
|
284
358
|
"""
|
|
285
359
|
Add files to archive.
|
|
286
360
|
|
|
@@ -290,7 +364,12 @@ class DIDClient(BaseClient):
|
|
|
290
364
|
"""
|
|
291
365
|
return self.attach_dids(scope=scope, name=name, dids=files)
|
|
292
366
|
|
|
293
|
-
def add_datasets_to_container(
|
|
367
|
+
def add_datasets_to_container(
|
|
368
|
+
self,
|
|
369
|
+
scope: str,
|
|
370
|
+
name: str,
|
|
371
|
+
dsns: "Sequence[Mapping[str, Any]]"
|
|
372
|
+
) -> bool:
|
|
294
373
|
"""
|
|
295
374
|
Add datasets to container.
|
|
296
375
|
|
|
@@ -300,7 +379,12 @@ class DIDClient(BaseClient):
|
|
|
300
379
|
"""
|
|
301
380
|
return self.attach_dids(scope=scope, name=name, dids=dsns)
|
|
302
381
|
|
|
303
|
-
def add_containers_to_container(
|
|
382
|
+
def add_containers_to_container(
|
|
383
|
+
self,
|
|
384
|
+
scope: str,
|
|
385
|
+
name: str,
|
|
386
|
+
cnts: "Sequence[Mapping[str, Any]]"
|
|
387
|
+
) -> bool:
|
|
304
388
|
"""
|
|
305
389
|
Add containers to container.
|
|
306
390
|
|
|
@@ -310,7 +394,11 @@ class DIDClient(BaseClient):
|
|
|
310
394
|
"""
|
|
311
395
|
return self.attach_dids(scope=scope, name=name, dids=cnts)
|
|
312
396
|
|
|
313
|
-
def list_content(
|
|
397
|
+
def list_content(
|
|
398
|
+
self,
|
|
399
|
+
scope: str,
|
|
400
|
+
name: str
|
|
401
|
+
) -> "Iterator[dict[str, Any]]":
|
|
314
402
|
"""
|
|
315
403
|
List data identifier contents.
|
|
316
404
|
|
|
@@ -326,7 +414,11 @@ class DIDClient(BaseClient):
|
|
|
326
414
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
327
415
|
raise exc_cls(exc_msg)
|
|
328
416
|
|
|
329
|
-
def list_content_history(
|
|
417
|
+
def list_content_history(
|
|
418
|
+
self,
|
|
419
|
+
scope: str,
|
|
420
|
+
name: str
|
|
421
|
+
) -> "Iterator[dict[str, Any]]":
|
|
330
422
|
"""
|
|
331
423
|
List data identifier contents history.
|
|
332
424
|
|
|
@@ -342,7 +434,12 @@ class DIDClient(BaseClient):
|
|
|
342
434
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
343
435
|
raise exc_cls(exc_msg)
|
|
344
436
|
|
|
345
|
-
def list_files(
|
|
437
|
+
def list_files(
|
|
438
|
+
self,
|
|
439
|
+
scope: str,
|
|
440
|
+
name: str,
|
|
441
|
+
long: Optional[bool] = None
|
|
442
|
+
) -> "Iterator[dict[str, Any]]":
|
|
346
443
|
"""
|
|
347
444
|
List data identifier file contents.
|
|
348
445
|
|
|
@@ -364,7 +461,31 @@ class DIDClient(BaseClient):
|
|
|
364
461
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
365
462
|
raise exc_cls(exc_msg)
|
|
366
463
|
|
|
367
|
-
def
|
|
464
|
+
def bulk_list_files(self, dids: list[dict[str, Any]]) -> "Iterator[dict[str, Any]]":
|
|
465
|
+
"""
|
|
466
|
+
List data identifier file contents.
|
|
467
|
+
|
|
468
|
+
:param dids: The list of DIDs.
|
|
469
|
+
"""
|
|
470
|
+
|
|
471
|
+
data = {'dids': dids}
|
|
472
|
+
path = '/'.join([self.DIDS_BASEURL, 'bulkfiles'])
|
|
473
|
+
url = build_url(choice(self.list_hosts), path=path)
|
|
474
|
+
|
|
475
|
+
r = self._send_request(url, type_='POST', data=dumps(data), stream=True)
|
|
476
|
+
if r.status_code == codes.ok:
|
|
477
|
+
return self._load_json_data(r)
|
|
478
|
+
else:
|
|
479
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
480
|
+
raise exc_cls(exc_msg)
|
|
481
|
+
|
|
482
|
+
def get_did(
|
|
483
|
+
self,
|
|
484
|
+
scope: str,
|
|
485
|
+
name: str,
|
|
486
|
+
dynamic: bool = False,
|
|
487
|
+
dynamic_depth: Optional[str] = None
|
|
488
|
+
) -> dict[str, Any]:
|
|
368
489
|
"""
|
|
369
490
|
Retrieve a single data identifier.
|
|
370
491
|
|
|
@@ -389,7 +510,12 @@ class DIDClient(BaseClient):
|
|
|
389
510
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
390
511
|
raise exc_cls(exc_msg)
|
|
391
512
|
|
|
392
|
-
def get_metadata(
|
|
513
|
+
def get_metadata(
|
|
514
|
+
self,
|
|
515
|
+
scope: str,
|
|
516
|
+
name: str,
|
|
517
|
+
plugin: str = 'DID_COLUMN'
|
|
518
|
+
) -> dict[str, Any]:
|
|
393
519
|
"""
|
|
394
520
|
Get data identifier metadata
|
|
395
521
|
|
|
@@ -409,7 +535,11 @@ class DIDClient(BaseClient):
|
|
|
409
535
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
410
536
|
raise exc_cls(exc_msg)
|
|
411
537
|
|
|
412
|
-
def get_metadata_bulk(
|
|
538
|
+
def get_metadata_bulk(
|
|
539
|
+
self,
|
|
540
|
+
dids: "Sequence[Mapping[str, Any]]",
|
|
541
|
+
inherit: bool = False
|
|
542
|
+
) -> "Iterator[dict[str, Any]]":
|
|
413
543
|
"""
|
|
414
544
|
Bulk get data identifier metadata
|
|
415
545
|
:param inherit: A boolean. If set to true, the metadata of the parent are concatenated.
|
|
@@ -424,7 +554,14 @@ class DIDClient(BaseClient):
|
|
|
424
554
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
425
555
|
raise exc_cls(exc_msg)
|
|
426
556
|
|
|
427
|
-
def set_metadata(
|
|
557
|
+
def set_metadata(
|
|
558
|
+
self,
|
|
559
|
+
scope: str,
|
|
560
|
+
name: str,
|
|
561
|
+
key: str,
|
|
562
|
+
value: Any,
|
|
563
|
+
recursive: bool = False
|
|
564
|
+
) -> bool:
|
|
428
565
|
"""
|
|
429
566
|
Set data identifier metadata
|
|
430
567
|
|
|
@@ -444,14 +581,19 @@ class DIDClient(BaseClient):
|
|
|
444
581
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
445
582
|
raise exc_cls(exc_msg)
|
|
446
583
|
|
|
447
|
-
def set_metadata_bulk(
|
|
584
|
+
def set_metadata_bulk(
|
|
585
|
+
self,
|
|
586
|
+
scope: str,
|
|
587
|
+
name: str,
|
|
588
|
+
meta: "Mapping[str, Any]",
|
|
589
|
+
recursive: bool = False
|
|
590
|
+
) -> bool:
|
|
448
591
|
"""
|
|
449
592
|
Set data identifier metadata in bulk.
|
|
450
593
|
|
|
451
594
|
:param scope: The scope name.
|
|
452
595
|
:param name: The data identifier name.
|
|
453
596
|
:param meta: the metadata key-values.
|
|
454
|
-
:type meta: dict
|
|
455
597
|
:param recursive: Option to propagate the metadata change to content.
|
|
456
598
|
"""
|
|
457
599
|
path = '/'.join([self.DIDS_BASEURL, quote_plus(scope), quote_plus(name), 'meta'])
|
|
@@ -464,11 +606,15 @@ class DIDClient(BaseClient):
|
|
|
464
606
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
465
607
|
raise exc_cls(exc_msg)
|
|
466
608
|
|
|
467
|
-
def set_dids_metadata_bulk(
|
|
609
|
+
def set_dids_metadata_bulk(
|
|
610
|
+
self,
|
|
611
|
+
dids: "Sequence[Mapping[str, Any]]",
|
|
612
|
+
recursive: bool = False
|
|
613
|
+
) -> bool:
|
|
468
614
|
"""
|
|
469
615
|
Set metadata to a list of data identifiers.
|
|
470
616
|
|
|
471
|
-
:param dids: A list of dids including metadata, i.e. [
|
|
617
|
+
:param dids: A list of dids including metadata, i.e. [{'scope': scope1, 'name': name1, 'meta': {key1: value1, key2: value2}] .
|
|
472
618
|
:param recursive: Option to propagate the metadata update to content.
|
|
473
619
|
"""
|
|
474
620
|
path = '/'.join([self.DIDS_BASEURL, 'bulkdidsmeta'])
|
|
@@ -481,7 +627,12 @@ class DIDClient(BaseClient):
|
|
|
481
627
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
482
628
|
raise exc_cls(exc_msg)
|
|
483
629
|
|
|
484
|
-
def set_status(
|
|
630
|
+
def set_status(
|
|
631
|
+
self,
|
|
632
|
+
scope: str,
|
|
633
|
+
name: str,
|
|
634
|
+
**kwargs
|
|
635
|
+
) -> bool:
|
|
485
636
|
"""
|
|
486
637
|
Set data identifier status
|
|
487
638
|
|
|
@@ -499,7 +650,11 @@ class DIDClient(BaseClient):
|
|
|
499
650
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
500
651
|
raise exc_cls(exc_msg)
|
|
501
652
|
|
|
502
|
-
def close(
|
|
653
|
+
def close(
|
|
654
|
+
self,
|
|
655
|
+
scope: str,
|
|
656
|
+
name: str
|
|
657
|
+
) -> bool:
|
|
503
658
|
"""
|
|
504
659
|
close dataset/container
|
|
505
660
|
|
|
@@ -508,7 +663,12 @@ class DIDClient(BaseClient):
|
|
|
508
663
|
"""
|
|
509
664
|
return self.set_status(scope=scope, name=name, open=False)
|
|
510
665
|
|
|
511
|
-
def delete_metadata(
|
|
666
|
+
def delete_metadata(
|
|
667
|
+
self,
|
|
668
|
+
scope: str,
|
|
669
|
+
name: str,
|
|
670
|
+
key: str
|
|
671
|
+
) -> bool:
|
|
512
672
|
"""
|
|
513
673
|
Delete data identifier metadata
|
|
514
674
|
|
|
@@ -526,7 +686,11 @@ class DIDClient(BaseClient):
|
|
|
526
686
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
527
687
|
raise exc_cls(exc_msg)
|
|
528
688
|
|
|
529
|
-
def list_did_rules(
|
|
689
|
+
def list_did_rules(
|
|
690
|
+
self,
|
|
691
|
+
scope: str,
|
|
692
|
+
name: str
|
|
693
|
+
) -> "Iterator[dict[str, Any]]":
|
|
530
694
|
"""
|
|
531
695
|
List the associated rules of a data identifier.
|
|
532
696
|
|
|
@@ -543,7 +707,11 @@ class DIDClient(BaseClient):
|
|
|
543
707
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
544
708
|
raise exc_cls(exc_msg)
|
|
545
709
|
|
|
546
|
-
def list_associated_rules_for_file(
|
|
710
|
+
def list_associated_rules_for_file(
|
|
711
|
+
self,
|
|
712
|
+
scope: str,
|
|
713
|
+
name: str
|
|
714
|
+
) -> "Iterator[dict[str, Any]]":
|
|
547
715
|
"""
|
|
548
716
|
List the associated rules a file is affected from..
|
|
549
717
|
|
|
@@ -560,7 +728,7 @@ class DIDClient(BaseClient):
|
|
|
560
728
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
561
729
|
raise exc_cls(exc_msg)
|
|
562
730
|
|
|
563
|
-
def get_dataset_by_guid(self, guid):
|
|
731
|
+
def get_dataset_by_guid(self, guid: str) -> "Iterator[dict[str, Any]]":
|
|
564
732
|
"""
|
|
565
733
|
Get the parent datasets for a given GUID.
|
|
566
734
|
:param guid: The GUID.
|
|
@@ -577,7 +745,12 @@ class DIDClient(BaseClient):
|
|
|
577
745
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
578
746
|
raise exc_cls(exc_msg)
|
|
579
747
|
|
|
580
|
-
def scope_list(
|
|
748
|
+
def scope_list(
|
|
749
|
+
self,
|
|
750
|
+
scope: str,
|
|
751
|
+
name: Optional[str] = None,
|
|
752
|
+
recursive: bool = False
|
|
753
|
+
) -> "Iterator[dict[str, Any]]":
|
|
581
754
|
"""
|
|
582
755
|
List data identifiers in a scope.
|
|
583
756
|
|
|
@@ -601,7 +774,11 @@ class DIDClient(BaseClient):
|
|
|
601
774
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
602
775
|
raise exc_cls(exc_msg)
|
|
603
776
|
|
|
604
|
-
def list_parent_dids(
|
|
777
|
+
def list_parent_dids(
|
|
778
|
+
self,
|
|
779
|
+
scope: str,
|
|
780
|
+
name: str
|
|
781
|
+
) -> "Iterator[dict[str, Any]]":
|
|
605
782
|
"""
|
|
606
783
|
List parent dataset/containers of a did.
|
|
607
784
|
|
|
@@ -619,7 +796,14 @@ class DIDClient(BaseClient):
|
|
|
619
796
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
620
797
|
raise exc_cls(exc_msg)
|
|
621
798
|
|
|
622
|
-
def create_did_sample(
|
|
799
|
+
def create_did_sample(
|
|
800
|
+
self,
|
|
801
|
+
input_scope: str,
|
|
802
|
+
input_name: str,
|
|
803
|
+
output_scope: str,
|
|
804
|
+
output_name: str,
|
|
805
|
+
nbfiles: int
|
|
806
|
+
) -> bool:
|
|
623
807
|
"""
|
|
624
808
|
Create a sample from an input collection.
|
|
625
809
|
|
|
@@ -646,7 +830,7 @@ class DIDClient(BaseClient):
|
|
|
646
830
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
647
831
|
raise exc_cls(exc_msg)
|
|
648
832
|
|
|
649
|
-
def resurrect(self, dids):
|
|
833
|
+
def resurrect(self, dids: "Sequence[Mapping[str, Any]]") -> bool:
|
|
650
834
|
"""
|
|
651
835
|
Resurrect a list of dids.
|
|
652
836
|
|
|
@@ -661,20 +845,11 @@ class DIDClient(BaseClient):
|
|
|
661
845
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
662
846
|
raise exc_cls(exc_msg)
|
|
663
847
|
|
|
664
|
-
def
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
"""
|
|
670
|
-
url = build_url(choice(self.list_hosts), path='tmp_dids')
|
|
671
|
-
r = self._send_request(url, type_='POST', data=dumps(dids))
|
|
672
|
-
if r.status_code == codes.created:
|
|
673
|
-
return True
|
|
674
|
-
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
675
|
-
raise exc_cls(exc_msg)
|
|
676
|
-
|
|
677
|
-
def list_archive_content(self, scope, name):
|
|
848
|
+
def list_archive_content(
|
|
849
|
+
self,
|
|
850
|
+
scope: str,
|
|
851
|
+
name: str
|
|
852
|
+
) -> "Iterator[dict[str, Any]]":
|
|
678
853
|
"""
|
|
679
854
|
List archive contents.
|
|
680
855
|
|