rucio-clients 37.7.1__py3-none-any.whl → 38.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/cli/bin_legacy/rucio.py +51 -107
- rucio/cli/bin_legacy/rucio_admin.py +26 -26
- rucio/cli/command.py +1 -0
- rucio/cli/did.py +2 -2
- rucio/cli/opendata.py +132 -0
- rucio/cli/replica.py +15 -5
- rucio/cli/rule.py +7 -2
- rucio/cli/scope.py +3 -2
- rucio/cli/utils.py +28 -4
- rucio/client/baseclient.py +9 -1
- rucio/client/client.py +2 -0
- rucio/client/diracclient.py +73 -12
- rucio/client/opendataclient.py +249 -0
- rucio/client/subscriptionclient.py +30 -0
- rucio/client/uploadclient.py +10 -13
- rucio/common/constants.py +4 -1
- rucio/common/exception.py +55 -0
- rucio/common/plugins.py +45 -8
- rucio/common/schema/generic.py +5 -3
- rucio/common/schema/generic_multi_vo.py +4 -2
- rucio/common/types.py +8 -7
- rucio/common/utils.py +176 -11
- rucio/rse/protocols/protocol.py +9 -5
- rucio/rse/translation.py +17 -6
- rucio/vcsversion.py +4 -4
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/data/etc/rucio.cfg.template +2 -2
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/data/requirements.client.txt +1 -1
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/scripts/rucio +2 -1
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/METADATA +2 -2
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/RECORD +39 -38
- rucio/client/fileclient.py +0 -57
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/data/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-37.7.1.data → rucio_clients-38.0.0rc1.data}/scripts/rucio-admin +0 -0
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/WHEEL +0 -0
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-37.7.1.dist-info → rucio_clients-38.0.0rc1.dist-info}/top_level.txt +0 -0
rucio/cli/opendata.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from typing import TYPE_CHECKING, Optional
|
|
17
|
+
|
|
18
|
+
import click
|
|
19
|
+
|
|
20
|
+
from rucio.cli.utils import JSONType
|
|
21
|
+
from rucio.common.constants import OPENDATA_DID_STATE_LITERAL_LIST
|
|
22
|
+
from rucio.common.utils import extract_scope
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from click import Context
|
|
26
|
+
|
|
27
|
+
from rucio.common.constants import OPENDATA_DID_STATE_LITERAL
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def is_valid_json(s: str) -> bool:
|
|
31
|
+
try:
|
|
32
|
+
json.loads(s)
|
|
33
|
+
return True
|
|
34
|
+
except json.JSONDecodeError:
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@click.group()
|
|
39
|
+
def opendata() -> None:
|
|
40
|
+
"""Manage Opendata resources"""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@opendata.group(name="did")
|
|
44
|
+
def opendata_did() -> None:
|
|
45
|
+
"""Manage Opendata DIDs"""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@opendata_did.command("list")
|
|
49
|
+
@click.option("--state", type=click.Choice(OPENDATA_DID_STATE_LITERAL_LIST, case_sensitive=False), required=False,
|
|
50
|
+
help="Filter on Opendata state")
|
|
51
|
+
@click.option("--public", required=False, is_flag=True, default=False,
|
|
52
|
+
help="Perform request against the public endpoint")
|
|
53
|
+
@click.pass_context
|
|
54
|
+
def list_opendata_dids(ctx: "Context", state: Optional["OPENDATA_DID_STATE_LITERAL"], public: bool) -> None:
|
|
55
|
+
"""
|
|
56
|
+
List Opendata DIDs, optionally filtered by state and public/private access
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
client = ctx.obj.client
|
|
60
|
+
result = client.list_opendata_dids(state=state, public=public)
|
|
61
|
+
print(json.dumps(result, indent=4, sort_keys=True, ensure_ascii=False))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@opendata_did.command("add")
|
|
65
|
+
@click.argument("did")
|
|
66
|
+
@click.pass_context
|
|
67
|
+
def add_opendata_did(ctx: "Context", did: str) -> None:
|
|
68
|
+
"""
|
|
69
|
+
Adds an existing DID to the Opendata catalog
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
client = ctx.obj.client
|
|
73
|
+
scope, name = extract_scope(did)
|
|
74
|
+
client.add_opendata_did(scope=scope, name=name)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@opendata_did.command("remove")
|
|
78
|
+
@click.argument("did")
|
|
79
|
+
@click.pass_context
|
|
80
|
+
def remove_opendata_did(ctx: "Context", did: str) -> None:
|
|
81
|
+
"""
|
|
82
|
+
Removes an existing Opendata DID from the Opendata catalog
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
client = ctx.obj.client
|
|
86
|
+
scope, name = extract_scope(did)
|
|
87
|
+
client.remove_opendata_did(scope=scope, name=name)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@opendata_did.command("show")
|
|
91
|
+
@click.argument("did")
|
|
92
|
+
@click.option("--meta", required=False, is_flag=True, default=False, help="Print only the opendata metadata")
|
|
93
|
+
@click.option("--files", required=False, is_flag=True, default=False,
|
|
94
|
+
help="Print the files associated with the opendata DID")
|
|
95
|
+
@click.option("--public", required=False, is_flag=True, default=False,
|
|
96
|
+
help="Perform request against the public endpoint")
|
|
97
|
+
@click.pass_context
|
|
98
|
+
def get_opendata_did(ctx: "Context", did: str, include_files: bool, include_metadata: bool, public: bool) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Get information about an Opendata DID, optionally including files and metadata.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
client = ctx.obj.client
|
|
104
|
+
scope, name = extract_scope(did)
|
|
105
|
+
result = client.get_opendata_did(scope=scope, name=name, public=public,
|
|
106
|
+
include_files=include_files, include_metadata=include_metadata,
|
|
107
|
+
include_doi=True)
|
|
108
|
+
# TODO: pretty print using tables, etc
|
|
109
|
+
print(json.dumps(result, indent=4, sort_keys=True, ensure_ascii=False))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@opendata_did.command("update")
|
|
113
|
+
@click.argument("did")
|
|
114
|
+
@click.option("--meta", type=JSONType(), required=False, help="Opendata JSON")
|
|
115
|
+
@click.option("--state", type=click.Choice(OPENDATA_DID_STATE_LITERAL_LIST, case_sensitive=False), required=False,
|
|
116
|
+
help="State of the Opendata DID")
|
|
117
|
+
@click.option("--doi", required=False,
|
|
118
|
+
help="Digital Object Identifier (DOI) for the Opendata DID (e.g., 10.1234/foo.bar)")
|
|
119
|
+
@click.pass_context
|
|
120
|
+
def update_opendata_did(ctx: "Context", did: str, meta: Optional[str],
|
|
121
|
+
state: Optional["OPENDATA_DID_STATE_LITERAL"],
|
|
122
|
+
doi: Optional[str]) -> None:
|
|
123
|
+
"""
|
|
124
|
+
Update an existing Opendata DID in the Opendata catalog.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
client = ctx.obj.client
|
|
128
|
+
if not any([meta, state, doi]):
|
|
129
|
+
raise ValueError("At least one of --meta, --state, or --doi must be provided.")
|
|
130
|
+
|
|
131
|
+
scope, name = extract_scope(did)
|
|
132
|
+
client.update_opendata_did(scope=scope, name=name, meta=meta, state=state, doi=doi)
|
rucio/cli/replica.py
CHANGED
|
@@ -11,12 +11,17 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
+
from typing import TYPE_CHECKING, Optional
|
|
15
|
+
|
|
14
16
|
import click
|
|
15
17
|
|
|
16
|
-
from rucio.cli.bin_legacy.rucio import list_dataset_replicas, list_file_replicas, list_suspicious_replicas
|
|
18
|
+
from rucio.cli.bin_legacy.rucio import list_dataset_replicas, list_datasets_rse, list_file_replicas, list_suspicious_replicas
|
|
17
19
|
from rucio.cli.bin_legacy.rucio_admin import declare_bad_file_replicas, declare_temporary_unavailable_replicas, quarantine_replicas, set_tombstone
|
|
18
20
|
from rucio.cli.utils import Arguments
|
|
19
21
|
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from collections.abc import Sequence
|
|
24
|
+
|
|
20
25
|
|
|
21
26
|
@click.group()
|
|
22
27
|
def replica():
|
|
@@ -60,13 +65,18 @@ def list_(ctx, dids, protocols, all_states, pfns, domain, link, missing, metalin
|
|
|
60
65
|
|
|
61
66
|
@replica_list.command("dataset")
|
|
62
67
|
@click.argument("dids", nargs=-1)
|
|
68
|
+
@click.option("--rse", default=None, help="RSE name to use a filter")
|
|
63
69
|
@click.option("--deep", default=False, is_flag=True, help="Make a deep check, checking the contents of datasets in datasets")
|
|
64
70
|
@click.option("--csv", help="Write output to comma separated values", is_flag=True, default=False)
|
|
65
71
|
@click.pass_context
|
|
66
|
-
def list_dataset(ctx, dids, deep, csv):
|
|
67
|
-
"""List dataset replicas"""
|
|
68
|
-
|
|
69
|
-
|
|
72
|
+
def list_dataset(ctx, dids: Optional["Sequence[str]"], rse: Optional[str], deep: bool, csv: bool):
|
|
73
|
+
"""List dataset replicas, or view all datasets at a RSE"""
|
|
74
|
+
if rse is None:
|
|
75
|
+
args = Arguments({"no_pager": ctx.obj.no_pager, "dids": dids, "deep": deep, "csv": csv})
|
|
76
|
+
list_dataset_replicas(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
77
|
+
else:
|
|
78
|
+
args = Arguments({"no_pager": ctx.obj.no_pager, "rse": rse})
|
|
79
|
+
list_datasets_rse(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
70
80
|
|
|
71
81
|
|
|
72
82
|
@replica.command("remove")
|
rucio/cli/rule.py
CHANGED
|
@@ -15,6 +15,7 @@ import click
|
|
|
15
15
|
|
|
16
16
|
from rucio.cli.bin_legacy.rucio import add_rule, delete_rule, info_rule, list_rules, list_rules_history, move_rule, update_rule
|
|
17
17
|
from rucio.cli.utils import Arguments
|
|
18
|
+
from rucio.common.exception import InputValidationError
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
@click.group()
|
|
@@ -149,7 +150,7 @@ def update(
|
|
|
149
150
|
|
|
150
151
|
|
|
151
152
|
@rule.command("list")
|
|
152
|
-
@click.
|
|
153
|
+
@click.option("--did", help="Filter by DID")
|
|
153
154
|
@click.option("--traverse", is_flag=True, default=False, help="Traverse the did tree and search for rules affecting this did")
|
|
154
155
|
@click.option("--csv", is_flag=True, default=False, help="Comma Separated Value output")
|
|
155
156
|
@click.option("--file", help="Filter by file")
|
|
@@ -158,5 +159,9 @@ def update(
|
|
|
158
159
|
@click.pass_context
|
|
159
160
|
def list_(ctx, did, traverse, csv, file, account, subscription):
|
|
160
161
|
"""List all rules impacting a given DID"""
|
|
161
|
-
|
|
162
|
+
# Done here to raise error==2
|
|
163
|
+
if not (did or file or account or subscription):
|
|
164
|
+
raise InputValidationError("At least one option has to be given. Use -h to list the options.")
|
|
165
|
+
|
|
166
|
+
args = Arguments({"no_pager": ctx.obj.no_pager, "did": did, "traverse": traverse, "csv": csv, "file": file, "rule_account": account, "subscription": subscription})
|
|
162
167
|
list_rules(args, ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
rucio/cli/scope.py
CHANGED
|
@@ -34,7 +34,8 @@ def add_(ctx, account, scope_name):
|
|
|
34
34
|
|
|
35
35
|
@scope.command("list")
|
|
36
36
|
@click.option("-a", "--account", help="Filter by associated account", required=False)
|
|
37
|
+
@click.option("--csv", is_flag=True, help="Output in CSV format", default=False)
|
|
37
38
|
@click.pass_context
|
|
38
|
-
def list_(ctx, account):
|
|
39
|
+
def list_(ctx: click.Context, account: str, csv: bool):
|
|
39
40
|
"""List existing scopes"""
|
|
40
|
-
list_scopes(Arguments({"no_pager": ctx.obj.no_pager, "account": account}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
|
41
|
+
list_scopes(Arguments({"no_pager": ctx.obj.no_pager, "account": account, "csv": csv}), ctx.obj.client, ctx.obj.logger, ctx.obj.console, ctx.obj.spinner)
|
rucio/cli/utils.py
CHANGED
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
+
|
|
14
15
|
import errno
|
|
16
|
+
import json
|
|
15
17
|
import logging
|
|
16
18
|
import os
|
|
17
19
|
import signal
|
|
@@ -20,6 +22,9 @@ import sys
|
|
|
20
22
|
import traceback
|
|
21
23
|
from configparser import NoOptionError, NoSectionError
|
|
22
24
|
from functools import wraps
|
|
25
|
+
from typing import Optional, Union
|
|
26
|
+
|
|
27
|
+
import click
|
|
23
28
|
|
|
24
29
|
from rucio.client.client import Client
|
|
25
30
|
from rucio.common.config import config_get
|
|
@@ -30,7 +35,7 @@ from rucio.common.exception import (
|
|
|
30
35
|
DataIdentifierNotFound,
|
|
31
36
|
Duplicate,
|
|
32
37
|
DuplicateContent,
|
|
33
|
-
|
|
38
|
+
InputValidationError,
|
|
34
39
|
InvalidRSEExpression,
|
|
35
40
|
MissingDependency,
|
|
36
41
|
RSENotFound,
|
|
@@ -52,12 +57,13 @@ def exception_handler(function):
|
|
|
52
57
|
def new_funct(*args, **kwargs):
|
|
53
58
|
try:
|
|
54
59
|
return function(*args, **kwargs)
|
|
60
|
+
except InputValidationError as error:
|
|
61
|
+
logger.error(error)
|
|
62
|
+
logger.debug("This means that one you provided an invalid combination of parameters, or incorrect types. Please check the command help (-h/--help).")
|
|
63
|
+
return FAILURE
|
|
55
64
|
except NotImplementedError as error:
|
|
56
65
|
logger.error(f"Cannot run that operation/command combination {error}")
|
|
57
66
|
return FAILURE
|
|
58
|
-
except InvalidObject as error:
|
|
59
|
-
logger.error(error)
|
|
60
|
-
return error.error_code
|
|
61
67
|
except DataIdentifierNotFound as error:
|
|
62
68
|
logger.error(error)
|
|
63
69
|
logger.debug("This means that the Data IDentifier you provided is not known by Rucio.")
|
|
@@ -224,3 +230,21 @@ class Arguments(dict):
|
|
|
224
230
|
__getattr__ = dict.get
|
|
225
231
|
__setattr__ = dict.__setitem__
|
|
226
232
|
__delattr__ = dict.__delitem__
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class JSONType(click.ParamType):
|
|
236
|
+
name = "json"
|
|
237
|
+
|
|
238
|
+
def convert(
|
|
239
|
+
self,
|
|
240
|
+
value: Union[str, None],
|
|
241
|
+
param: "Optional[click.Parameter]",
|
|
242
|
+
ctx: "Optional[click.Context]",
|
|
243
|
+
) -> Optional[dict]:
|
|
244
|
+
if value is None:
|
|
245
|
+
return None
|
|
246
|
+
|
|
247
|
+
try:
|
|
248
|
+
return json.loads(value)
|
|
249
|
+
except json.JSONDecodeError as e:
|
|
250
|
+
self.fail(f"Invalid JSON: {e}", param, ctx)
|
rucio/client/baseclient.py
CHANGED
|
@@ -41,7 +41,7 @@ from rucio.common.config import config_get, config_get_bool, config_get_int, con
|
|
|
41
41
|
from rucio.common.constants import DEFAULT_VO
|
|
42
42
|
from rucio.common.exception import CannotAuthenticate, ClientProtocolNotFound, ClientProtocolNotSupported, ConfigNotFound, MissingClientParameter, MissingModuleException, NoAuthInformation, ServerConnectionException
|
|
43
43
|
from rucio.common.extra import import_extras
|
|
44
|
-
from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign
|
|
44
|
+
from rucio.common.utils import build_url, get_tmp_dir, my_key_generator, parse_response, setup_logger, ssh_sign, wlcg_token_discovery
|
|
45
45
|
|
|
46
46
|
if TYPE_CHECKING:
|
|
47
47
|
from collections.abc import Generator
|
|
@@ -944,6 +944,14 @@ class BaseClient:
|
|
|
944
944
|
|
|
945
945
|
:return: True if a token could be read. False if no file exists.
|
|
946
946
|
"""
|
|
947
|
+
|
|
948
|
+
if self.auth_type == "oidc":
|
|
949
|
+
token = wlcg_token_discovery()
|
|
950
|
+
if token:
|
|
951
|
+
self.auth_token = token
|
|
952
|
+
self.headers['X-Rucio-Auth-Token'] = self.auth_token
|
|
953
|
+
return True
|
|
954
|
+
|
|
947
955
|
if not os.path.exists(self.token_file):
|
|
948
956
|
return False
|
|
949
957
|
|
rucio/client/client.py
CHANGED
|
@@ -27,6 +27,7 @@ from rucio.client.importclient import ImportClient
|
|
|
27
27
|
from rucio.client.lifetimeclient import LifetimeClient
|
|
28
28
|
from rucio.client.lockclient import LockClient
|
|
29
29
|
from rucio.client.metaconventionsclient import MetaConventionClient
|
|
30
|
+
from rucio.client.opendataclient import OpenDataClient
|
|
30
31
|
from rucio.client.pingclient import PingClient
|
|
31
32
|
from rucio.client.replicaclient import ReplicaClient
|
|
32
33
|
from rucio.client.requestclient import RequestClient
|
|
@@ -46,6 +47,7 @@ class Client(AccountClient,
|
|
|
46
47
|
RSEClient,
|
|
47
48
|
ScopeClient,
|
|
48
49
|
DIDClient,
|
|
50
|
+
OpenDataClient,
|
|
49
51
|
RuleClient,
|
|
50
52
|
SubscriptionClient,
|
|
51
53
|
LockClient,
|
rucio/client/diracclient.py
CHANGED
|
@@ -25,8 +25,13 @@ if TYPE_CHECKING:
|
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class DiracClient(BaseClient):
|
|
28
|
+
"""
|
|
29
|
+
Client for the DIRAC integration layer.
|
|
28
30
|
|
|
29
|
-
|
|
31
|
+
This client wraps the REST calls used by the ``RucioFileCatalog`` plugin in DIRAC.
|
|
32
|
+
Only `add_files` is currently provided and it behaves like any other ``BaseClient``
|
|
33
|
+
method by handling authentication tokens and host selection automatically.
|
|
34
|
+
"""
|
|
30
35
|
|
|
31
36
|
DIRAC_BASEURL = 'dirac'
|
|
32
37
|
|
|
@@ -37,25 +42,81 @@ class DiracClient(BaseClient):
|
|
|
37
42
|
parents_metadata: Optional["Mapping[str, Mapping[str, Any]]"] = None
|
|
38
43
|
) -> Literal[True]:
|
|
39
44
|
"""
|
|
40
|
-
|
|
41
|
-
* Create the file and replica.
|
|
45
|
+
Register files and create missing parent structures.
|
|
42
46
|
|
|
43
|
-
|
|
47
|
+
For each entry in ``lfns`` the method:
|
|
44
48
|
|
|
45
|
-
|
|
49
|
+
* Creates the file and its replica on the specified RSE.
|
|
50
|
+
* If the containing dataset does not exist, it is created with a replication
|
|
51
|
+
rule using the RSE expression ``ANY=true``. This places the dataset on any
|
|
52
|
+
RSE advertising the ``ANY`` attribute.
|
|
53
|
+
* Creates all ancestor containers when needed.
|
|
54
|
+
* Attaches metadata from ``parents_metadata`` to those parents.
|
|
46
55
|
|
|
47
56
|
Parameters
|
|
48
57
|
----------
|
|
49
|
-
lfns
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
58
|
+
lfns
|
|
59
|
+
Iterable of dictionaries describing the files. Each dictionary must contain:
|
|
60
|
+
|
|
61
|
+
* **``lfn``** full logical file name with scope
|
|
62
|
+
* **``rse``** destination RSE name
|
|
63
|
+
* **``bytes``** file size in bytes
|
|
64
|
+
* **``adler32``** Adler‑32 checksum
|
|
65
|
+
|
|
66
|
+
Optional keys include ``guid`` ``pfn`` and ``meta``.
|
|
67
|
+
|
|
68
|
+
ignore_availability
|
|
69
|
+
When ``True``, the availability status of RSEs is ignored and blocked RSEs are
|
|
70
|
+
still accepted. Defaults to ``False`` which rejects blocked RSEs.
|
|
71
|
+
parents_metadata
|
|
72
|
+
Mapping of parent logical path names to metadata {'lpn': {key : value}}.
|
|
73
|
+
Entries are only applied when new datasets or containers are created.
|
|
74
|
+
Defaults to None.
|
|
75
|
+
|
|
76
|
+
Returns
|
|
77
|
+
-------
|
|
78
|
+
Literal[True]
|
|
79
|
+
When the server confirms the creation.
|
|
80
|
+
|
|
81
|
+
Raises
|
|
82
|
+
------
|
|
83
|
+
RucioException
|
|
84
|
+
Raised when the HTTP request is not successful.
|
|
85
|
+
|
|
86
|
+
Examples
|
|
87
|
+
--------
|
|
88
|
+
??? Example
|
|
89
|
+
|
|
90
|
+
Register a file using the DIRAC naming style. Dirac's scope extraction is
|
|
91
|
+
required to be set for this to work:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
>>> from rucio.client.diracclient import DiracClient
|
|
95
|
+
>>> from rucio.common.utils import generate_uuid
|
|
96
|
+
|
|
97
|
+
>>> dc = DiracClient()
|
|
98
|
+
>>> lfn = f"/belle/mock/cont_{generate_uuid()}/dataset_{generate_uuid()}/file_{generate_uuid()}"
|
|
99
|
+
>>> files = [{
|
|
100
|
+
... "lfn": lfn,
|
|
101
|
+
... "rse": "XRD1",
|
|
102
|
+
... "bytes": 1,
|
|
103
|
+
... "adler32": "0cc737eb",
|
|
104
|
+
... 'guid': generate_uuid()
|
|
105
|
+
... }]
|
|
106
|
+
|
|
107
|
+
>>> dc.add_files(files)
|
|
108
|
+
True
|
|
109
|
+
```
|
|
55
110
|
"""
|
|
56
111
|
path = '/'.join([self.DIRAC_BASEURL, 'addfiles'])
|
|
57
112
|
url = build_url(choice(self.list_hosts), path=path)
|
|
58
|
-
|
|
113
|
+
|
|
114
|
+
r = self._send_request(
|
|
115
|
+
url,
|
|
116
|
+
type_='POST',
|
|
117
|
+
data=dumps({'lfns': lfns, 'ignore_availability': ignore_availability, 'parents_metadata': parents_metadata})
|
|
118
|
+
)
|
|
119
|
+
|
|
59
120
|
if r.status_code == codes.created:
|
|
60
121
|
return True
|
|
61
122
|
else:
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
17
|
+
from urllib.parse import quote_plus
|
|
18
|
+
|
|
19
|
+
from requests.status_codes import codes
|
|
20
|
+
|
|
21
|
+
from rucio.client.baseclient import BaseClient, choice
|
|
22
|
+
from rucio.common.config import config_get
|
|
23
|
+
from rucio.common.utils import build_url, render_json
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from rucio.common.constants import OPENDATA_DID_STATE_LITERAL
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class OpenDataClient(BaseClient):
|
|
30
|
+
opendata_public_base_url = "opendata/public"
|
|
31
|
+
opendata_private_base_url = "opendata"
|
|
32
|
+
|
|
33
|
+
opendata_public_dids_base_url = f"{opendata_public_base_url}/dids"
|
|
34
|
+
opendata_private_dids_base_url = f"{opendata_private_base_url}/dids"
|
|
35
|
+
|
|
36
|
+
opendata_host_from_config = config_get('client', 'opendata_host', raise_exception=False, default=None)
|
|
37
|
+
|
|
38
|
+
def get_opendata_host(self, *, public: bool) -> str:
|
|
39
|
+
"""
|
|
40
|
+
Get the Opendata host URL for the public or private endpoint.
|
|
41
|
+
The private opendata host is the regular rucio server, while the public opendata host can be configured separately (defaults to the same as the private one).
|
|
42
|
+
|
|
43
|
+
Parameters:
|
|
44
|
+
public: If True, return the public Opendata host URL. If False, return the private Opendata host URL.
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
The Opendata host URL.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
if public and self.opendata_host_from_config is not None:
|
|
51
|
+
return self.opendata_host_from_config
|
|
52
|
+
|
|
53
|
+
return choice(self.list_hosts)
|
|
54
|
+
|
|
55
|
+
def list_opendata_dids(
|
|
56
|
+
self,
|
|
57
|
+
*,
|
|
58
|
+
state: Optional["OPENDATA_DID_STATE_LITERAL"] = None,
|
|
59
|
+
public: bool = False,
|
|
60
|
+
) -> dict[str, Any]:
|
|
61
|
+
"""
|
|
62
|
+
Return a list of Opendata DIDs, optionally filtered by state and access type.
|
|
63
|
+
|
|
64
|
+
Parameters:
|
|
65
|
+
state: The state to filter DIDs by. If None, all states are included.
|
|
66
|
+
public: If True, queries the public Opendata endpoint. Defaults to False.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
A dictionary containing the list of Opendata DIDs.
|
|
70
|
+
|
|
71
|
+
Raises:
|
|
72
|
+
ValueError: If both `state` and `public=True` are provided.
|
|
73
|
+
Exception: If the request fails or the server returns an error.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
base_url = self.opendata_public_dids_base_url if public else self.opendata_private_dids_base_url
|
|
77
|
+
path = '/'.join([base_url])
|
|
78
|
+
|
|
79
|
+
params = {}
|
|
80
|
+
|
|
81
|
+
if state is not None:
|
|
82
|
+
params['state'] = state
|
|
83
|
+
|
|
84
|
+
if state is not None and public:
|
|
85
|
+
raise ValueError('state and public cannot be provided at the same time.')
|
|
86
|
+
|
|
87
|
+
url = build_url(self.get_opendata_host(public=public), path=path)
|
|
88
|
+
r = self._send_request(url, type_='GET', params=params)
|
|
89
|
+
if r.status_code == codes.ok:
|
|
90
|
+
return json.loads(r.content.decode('utf-8'))
|
|
91
|
+
else:
|
|
92
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
93
|
+
raise exc_cls(exc_msg)
|
|
94
|
+
|
|
95
|
+
def add_opendata_did(
|
|
96
|
+
self,
|
|
97
|
+
*,
|
|
98
|
+
scope: str,
|
|
99
|
+
name: str,
|
|
100
|
+
) -> bool:
|
|
101
|
+
"""
|
|
102
|
+
Adds an existing Rucio DID (Data Identifier) to the Opendata catalog.
|
|
103
|
+
|
|
104
|
+
Parameters:
|
|
105
|
+
scope: The scope under which the DID is registered.
|
|
106
|
+
name: The name of the DID.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
True if the DID was successfully added to the Opendata catalog, otherwise raises an exception.
|
|
110
|
+
|
|
111
|
+
Raises:
|
|
112
|
+
Exception: If the request fails or the server returns an error.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
path = '/'.join([self.opendata_private_dids_base_url, quote_plus(scope), quote_plus(name)])
|
|
116
|
+
url = build_url(self.get_opendata_host(public=False), path=path)
|
|
117
|
+
|
|
118
|
+
r = self._send_request(url, type_='POST')
|
|
119
|
+
|
|
120
|
+
if r.status_code == codes.created:
|
|
121
|
+
return True
|
|
122
|
+
else:
|
|
123
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
124
|
+
raise exc_cls(exc_msg)
|
|
125
|
+
|
|
126
|
+
def remove_opendata_did(
|
|
127
|
+
self,
|
|
128
|
+
*,
|
|
129
|
+
scope: str,
|
|
130
|
+
name: str,
|
|
131
|
+
) -> bool:
|
|
132
|
+
"""
|
|
133
|
+
Remove an existing Opendata DID from the Opendata catalog.
|
|
134
|
+
|
|
135
|
+
Parameters:
|
|
136
|
+
scope: The scope under which the DID is registered.
|
|
137
|
+
name: The name of the DID.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
True if the DID was successfully removed, otherwise raises an exception.
|
|
141
|
+
|
|
142
|
+
Raises:
|
|
143
|
+
Exception: If the request fails or the server returns an error.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
path = '/'.join([self.opendata_private_dids_base_url, quote_plus(scope), quote_plus(name)])
|
|
147
|
+
url = build_url(self.get_opendata_host(public=False), path=path)
|
|
148
|
+
|
|
149
|
+
r = self._send_request(url, type_='DEL')
|
|
150
|
+
|
|
151
|
+
if r.status_code == codes.no_content:
|
|
152
|
+
return True
|
|
153
|
+
else:
|
|
154
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
155
|
+
raise exc_cls(exc_msg)
|
|
156
|
+
|
|
157
|
+
def update_opendata_did(
|
|
158
|
+
self,
|
|
159
|
+
*,
|
|
160
|
+
scope: str,
|
|
161
|
+
name: str,
|
|
162
|
+
state: Optional["OPENDATA_DID_STATE_LITERAL"] = None,
|
|
163
|
+
meta: Optional[dict] = None,
|
|
164
|
+
doi: Optional[str] = None,
|
|
165
|
+
) -> bool:
|
|
166
|
+
"""
|
|
167
|
+
Update an existing Opendata DID in the Opendata catalog.
|
|
168
|
+
|
|
169
|
+
Parameters:
|
|
170
|
+
scope: The scope under which the DID is registered.
|
|
171
|
+
name: The name of the DID.
|
|
172
|
+
state: The new state to set for the DID.
|
|
173
|
+
meta: Metadata to update for the DID. Must be a valid JSON object.
|
|
174
|
+
doi: DOI to associate with the DID. Must be a valid DOI string (e.g., "10.1234/foo.bar").
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
True if the update was successful.
|
|
178
|
+
|
|
179
|
+
Raises:
|
|
180
|
+
ValueError: If none of 'meta', 'state', or 'doi' are provided.
|
|
181
|
+
Exception: If the request fails or the server returns an error.
|
|
182
|
+
"""
|
|
183
|
+
|
|
184
|
+
path = '/'.join([self.opendata_private_dids_base_url, quote_plus(scope), quote_plus(name)])
|
|
185
|
+
url = build_url(self.get_opendata_host(public=False), path=path)
|
|
186
|
+
|
|
187
|
+
if not any([meta, state, doi]):
|
|
188
|
+
raise ValueError("Either 'meta', 'state', or 'doi' must be provided.")
|
|
189
|
+
|
|
190
|
+
data: dict[str, Any] = {}
|
|
191
|
+
|
|
192
|
+
if meta is not None:
|
|
193
|
+
data['meta'] = meta
|
|
194
|
+
|
|
195
|
+
if state is not None:
|
|
196
|
+
data['state'] = state
|
|
197
|
+
|
|
198
|
+
if doi is not None:
|
|
199
|
+
data['doi'] = doi
|
|
200
|
+
|
|
201
|
+
r = self._send_request(url, type_='PUT', data=render_json(**data))
|
|
202
|
+
|
|
203
|
+
if r.status_code == codes.ok:
|
|
204
|
+
return True
|
|
205
|
+
else:
|
|
206
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
207
|
+
raise exc_cls(exc_msg)
|
|
208
|
+
|
|
209
|
+
def get_opendata_did(
|
|
210
|
+
self,
|
|
211
|
+
*,
|
|
212
|
+
scope: str,
|
|
213
|
+
name: str,
|
|
214
|
+
include_files: bool = False,
|
|
215
|
+
include_metadata: bool = False,
|
|
216
|
+
include_doi: bool = True,
|
|
217
|
+
public: bool = False,
|
|
218
|
+
) -> dict[str, Any]:
|
|
219
|
+
"""
|
|
220
|
+
Retrieve information about an OpenData DID (Data Identifier).
|
|
221
|
+
|
|
222
|
+
Parameters:
|
|
223
|
+
scope: The scope under which the DID is registered.
|
|
224
|
+
name: The name of the DID.
|
|
225
|
+
include_files: If True, include a list of associated files. Defaults to False.
|
|
226
|
+
include_metadata: If True, include extended metadata. Defaults to False.
|
|
227
|
+
include_doi: If True, include DOI (Digital Object Identifier) information. Defaults to True.
|
|
228
|
+
public: If True, only return data if the DID is publicly accessible. Defaults to False.
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
A dictionary containing metadata about the specified DID.
|
|
232
|
+
May include file list, extended metadata, and DOI details depending on the parameters.
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
base_url = self.opendata_public_dids_base_url if public else self.opendata_private_dids_base_url
|
|
236
|
+
path = '/'.join([base_url, quote_plus(scope), quote_plus(name)])
|
|
237
|
+
url = build_url(self.get_opendata_host(public=public), path=path)
|
|
238
|
+
|
|
239
|
+
r = self._send_request(url, type_='GET', params={
|
|
240
|
+
'files': 1 if include_files else 0,
|
|
241
|
+
'meta': 1 if include_metadata else 0,
|
|
242
|
+
'doi': 1 if include_doi else 0,
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
if r.status_code == codes.ok:
|
|
246
|
+
return json.loads(r.content.decode('utf-8'))
|
|
247
|
+
else:
|
|
248
|
+
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
249
|
+
raise exc_cls(exc_msg)
|