rucio-clients 37.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/__init__.py +17 -0
- rucio/alembicrevision.py +15 -0
- rucio/cli/__init__.py +14 -0
- rucio/cli/account.py +216 -0
- rucio/cli/bin_legacy/__init__.py +13 -0
- rucio/cli/bin_legacy/rucio.py +2825 -0
- rucio/cli/bin_legacy/rucio_admin.py +2500 -0
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/__init__.py +15 -0
- rucio/client/accountclient.py +432 -0
- rucio/client/accountlimitclient.py +183 -0
- rucio/client/baseclient.py +983 -0
- rucio/client/client.py +120 -0
- rucio/client/configclient.py +126 -0
- rucio/client/credentialclient.py +59 -0
- rucio/client/didclient.py +868 -0
- rucio/client/diracclient.py +56 -0
- rucio/client/downloadclient.py +1783 -0
- rucio/client/exportclient.py +44 -0
- rucio/client/fileclient.py +50 -0
- rucio/client/importclient.py +42 -0
- rucio/client/lifetimeclient.py +90 -0
- rucio/client/lockclient.py +109 -0
- rucio/client/metaconventionsclient.py +140 -0
- rucio/client/pingclient.py +44 -0
- rucio/client/replicaclient.py +452 -0
- rucio/client/requestclient.py +125 -0
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +746 -0
- rucio/client/ruleclient.py +294 -0
- rucio/client/scopeclient.py +90 -0
- rucio/client/subscriptionclient.py +173 -0
- rucio/client/touchclient.py +82 -0
- rucio/client/uploadclient.py +969 -0
- rucio/common/__init__.py +13 -0
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +111 -0
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +788 -0
- rucio/common/constants.py +217 -0
- rucio/common/constraints.py +17 -0
- rucio/common/didtype.py +237 -0
- rucio/common/exception.py +1208 -0
- rucio/common/extra.py +31 -0
- rucio/common/logging.py +420 -0
- rucio/common/pcache.py +1409 -0
- rucio/common/plugins.py +185 -0
- rucio/common/policy.py +93 -0
- rucio/common/schema/__init__.py +200 -0
- rucio/common/schema/generic.py +416 -0
- rucio/common/schema/generic_multi_vo.py +395 -0
- rucio/common/stomp_utils.py +423 -0
- rucio/common/stopwatch.py +55 -0
- rucio/common/test_rucio_server.py +154 -0
- rucio/common/types.py +483 -0
- rucio/common/utils.py +1688 -0
- rucio/rse/__init__.py +96 -0
- rucio/rse/protocols/__init__.py +13 -0
- rucio/rse/protocols/bittorrent.py +194 -0
- rucio/rse/protocols/cache.py +111 -0
- rucio/rse/protocols/dummy.py +100 -0
- rucio/rse/protocols/gfal.py +708 -0
- rucio/rse/protocols/globus.py +243 -0
- rucio/rse/protocols/http_cache.py +82 -0
- rucio/rse/protocols/mock.py +123 -0
- rucio/rse/protocols/ngarc.py +209 -0
- rucio/rse/protocols/posix.py +250 -0
- rucio/rse/protocols/protocol.py +361 -0
- rucio/rse/protocols/rclone.py +365 -0
- rucio/rse/protocols/rfio.py +145 -0
- rucio/rse/protocols/srm.py +338 -0
- rucio/rse/protocols/ssh.py +414 -0
- rucio/rse/protocols/storm.py +195 -0
- rucio/rse/protocols/webdav.py +594 -0
- rucio/rse/protocols/xrootd.py +302 -0
- rucio/rse/rsemanager.py +881 -0
- rucio/rse/translation.py +260 -0
- rucio/vcsversion.py +11 -0
- rucio/version.py +45 -0
- rucio_clients-37.0.0rc1.data/data/etc/rse-accounts.cfg.template +25 -0
- rucio_clients-37.0.0rc1.data/data/etc/rucio.cfg.atlas.client.template +43 -0
- rucio_clients-37.0.0rc1.data/data/etc/rucio.cfg.template +241 -0
- rucio_clients-37.0.0rc1.data/data/requirements.client.txt +19 -0
- rucio_clients-37.0.0rc1.data/data/rucio_client/merge_rucio_configs.py +144 -0
- rucio_clients-37.0.0rc1.data/scripts/rucio +133 -0
- rucio_clients-37.0.0rc1.data/scripts/rucio-admin +97 -0
- rucio_clients-37.0.0rc1.dist-info/METADATA +54 -0
- rucio_clients-37.0.0rc1.dist-info/RECORD +104 -0
- rucio_clients-37.0.0rc1.dist-info/WHEEL +5 -0
- rucio_clients-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
- rucio_clients-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
- rucio_clients-37.0.0rc1.dist-info/top_level.txt +1 -0
rucio/rse/translation.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
import hashlib
|
|
15
|
+
import importlib
|
|
16
|
+
import logging
|
|
17
|
+
from configparser import NoOptionError, NoSectionError
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
19
|
+
|
|
20
|
+
from rucio.common import config
|
|
21
|
+
from rucio.common.constants import RseAttr
|
|
22
|
+
from rucio.common.exception import ConfigNotFound
|
|
23
|
+
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from collections.abc import Callable, Mapping
|
|
27
|
+
|
|
28
|
+
from rucio.common.types import RSESettingsDict
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
|
|
32
|
+
"""
|
|
33
|
+
Translates a pfn dictionary into a scope and name
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_algorithm_type = "pfn2lfn"
|
|
37
|
+
|
|
38
|
+
def __init__(self, vo: str = 'def'):
|
|
39
|
+
super().__init__()
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
algorithm_name = config.config_get('policy', self._algorithm_type)
|
|
45
|
+
except (ConfigNotFound, NoOptionError, NoSectionError, RuntimeError):
|
|
46
|
+
logger.debug("PFN2LFN: no algorithm specified in the config.")
|
|
47
|
+
if super()._supports(self._algorithm_type, vo):
|
|
48
|
+
algorithm_name = vo
|
|
49
|
+
else:
|
|
50
|
+
algorithm_name = "def"
|
|
51
|
+
logger.debug("PFN2LFN: Falling back to %s algorithm.", 'default' if algorithm_name == 'def' else algorithm_name)
|
|
52
|
+
|
|
53
|
+
self.parser = self.get_parser(algorithm_name)
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def _module_init_(cls) -> None:
|
|
57
|
+
"""
|
|
58
|
+
Registers the included scope extraction algorithms
|
|
59
|
+
"""
|
|
60
|
+
cls.register(cls._default, "def")
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def get_parser(cls, algorithm_name: str) -> 'Callable[..., Any]':
|
|
64
|
+
return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def register(
|
|
68
|
+
cls,
|
|
69
|
+
pfn2lfn_callable: 'Callable',
|
|
70
|
+
name: Optional[str] = None
|
|
71
|
+
) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Provided a callable function, register it as one of the valid PFN2LFN algorithms.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
:param pfn2lfn_callable: Callable function to use.
|
|
77
|
+
:param name: Algorithm name used for registration.
|
|
78
|
+
"""
|
|
79
|
+
if name is None:
|
|
80
|
+
name = pfn2lfn_callable.__name__
|
|
81
|
+
algorithm_dict = {name: pfn2lfn_callable}
|
|
82
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
83
|
+
|
|
84
|
+
@staticmethod
|
|
85
|
+
def _default(parsed_pfn: 'Mapping[str, str]') -> tuple[str, str]:
|
|
86
|
+
""" Translate pfn to name/scope pair
|
|
87
|
+
|
|
88
|
+
:param parsed_pfn: dictionary representing pfn containing:
|
|
89
|
+
- path: str,
|
|
90
|
+
- name: str
|
|
91
|
+
:return: tuple containing name, scope
|
|
92
|
+
"""
|
|
93
|
+
path = parsed_pfn['path']
|
|
94
|
+
scope = path.lstrip('/').split('/')[0]
|
|
95
|
+
name = parsed_pfn['name']
|
|
96
|
+
return name, scope
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
RSEDeterministicScopeTranslation._module_init_() # pylint: disable=protected-access
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
103
|
+
"""
|
|
104
|
+
Execute the logic for translating a LFN to a path.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
_DEFAULT_LFN2PFN = "hash"
|
|
108
|
+
_algorithm_type = "lfn2pfn"
|
|
109
|
+
|
|
110
|
+
def __init__(
|
|
111
|
+
self,
|
|
112
|
+
rse: Optional[str] = None,
|
|
113
|
+
rse_attributes: Optional["RSESettingsDict"] = None,
|
|
114
|
+
protocol_attributes: Optional[dict[str, Any]] = None
|
|
115
|
+
):
|
|
116
|
+
"""
|
|
117
|
+
Initialize a translator object from the RSE, its attributes, and the protocol-specific
|
|
118
|
+
attributes.
|
|
119
|
+
|
|
120
|
+
:param rse: Name of RSE for this translation.
|
|
121
|
+
:param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
|
|
122
|
+
:param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
|
|
123
|
+
"""
|
|
124
|
+
super().__init__()
|
|
125
|
+
self.rse = rse
|
|
126
|
+
self.rse_attributes = rse_attributes if rse_attributes else {}
|
|
127
|
+
self.protocol_attributes = protocol_attributes if protocol_attributes else {}
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def supports(
|
|
131
|
+
cls,
|
|
132
|
+
name: str
|
|
133
|
+
) -> bool:
|
|
134
|
+
"""
|
|
135
|
+
Check to see if a specific algorithm is supported.
|
|
136
|
+
|
|
137
|
+
:param name: Name of the deterministic algorithm.
|
|
138
|
+
:returns: True if `name` is an algorithm supported by the translator class, False otherwise
|
|
139
|
+
"""
|
|
140
|
+
return super()._supports(cls._algorithm_type, name)
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def register(
|
|
144
|
+
cls,
|
|
145
|
+
lfn2pfn_callable: 'Callable',
|
|
146
|
+
name: Optional[str] = None
|
|
147
|
+
) -> None:
|
|
148
|
+
"""
|
|
149
|
+
Provided a callable function, register it as one of the valid LFN2PFN algorithms.
|
|
150
|
+
|
|
151
|
+
The callable will receive five arguments:
|
|
152
|
+
- scope: Scope of the LFN.
|
|
153
|
+
- name: LFN's path name
|
|
154
|
+
- rse: RSE name the translation is being done for.
|
|
155
|
+
- rse_attributes: Attributes of the RSE.
|
|
156
|
+
- protocol_attributes: Attributes of the RSE's protocol
|
|
157
|
+
The return value should be the last part of the PFN - it will be appended to the
|
|
158
|
+
rest of the URL.
|
|
159
|
+
|
|
160
|
+
:param lfn2pfn_callable: Callable function to use for generating paths.
|
|
161
|
+
:param name: Algorithm name used for registration. If None, then `lfn2pfn_callable.__name__` is used.
|
|
162
|
+
"""
|
|
163
|
+
if name is None:
|
|
164
|
+
name = lfn2pfn_callable.__name__
|
|
165
|
+
algorithm_dict = {name: lfn2pfn_callable}
|
|
166
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
def __hash(
|
|
170
|
+
scope: str,
|
|
171
|
+
name: str,
|
|
172
|
+
rse: str,
|
|
173
|
+
rse_attrs: dict[str, Any],
|
|
174
|
+
protocol_attrs: dict[str, Any]
|
|
175
|
+
) -> str:
|
|
176
|
+
"""
|
|
177
|
+
Given a LFN, turn it into a sub-directory structure using a hash function.
|
|
178
|
+
|
|
179
|
+
This takes the MD5 of the LFN and uses the first four characters as a subdirectory
|
|
180
|
+
name.
|
|
181
|
+
|
|
182
|
+
:param scope: Scope of the LFN.
|
|
183
|
+
:param name: File name of the LFN.
|
|
184
|
+
:param rse: RSE for PFN (ignored)
|
|
185
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
186
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
187
|
+
:returns: Path for use in the PFN generation.
|
|
188
|
+
"""
|
|
189
|
+
del rse
|
|
190
|
+
del rse_attrs
|
|
191
|
+
del protocol_attrs
|
|
192
|
+
hstr = hashlib.md5(('%s:%s' % (scope, name)).encode('utf-8')).hexdigest()
|
|
193
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
194
|
+
scope = scope.replace('.', '/')
|
|
195
|
+
return '%s/%s/%s/%s' % (scope, hstr[0:2], hstr[2:4], name)
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def __identity(
|
|
199
|
+
scope: str,
|
|
200
|
+
name: str,
|
|
201
|
+
rse: str,
|
|
202
|
+
rse_attrs: dict[str, Any],
|
|
203
|
+
protocol_attrs: dict[str, Any]
|
|
204
|
+
) -> str:
|
|
205
|
+
"""
|
|
206
|
+
Given a LFN, convert it directly to a path using the mapping:
|
|
207
|
+
|
|
208
|
+
scope:path -> scope/path
|
|
209
|
+
|
|
210
|
+
:param scope: Scope of the LFN.
|
|
211
|
+
:param name: File name of the LFN.
|
|
212
|
+
:param rse: RSE for PFN (ignored)
|
|
213
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
214
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
215
|
+
:returns: Path for use in the PFN generation.
|
|
216
|
+
"""
|
|
217
|
+
del rse
|
|
218
|
+
del rse_attrs
|
|
219
|
+
del protocol_attrs
|
|
220
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
221
|
+
scope = scope.replace('.', '/')
|
|
222
|
+
return '%s/%s' % (scope, name)
|
|
223
|
+
|
|
224
|
+
@classmethod
|
|
225
|
+
def _module_init_(cls) -> None:
|
|
226
|
+
"""
|
|
227
|
+
Initialize the class object on first module load.
|
|
228
|
+
"""
|
|
229
|
+
cls.register(cls.__hash, "hash")
|
|
230
|
+
cls.register(cls.__identity, "identity")
|
|
231
|
+
policy_module = None
|
|
232
|
+
try:
|
|
233
|
+
policy_module = config.config_get('policy', 'lfn2pfn_module')
|
|
234
|
+
except (ConfigNotFound, NoOptionError, NoSectionError):
|
|
235
|
+
pass
|
|
236
|
+
if policy_module:
|
|
237
|
+
importlib.import_module(policy_module)
|
|
238
|
+
|
|
239
|
+
cls._DEFAULT_LFN2PFN = config.get_lfn2pfn_algorithm_default()
|
|
240
|
+
|
|
241
|
+
def path(
|
|
242
|
+
self,
|
|
243
|
+
scope: str,
|
|
244
|
+
name: str
|
|
245
|
+
) -> str:
|
|
246
|
+
""" Transforms the logical file name into a PFN's path.
|
|
247
|
+
|
|
248
|
+
:param lfn: filename
|
|
249
|
+
:param scope: scope
|
|
250
|
+
|
|
251
|
+
:returns: RSE specific URI of the physical file
|
|
252
|
+
"""
|
|
253
|
+
algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
|
|
254
|
+
if algorithm == 'default':
|
|
255
|
+
algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
|
|
256
|
+
algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
|
|
257
|
+
return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
|
rucio/vcsversion.py
ADDED
rucio/version.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from rucio.vcsversion import VERSION_INFO
|
|
16
|
+
|
|
17
|
+
RUCIO_VERSION = [VERSION_INFO['version'], ]
|
|
18
|
+
FINAL = VERSION_INFO['final'] # This becomes true at Release Candidate time
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def canonical_version_string() -> str:
|
|
22
|
+
""" Get the canonical string """
|
|
23
|
+
return '.'.join(filter(None, RUCIO_VERSION))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def version_string() -> str:
|
|
27
|
+
""" Get the version string """
|
|
28
|
+
return canonical_version_string()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def vcs_version_string() -> str:
|
|
32
|
+
""" Get the VCS version string """
|
|
33
|
+
return "%s:%s" % (VERSION_INFO['branch_nick'], VERSION_INFO['revision_id'])
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def version_string_with_vcs() -> str:
|
|
37
|
+
""" Get the version string with VCS """
|
|
38
|
+
|
|
39
|
+
return "%s-%s" % (canonical_version_string(), vcs_version_string())
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def current_version() -> str:
|
|
43
|
+
""" Get the current version """
|
|
44
|
+
components = 2 if version_string().startswith("1.") else 1
|
|
45
|
+
return ".".join(version_string().split(".")[:components])
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"lxplus.cern.ch":{
|
|
3
|
+
"username":"USERNAME",
|
|
4
|
+
"password":"PASSWORD",
|
|
5
|
+
"host":"lxplus.cern.ch"
|
|
6
|
+
},
|
|
7
|
+
"swift.cern.ch": {
|
|
8
|
+
"access_key": "ACCESSKEY",
|
|
9
|
+
"secret_key": "PASSWORD",
|
|
10
|
+
"host_base": "swift.cern.ch",
|
|
11
|
+
"host_bucket": "swift.cern.ch",
|
|
12
|
+
"progress_meter": "False",
|
|
13
|
+
"skip_existing": "False"
|
|
14
|
+
},
|
|
15
|
+
"posix": {},
|
|
16
|
+
"DESY-ZN_SCRATCHDISK": {
|
|
17
|
+
"auth_type": "cert",
|
|
18
|
+
"timeout":300
|
|
19
|
+
},
|
|
20
|
+
"TAIWAN-LCG2_PPSSCRATCHDISK": {
|
|
21
|
+
"cert": ["/tmp/x509up","/tmp/x509up"],
|
|
22
|
+
"auth_type": "cert",
|
|
23
|
+
"timeout":300
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Copyright 2014-2021 CERN
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
# Authors:
|
|
16
|
+
# - Vincent Garonne <vincent.garonne@cern.ch>, 2014
|
|
17
|
+
# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
|
|
18
|
+
# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
|
|
19
|
+
# - Tomas Javurek <tomas.javurek@cern.ch>, 2019
|
|
20
|
+
# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
|
|
21
|
+
|
|
22
|
+
[common]
|
|
23
|
+
|
|
24
|
+
[client]
|
|
25
|
+
rucio_host = https://voatlasrucio-server-prod.cern.ch:443
|
|
26
|
+
auth_host = https://voatlasrucio-auth-prod.cern.ch:443
|
|
27
|
+
client_x509_proxy = $X509_USER_PROXY
|
|
28
|
+
request_retries = 3
|
|
29
|
+
auth_type = x509_proxy
|
|
30
|
+
|
|
31
|
+
[policy]
|
|
32
|
+
package = atlas_rucio_policy_package
|
|
33
|
+
#permission = atlas
|
|
34
|
+
#schema = atlas
|
|
35
|
+
lfn2pfn_algorithm_default = hash
|
|
36
|
+
support = hn-atlas-dist-analysis-help@cern.ch
|
|
37
|
+
support_rucio = https://github.com/rucio/rucio/issues/
|
|
38
|
+
|
|
39
|
+
[upload]
|
|
40
|
+
#transfer_timeout = 3600
|
|
41
|
+
|
|
42
|
+
[download]
|
|
43
|
+
#transfer_timeout = 3600
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# Copyright 2012-2022 CERN
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
#
|
|
15
|
+
# Authors:
|
|
16
|
+
# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
|
|
17
|
+
# - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2017
|
|
18
|
+
# - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2021
|
|
19
|
+
# - Luis Rodrigues <lfrodrigues@gmail.com>, 2013
|
|
20
|
+
# - Cedric Serfon <cedric.serfon@cern.ch>, 2013
|
|
21
|
+
# - Wen Guan <wen.guan@cern.ch>, 2014
|
|
22
|
+
# - Fernando López <felopez@cern.ch>, 2015
|
|
23
|
+
# - Martin Barisits <martin.barisits@cern.ch>, 2015-2021
|
|
24
|
+
# - Vitjan Zavrtanik <vitjan.zavrtanik@cern.ch>, 2017
|
|
25
|
+
# - Stefan Prenner <stefan.prenner@cern.ch>, 2018
|
|
26
|
+
# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
|
|
27
|
+
# - Frank Berghaus <berghaus@cern.ch>, 2019
|
|
28
|
+
# - Dilaksun Bavarajan <dilaksun.bavarajan@cern.ch>, 2019
|
|
29
|
+
# - James Perry <j.perry@epcc.ed.ac.uk>, 2019
|
|
30
|
+
# - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
|
|
31
|
+
# - Jaroslav Guenther <jaroslav.guenther@cern.ch>, 2019
|
|
32
|
+
# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
|
|
33
|
+
# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
|
|
34
|
+
# - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
|
|
35
|
+
# - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
|
|
36
|
+
# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
|
|
37
|
+
|
|
38
|
+
[common]
|
|
39
|
+
logdir = /var/log/rucio
|
|
40
|
+
loglevel = DEBUG
|
|
41
|
+
logformat = %%(asctime)s\t%%(process)d\t%%(levelname)s\t%%(message)s
|
|
42
|
+
mailtemplatedir=/opt/rucio/etc/mail_templates
|
|
43
|
+
|
|
44
|
+
[client]
|
|
45
|
+
rucio_host = https://rucio-server-prod.cern.ch:443
|
|
46
|
+
auth_host = https://rucio-auth-prod.cern.ch:443
|
|
47
|
+
auth_type = userpass
|
|
48
|
+
#auth_type = gss
|
|
49
|
+
#auth_type = x509_proxy
|
|
50
|
+
#auth_type = ssh
|
|
51
|
+
username = ddmlab
|
|
52
|
+
password = secret
|
|
53
|
+
ca_cert = /opt/rucio/etc/web/ca.crt
|
|
54
|
+
#client_cert = /opt/rucio/etc/web/client.crt
|
|
55
|
+
#client_key = /opt/rucio/etc/web/client.key
|
|
56
|
+
#client_x509_proxy = $X509_USER_PROXY
|
|
57
|
+
#ssh_private_key = $HOME/.ssh/id_rsa
|
|
58
|
+
account = root
|
|
59
|
+
request_retries = 3
|
|
60
|
+
protocol_stat_retries = 6
|
|
61
|
+
|
|
62
|
+
[upload]
|
|
63
|
+
#transfer_timeout = 3600
|
|
64
|
+
#preferred_impl = xrootd, rclone
|
|
65
|
+
|
|
66
|
+
[download]
|
|
67
|
+
#transfer_timeout = 3600
|
|
68
|
+
#preferred_impl = xrootd, rclone
|
|
69
|
+
|
|
70
|
+
[core]
|
|
71
|
+
geoip_licence_key = LICENCEKEYGOESHERE # Get a free licence key at https://www.maxmind.com/en/geolite2/signup
|
|
72
|
+
default_mail_from = spamspamspam@cern.ch
|
|
73
|
+
|
|
74
|
+
[database]
|
|
75
|
+
default = sqlite:////tmp/rucio.db
|
|
76
|
+
#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
|
|
77
|
+
#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
|
|
78
|
+
#schema=atlas_rucio # only for cern oracle
|
|
79
|
+
#default = mysql+pymysql://rucio:rucio@localhost/rucio
|
|
80
|
+
#default = postgresql+psycopg://rucio:rucio@localhost/rucio
|
|
81
|
+
pool_recycle=3600
|
|
82
|
+
echo=0
|
|
83
|
+
pool_reset_on_return=rollback
|
|
84
|
+
# Uncomment the following line to disable database connection pooling.
|
|
85
|
+
#poolclass = nullpool
|
|
86
|
+
|
|
87
|
+
[bootstrap]
|
|
88
|
+
# Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest()
|
|
89
|
+
userpass_identity = ddmlab
|
|
90
|
+
userpass_pwd = 2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583
|
|
91
|
+
userpass_email = ph-adp-ddm-lab@cern.ch
|
|
92
|
+
|
|
93
|
+
# Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt
|
|
94
|
+
x509_identity = emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH
|
|
95
|
+
x509_email = ph-adp-ddm-lab@cern.ch
|
|
96
|
+
|
|
97
|
+
# Default DDMLAB cern account
|
|
98
|
+
gss_identity = ddmlab@CERN.CH
|
|
99
|
+
gss_email = ph-adp-ddm-lab@cern.ch
|
|
100
|
+
|
|
101
|
+
# Default DDMLAB ssh key
|
|
102
|
+
ssh_identity = ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydnVy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab
|
|
103
|
+
ssh_email = ph-adp-ddm-lab@cern.ch
|
|
104
|
+
|
|
105
|
+
# Default DDMLAB SAML Name ID
|
|
106
|
+
saml_id = ddmlab
|
|
107
|
+
saml_email = ph-adp-ddm-lab@cern.ch
|
|
108
|
+
|
|
109
|
+
[monitor]
|
|
110
|
+
carbon_server = rucio-graphite-int.cern.ch
|
|
111
|
+
carbon_port = 8125
|
|
112
|
+
user_scope = your_username
|
|
113
|
+
enable_metrics = False
|
|
114
|
+
metrics_port = 8080
|
|
115
|
+
|
|
116
|
+
[conveyor]
|
|
117
|
+
scheme = srm,gsiftp,root,http,https
|
|
118
|
+
transfertool = fts3
|
|
119
|
+
ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
|
|
120
|
+
cacert = /opt/rucio/etc/web/ca.crt
|
|
121
|
+
usercert = /opt/rucio/tools/x509up
|
|
122
|
+
|
|
123
|
+
[messaging-fts3]
|
|
124
|
+
port = 61123
|
|
125
|
+
ssl_key_file = /home/mario/.ssh/hostkey.pem
|
|
126
|
+
ssl_cert_file = /home/mario/.ssh/hostcert.pem
|
|
127
|
+
destination = /topic/transfer.fts_monitoring_queue_state
|
|
128
|
+
brokers = dashb-test-mb.cern.ch
|
|
129
|
+
voname = atlas
|
|
130
|
+
|
|
131
|
+
[messaging-hermes]
|
|
132
|
+
username = ____
|
|
133
|
+
password = ____
|
|
134
|
+
port = 61023
|
|
135
|
+
nonssl_port = 61013
|
|
136
|
+
use_ssl = True
|
|
137
|
+
ssl_key_file = /etc/grid-security/hostkey.pem
|
|
138
|
+
ssl_cert_file = /etc/grid-security/hostcert.pem
|
|
139
|
+
destination = /topic/rucio.events
|
|
140
|
+
brokers = localhost
|
|
141
|
+
voname = atlas
|
|
142
|
+
email_from = Rucio <spamspamspam@cern.ch>
|
|
143
|
+
email_test = spamspamspam@cern.ch
|
|
144
|
+
|
|
145
|
+
[transmogrifier]
|
|
146
|
+
maxdids = 100000
|
|
147
|
+
|
|
148
|
+
[accounts]
|
|
149
|
+
# These are accounts that can write into scopes owned by another account
|
|
150
|
+
special_accounts = panda, tier0
|
|
151
|
+
|
|
152
|
+
[trace]
|
|
153
|
+
tracedir = /var/log/rucio/trace
|
|
154
|
+
brokers=localhost
|
|
155
|
+
port=61013
|
|
156
|
+
username = _________
|
|
157
|
+
password = _________
|
|
158
|
+
topic = /topic/rucio.tracer
|
|
159
|
+
trace_host = https://rucio-server-prod.cern.ch:443
|
|
160
|
+
|
|
161
|
+
[tracer-kronos]
|
|
162
|
+
brokers=localhost
|
|
163
|
+
port=61013
|
|
164
|
+
ssl_key_file = /etc/grid-security/hostkey.pem
|
|
165
|
+
ssl_cert_file = /etc/grid-security/hostcert.pem
|
|
166
|
+
queue = /queue/Consumer.kronos.rucio.tracer
|
|
167
|
+
prefetch_size = 10
|
|
168
|
+
chunksize = 10
|
|
169
|
+
subscription_id = rucio-tracer-listener
|
|
170
|
+
use_ssl = False
|
|
171
|
+
reconnect_attempts = 100
|
|
172
|
+
excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch
|
|
173
|
+
username = _________
|
|
174
|
+
password = _________
|
|
175
|
+
dataset_wait = 60
|
|
176
|
+
|
|
177
|
+
[injector]
|
|
178
|
+
file = /opt/rucio/tools/test.file.1000
|
|
179
|
+
bytes = 1000
|
|
180
|
+
md5 = fd21ce524a9e45060fd3f62c4ef6a386
|
|
181
|
+
adler32 = 52590737
|
|
182
|
+
#file = /opt/rucio/tools/test.file.1G
|
|
183
|
+
#bytes = 1000000000
|
|
184
|
+
#md5sum = 1a3031f43cc0c0cf4f8341920a9faf52
|
|
185
|
+
#adler32 = 5ff2d417
|
|
186
|
+
|
|
187
|
+
[alembic]
|
|
188
|
+
cfg = /opt/rucio/etc/alembic.ini
|
|
189
|
+
|
|
190
|
+
[messaging-cache]
|
|
191
|
+
port = 61023
|
|
192
|
+
ssl_key_file = /etc/grid-security/hostkey.pem
|
|
193
|
+
ssl_cert_file = /etc/grid-security/hostcert.pem
|
|
194
|
+
destination = /topic/rucio.fax
|
|
195
|
+
brokers = localhost
|
|
196
|
+
voname = atlas
|
|
197
|
+
account = cache_mb
|
|
198
|
+
|
|
199
|
+
[test]
|
|
200
|
+
cacert = /opt/rucio/etc/web/ca.crt
|
|
201
|
+
#cacert = /etc/pki/tls/certs/CERN-bundle.pem
|
|
202
|
+
usercert = /opt/rucio/etc/web/usercert.pem
|
|
203
|
+
#usercert = /home/mario/.ssh/usercert_with_key.pem
|
|
204
|
+
|
|
205
|
+
[nagios]
|
|
206
|
+
proxy = /opt/rucio/etc/ddmadmin.proxy.nagios
|
|
207
|
+
rfcproxy = /opt/rucio/etc/ddmusr01.rfc.proxy
|
|
208
|
+
fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446, https://fts3.cern.ch:8446, https://lcgfts3.gridpp.rl.ac.uk:8446, https://fts3-test.gridpp.rl.ac.uk:8446, https://fts.usatlas.bnl.gov:8446
|
|
209
|
+
|
|
210
|
+
[auditor]
|
|
211
|
+
cache = /opt/rucio/auditor-cache
|
|
212
|
+
results = /opt/rucio/auditor-results
|
|
213
|
+
|
|
214
|
+
[policy]
|
|
215
|
+
package = atlas_rucio_policy_package
|
|
216
|
+
permission = atlas
|
|
217
|
+
schema = atlas
|
|
218
|
+
extract_scope = atlas
|
|
219
|
+
lfn2pfn_algorithm_default = hash
|
|
220
|
+
support = hn-atlas-dist-analysis-help@cern.ch
|
|
221
|
+
support_rucio = https://github.com/rucio/rucio/issues/
|
|
222
|
+
|
|
223
|
+
[webui]
|
|
224
|
+
usercert = /opt/rucio/etc/usercert_with_key.pem
|
|
225
|
+
|
|
226
|
+
[credentials]
|
|
227
|
+
gcs = /opt/rucio/etc/google-cloud-storage-test.json
|
|
228
|
+
signature_lifetime = 3600
|
|
229
|
+
|
|
230
|
+
[saml]
|
|
231
|
+
config_path = /opt/rucio/lib/rucio/web/ui/common/saml/
|
|
232
|
+
|
|
233
|
+
[oidc]
|
|
234
|
+
idpsecrets = /opt/rucio/etc/idpsecrets.json
|
|
235
|
+
admin_issuer = wlcg
|
|
236
|
+
|
|
237
|
+
[api]
|
|
238
|
+
#endpoints = accountlimits, accounts, config, credentials, dids, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions
|
|
239
|
+
|
|
240
|
+
[transfers]
|
|
241
|
+
srm_https_compatibility = False
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# All dependencies needed to run rucio client should be defined here
|
|
2
|
+
requests>=2.32.3 # Python HTTP for Humans.
|
|
3
|
+
urllib3>=2.3.0 # HTTP library with thread-safe connection pooling, file post, etc.
|
|
4
|
+
dogpile-cache>=1.2.2 # Caching API plugins (1.1.2 is the first version to support pymemcache)
|
|
5
|
+
tabulate>=0.9.0 # Pretty-print tabular data
|
|
6
|
+
jsonschema>=4.23.0 # For JSON schema validation (Policy modules)
|
|
7
|
+
packaging>=24.2 # Packaging utilities
|
|
8
|
+
rich>=13.9.4 # For Rich terminal display
|
|
9
|
+
typing-extensions>=4.12.2 # Type annotations that are not yet supported in the `typing` module
|
|
10
|
+
click>=8.1.7 # CLI engine
|
|
11
|
+
|
|
12
|
+
# All dependencies needed in extras for rucio client should be defined here
|
|
13
|
+
paramiko>=3.5.1 # ssh_extras; SSH2 protocol library (also needed in the server)
|
|
14
|
+
kerberos>=1.3.1 # kerberos_extras for client and server
|
|
15
|
+
pykerberos>=1.2.4 # kerberos_extras for client and server
|
|
16
|
+
requests-kerberos>=0.15.0 # kerberos_extras for client and server
|
|
17
|
+
python-swiftclient>=4.7.0 # swift_extras
|
|
18
|
+
argcomplete>=3.5.3 # argcomplete_extras; Bash tab completion for argparse
|
|
19
|
+
python-magic>=0.4.27 # dumper_extras; File type identification using libmagic
|