rucio-clients 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/cli/__init__.py +14 -0
- rucio/cli/account.py +216 -0
- rucio/cli/bin_legacy/__init__.py +13 -0
- rucio_clients-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio_clients-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rucio.cfg.template +3 -19
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/requirements.client.txt +11 -7
- rucio_clients-37.0.0.data/scripts/rucio +133 -0
- rucio_clients-37.0.0.data/scripts/rucio-admin +97 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/METADATA +18 -14
- rucio_clients-37.0.0.dist-info/RECORD +104 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio_clients-35.7.0.dist-info/RECORD +0 -88
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/rucio_client/merge_rucio_configs.py +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/WHEEL +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/licenses/LICENSE +0 -0
- {rucio_clients-35.7.0.dist-info → rucio_clients-37.0.0.dist-info}/top_level.txt +0 -0
rucio/rse/translation.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
import hashlib
|
|
15
|
+
import importlib
|
|
16
|
+
import logging
|
|
17
|
+
from configparser import NoOptionError, NoSectionError
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
19
|
+
|
|
20
|
+
from rucio.common import config
|
|
21
|
+
from rucio.common.constants import RseAttr
|
|
22
|
+
from rucio.common.exception import ConfigNotFound
|
|
23
|
+
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from collections.abc import Callable, Mapping
|
|
27
|
+
|
|
28
|
+
from rucio.common.types import RSESettingsDict
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
|
|
32
|
+
"""
|
|
33
|
+
Translates a pfn dictionary into a scope and name
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_algorithm_type = "pfn2lfn"
|
|
37
|
+
|
|
38
|
+
def __init__(self, vo: str = 'def'):
|
|
39
|
+
super().__init__()
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
algorithm_name = config.config_get('policy', self._algorithm_type)
|
|
45
|
+
except (ConfigNotFound, NoOptionError, NoSectionError, RuntimeError):
|
|
46
|
+
logger.debug("PFN2LFN: no algorithm specified in the config.")
|
|
47
|
+
if super()._supports(self._algorithm_type, vo):
|
|
48
|
+
algorithm_name = vo
|
|
49
|
+
else:
|
|
50
|
+
algorithm_name = "def"
|
|
51
|
+
logger.debug("PFN2LFN: Falling back to %s algorithm.", 'default' if algorithm_name == 'def' else algorithm_name)
|
|
52
|
+
|
|
53
|
+
self.parser = self.get_parser(algorithm_name)
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def _module_init_(cls) -> None:
|
|
57
|
+
"""
|
|
58
|
+
Registers the included scope extraction algorithms
|
|
59
|
+
"""
|
|
60
|
+
cls.register(cls._default, "def")
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def get_parser(cls, algorithm_name: str) -> 'Callable[..., Any]':
|
|
64
|
+
return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def register(
|
|
68
|
+
cls,
|
|
69
|
+
pfn2lfn_callable: 'Callable',
|
|
70
|
+
name: Optional[str] = None
|
|
71
|
+
) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Provided a callable function, register it as one of the valid PFN2LFN algorithms.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
:param pfn2lfn_callable: Callable function to use.
|
|
77
|
+
:param name: Algorithm name used for registration.
|
|
78
|
+
"""
|
|
79
|
+
if name is None:
|
|
80
|
+
name = pfn2lfn_callable.__name__
|
|
81
|
+
algorithm_dict = {name: pfn2lfn_callable}
|
|
82
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
83
|
+
|
|
84
|
+
@staticmethod
|
|
85
|
+
def _default(parsed_pfn: 'Mapping[str, str]') -> tuple[str, str]:
|
|
86
|
+
""" Translate pfn to name/scope pair
|
|
87
|
+
|
|
88
|
+
:param parsed_pfn: dictionary representing pfn containing:
|
|
89
|
+
- path: str,
|
|
90
|
+
- name: str
|
|
91
|
+
:return: tuple containing name, scope
|
|
92
|
+
"""
|
|
93
|
+
path = parsed_pfn['path']
|
|
94
|
+
scope = path.lstrip('/').split('/')[0]
|
|
95
|
+
name = parsed_pfn['name']
|
|
96
|
+
return name, scope
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
RSEDeterministicScopeTranslation._module_init_() # pylint: disable=protected-access
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
103
|
+
"""
|
|
104
|
+
Execute the logic for translating a LFN to a path.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
_DEFAULT_LFN2PFN = "hash"
|
|
108
|
+
_algorithm_type = "lfn2pfn"
|
|
109
|
+
|
|
110
|
+
def __init__(
|
|
111
|
+
self,
|
|
112
|
+
rse: Optional[str] = None,
|
|
113
|
+
rse_attributes: Optional["RSESettingsDict"] = None,
|
|
114
|
+
protocol_attributes: Optional[dict[str, Any]] = None
|
|
115
|
+
):
|
|
116
|
+
"""
|
|
117
|
+
Initialize a translator object from the RSE, its attributes, and the protocol-specific
|
|
118
|
+
attributes.
|
|
119
|
+
|
|
120
|
+
:param rse: Name of RSE for this translation.
|
|
121
|
+
:param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
|
|
122
|
+
:param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
|
|
123
|
+
"""
|
|
124
|
+
super().__init__()
|
|
125
|
+
self.rse = rse
|
|
126
|
+
self.rse_attributes = rse_attributes if rse_attributes else {}
|
|
127
|
+
self.protocol_attributes = protocol_attributes if protocol_attributes else {}
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def supports(
|
|
131
|
+
cls,
|
|
132
|
+
name: str
|
|
133
|
+
) -> bool:
|
|
134
|
+
"""
|
|
135
|
+
Check to see if a specific algorithm is supported.
|
|
136
|
+
|
|
137
|
+
:param name: Name of the deterministic algorithm.
|
|
138
|
+
:returns: True if `name` is an algorithm supported by the translator class, False otherwise
|
|
139
|
+
"""
|
|
140
|
+
return super()._supports(cls._algorithm_type, name)
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def register(
|
|
144
|
+
cls,
|
|
145
|
+
lfn2pfn_callable: 'Callable',
|
|
146
|
+
name: Optional[str] = None
|
|
147
|
+
) -> None:
|
|
148
|
+
"""
|
|
149
|
+
Provided a callable function, register it as one of the valid LFN2PFN algorithms.
|
|
150
|
+
|
|
151
|
+
The callable will receive five arguments:
|
|
152
|
+
- scope: Scope of the LFN.
|
|
153
|
+
- name: LFN's path name
|
|
154
|
+
- rse: RSE name the translation is being done for.
|
|
155
|
+
- rse_attributes: Attributes of the RSE.
|
|
156
|
+
- protocol_attributes: Attributes of the RSE's protocol
|
|
157
|
+
The return value should be the last part of the PFN - it will be appended to the
|
|
158
|
+
rest of the URL.
|
|
159
|
+
|
|
160
|
+
:param lfn2pfn_callable: Callable function to use for generating paths.
|
|
161
|
+
:param name: Algorithm name used for registration. If None, then `lfn2pfn_callable.__name__` is used.
|
|
162
|
+
"""
|
|
163
|
+
if name is None:
|
|
164
|
+
name = lfn2pfn_callable.__name__
|
|
165
|
+
algorithm_dict = {name: lfn2pfn_callable}
|
|
166
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
def __hash(
|
|
170
|
+
scope: str,
|
|
171
|
+
name: str,
|
|
172
|
+
rse: str,
|
|
173
|
+
rse_attrs: dict[str, Any],
|
|
174
|
+
protocol_attrs: dict[str, Any]
|
|
175
|
+
) -> str:
|
|
176
|
+
"""
|
|
177
|
+
Given a LFN, turn it into a sub-directory structure using a hash function.
|
|
178
|
+
|
|
179
|
+
This takes the MD5 of the LFN and uses the first four characters as a subdirectory
|
|
180
|
+
name.
|
|
181
|
+
|
|
182
|
+
:param scope: Scope of the LFN.
|
|
183
|
+
:param name: File name of the LFN.
|
|
184
|
+
:param rse: RSE for PFN (ignored)
|
|
185
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
186
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
187
|
+
:returns: Path for use in the PFN generation.
|
|
188
|
+
"""
|
|
189
|
+
del rse
|
|
190
|
+
del rse_attrs
|
|
191
|
+
del protocol_attrs
|
|
192
|
+
hstr = hashlib.md5(('%s:%s' % (scope, name)).encode('utf-8')).hexdigest()
|
|
193
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
194
|
+
scope = scope.replace('.', '/')
|
|
195
|
+
return '%s/%s/%s/%s' % (scope, hstr[0:2], hstr[2:4], name)
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def __identity(
|
|
199
|
+
scope: str,
|
|
200
|
+
name: str,
|
|
201
|
+
rse: str,
|
|
202
|
+
rse_attrs: dict[str, Any],
|
|
203
|
+
protocol_attrs: dict[str, Any]
|
|
204
|
+
) -> str:
|
|
205
|
+
"""
|
|
206
|
+
Given a LFN, convert it directly to a path using the mapping:
|
|
207
|
+
|
|
208
|
+
scope:path -> scope/path
|
|
209
|
+
|
|
210
|
+
:param scope: Scope of the LFN.
|
|
211
|
+
:param name: File name of the LFN.
|
|
212
|
+
:param rse: RSE for PFN (ignored)
|
|
213
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
214
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
215
|
+
:returns: Path for use in the PFN generation.
|
|
216
|
+
"""
|
|
217
|
+
del rse
|
|
218
|
+
del rse_attrs
|
|
219
|
+
del protocol_attrs
|
|
220
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
221
|
+
scope = scope.replace('.', '/')
|
|
222
|
+
return '%s/%s' % (scope, name)
|
|
223
|
+
|
|
224
|
+
@classmethod
|
|
225
|
+
def _module_init_(cls) -> None:
|
|
226
|
+
"""
|
|
227
|
+
Initialize the class object on first module load.
|
|
228
|
+
"""
|
|
229
|
+
cls.register(cls.__hash, "hash")
|
|
230
|
+
cls.register(cls.__identity, "identity")
|
|
231
|
+
policy_module = None
|
|
232
|
+
try:
|
|
233
|
+
policy_module = config.config_get('policy', 'lfn2pfn_module')
|
|
234
|
+
except (ConfigNotFound, NoOptionError, NoSectionError):
|
|
235
|
+
pass
|
|
236
|
+
if policy_module:
|
|
237
|
+
importlib.import_module(policy_module)
|
|
238
|
+
|
|
239
|
+
cls._DEFAULT_LFN2PFN = config.get_lfn2pfn_algorithm_default()
|
|
240
|
+
|
|
241
|
+
def path(
|
|
242
|
+
self,
|
|
243
|
+
scope: str,
|
|
244
|
+
name: str
|
|
245
|
+
) -> str:
|
|
246
|
+
""" Transforms the logical file name into a PFN's path.
|
|
247
|
+
|
|
248
|
+
:param lfn: filename
|
|
249
|
+
:param scope: scope
|
|
250
|
+
|
|
251
|
+
:returns: RSE specific URI of the physical file
|
|
252
|
+
"""
|
|
253
|
+
algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
|
|
254
|
+
if algorithm == 'default':
|
|
255
|
+
algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
|
|
256
|
+
algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
|
|
257
|
+
return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
|
rucio/vcsversion.py
CHANGED
|
@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
|
|
|
4
4
|
'''
|
|
5
5
|
VERSION_INFO = {
|
|
6
6
|
'final': True,
|
|
7
|
-
'version': '
|
|
8
|
-
'branch_nick': '
|
|
9
|
-
'revision_id': '
|
|
10
|
-
'revno':
|
|
7
|
+
'version': '37.0.0',
|
|
8
|
+
'branch_nick': 'master',
|
|
9
|
+
'revision_id': '4618b6bc894b3f8554115a6ff58bea415295ff8e',
|
|
10
|
+
'revno': 13616
|
|
11
11
|
}
|
rucio/version.py
CHANGED
|
@@ -35,4 +35,11 @@ def vcs_version_string() -> str:
|
|
|
35
35
|
|
|
36
36
|
def version_string_with_vcs() -> str:
|
|
37
37
|
""" Get the version string with VCS """
|
|
38
|
+
|
|
38
39
|
return "%s-%s" % (canonical_version_string(), vcs_version_string())
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def current_version() -> str:
|
|
43
|
+
""" Get the current version """
|
|
44
|
+
components = 2 if version_string().startswith("1.") else 1
|
|
45
|
+
return ".".join(version_string().split(".")[:components])
|
{rucio_clients-35.7.0.data → rucio_clients-37.0.0.data}/data/etc/rucio.cfg.atlas.client.template
RENAMED
|
@@ -29,8 +29,9 @@ request_retries = 3
|
|
|
29
29
|
auth_type = x509_proxy
|
|
30
30
|
|
|
31
31
|
[policy]
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
package = atlas_rucio_policy_package
|
|
33
|
+
#permission = atlas
|
|
34
|
+
#schema = atlas
|
|
34
35
|
lfn2pfn_algorithm_default = hash
|
|
35
36
|
support = hn-atlas-dist-analysis-help@cern.ch
|
|
36
37
|
support_rucio = https://github.com/rucio/rucio/issues/
|
|
@@ -77,7 +77,7 @@ default = sqlite:////tmp/rucio.db
|
|
|
77
77
|
#default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
|
|
78
78
|
#schema=atlas_rucio # only for cern oracle
|
|
79
79
|
#default = mysql+pymysql://rucio:rucio@localhost/rucio
|
|
80
|
-
#default = postgresql://rucio:rucio@localhost/rucio
|
|
80
|
+
#default = postgresql+psycopg://rucio:rucio@localhost/rucio
|
|
81
81
|
pool_recycle=3600
|
|
82
82
|
echo=0
|
|
83
83
|
pool_reset_on_return=rollback
|
|
@@ -211,27 +211,11 @@ fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446,
|
|
|
211
211
|
cache = /opt/rucio/auditor-cache
|
|
212
212
|
results = /opt/rucio/auditor-results
|
|
213
213
|
|
|
214
|
-
[c3po]
|
|
215
|
-
placement_algorithm = t2_free_space
|
|
216
|
-
elastic_url = http://aianalytics01.cern.ch:9200
|
|
217
|
-
redis_host = localhost
|
|
218
|
-
redis_port = 6379
|
|
219
|
-
|
|
220
|
-
[c3po-popularity]
|
|
221
|
-
elastic_url = http://rucio-logger-prod-01.cern.ch:9200
|
|
222
|
-
|
|
223
|
-
[c3po-site-mapper]
|
|
224
|
-
panda_url = http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json
|
|
225
|
-
ddm_url = http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json
|
|
226
|
-
|
|
227
|
-
[c3po-workload]
|
|
228
|
-
panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
|
|
229
|
-
window = 604800
|
|
230
|
-
|
|
231
214
|
[policy]
|
|
232
|
-
package =
|
|
215
|
+
package = atlas_rucio_policy_package
|
|
233
216
|
permission = atlas
|
|
234
217
|
schema = atlas
|
|
218
|
+
extract_scope = atlas
|
|
235
219
|
lfn2pfn_algorithm_default = hash
|
|
236
220
|
support = hn-atlas-dist-analysis-help@cern.ch
|
|
237
221
|
support_rucio = https://github.com/rucio/rucio/issues/
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
# All dependencies needed to run rucio client should be defined here
|
|
2
|
-
requests>=2.32.
|
|
3
|
-
urllib3>=
|
|
2
|
+
requests>=2.32.3 # Python HTTP for Humans.
|
|
3
|
+
urllib3>=2.3.0 # HTTP library with thread-safe connection pooling, file post, etc.
|
|
4
4
|
dogpile-cache>=1.2.2 # Caching API plugins (1.1.2 is the first version to support pymemcache)
|
|
5
5
|
tabulate>=0.9.0 # Pretty-print tabular data
|
|
6
|
-
jsonschema>=4.
|
|
6
|
+
jsonschema>=4.23.0 # For JSON schema validation (Policy modules)
|
|
7
|
+
packaging>=24.2 # Packaging utilities
|
|
8
|
+
rich>=13.9.4 # For Rich terminal display
|
|
9
|
+
typing-extensions>=4.12.2 # Type annotations that are not yet supported in the `typing` module
|
|
10
|
+
click>=8.1.7 # CLI engine
|
|
7
11
|
|
|
8
12
|
# All dependencies needed in extras for rucio client should be defined here
|
|
9
|
-
paramiko>=3.
|
|
13
|
+
paramiko>=3.5.1 # ssh_extras; SSH2 protocol library (also needed in the server)
|
|
10
14
|
kerberos>=1.3.1 # kerberos_extras for client and server
|
|
11
15
|
pykerberos>=1.2.4 # kerberos_extras for client and server
|
|
12
|
-
requests-kerberos>=0.
|
|
13
|
-
python-swiftclient>=4.
|
|
14
|
-
argcomplete>=3.
|
|
16
|
+
requests-kerberos>=0.15.0 # kerberos_extras for client and server
|
|
17
|
+
python-swiftclient>=4.7.0 # swift_extras
|
|
18
|
+
argcomplete>=3.5.3 # argcomplete_extras; Bash tab completion for argparse
|
|
15
19
|
python-magic>=0.4.27 # dumper_extras; File type identification using libmagic
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
#!python
|
|
2
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import sys
|
|
18
|
+
from typing import TYPE_CHECKING, Optional
|
|
19
|
+
|
|
20
|
+
from rucio.cli.bin_legacy.rucio import main as main_legacy
|
|
21
|
+
from rucio.cli.command import main
|
|
22
|
+
from rucio.common.utils import setup_logger
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from logging import Logger
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _get_first_command(args: list[str]) -> Optional[str]:
|
|
29
|
+
return next(
|
|
30
|
+
(
|
|
31
|
+
arg for arg in args
|
|
32
|
+
if arg != 'rucio' and (arg[0] != "-" or arg in ("-h", "--help", "--version"))
|
|
33
|
+
),
|
|
34
|
+
None
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def make_warning(logger: "Logger") -> None:
|
|
39
|
+
base_warning = "This method is being deprecated."
|
|
40
|
+
new_command = map_legacy_command()
|
|
41
|
+
if new_command is not None:
|
|
42
|
+
warning = f"{base_warning} Please replace your command with `rucio {' '.join(new_command)}`"
|
|
43
|
+
else:
|
|
44
|
+
warning = base_warning + " Please view rucio -h for an updated help menu."
|
|
45
|
+
|
|
46
|
+
logger.warning(warning)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def map_legacy_command() -> Optional[list[str]]:
|
|
50
|
+
command = _get_first_command(sys.argv[1:])
|
|
51
|
+
|
|
52
|
+
new_command = None
|
|
53
|
+
if command not in ("-h", "--version", None):
|
|
54
|
+
command_map = {
|
|
55
|
+
"list-file-replicas": ["replica", "list", "file"],
|
|
56
|
+
"list-dataset-replicas": ["replica", "list", "dataset"],
|
|
57
|
+
"add-dataset": ["did", "add", "--type dataset"],
|
|
58
|
+
"add-container": ["did", "add", "--type container"],
|
|
59
|
+
"attach": ["did", "content", "add"],
|
|
60
|
+
"detach": ["did", "content", "remove"],
|
|
61
|
+
"ls": ["did", "list"],
|
|
62
|
+
"list-dids": ["did", "list"],
|
|
63
|
+
"list-parent-dids": ["did", "list", "--parent"],
|
|
64
|
+
"list-scopes": ["scope", "list"],
|
|
65
|
+
"close": ["did", "update", "--close"],
|
|
66
|
+
"reopen": ["did", "update", "--open"],
|
|
67
|
+
"stat": ["did", "show"],
|
|
68
|
+
"erase": ["did", "remove"],
|
|
69
|
+
"list-content": ["did", "content", "list"],
|
|
70
|
+
"list-content-history": ["did", "content", "history"],
|
|
71
|
+
"upload": ["upload"],
|
|
72
|
+
"get": ["download"],
|
|
73
|
+
"download": ["download"],
|
|
74
|
+
"get-metadata": ["did", "metadata", "list"],
|
|
75
|
+
"set-metadata": ["did", "metadata", "add"],
|
|
76
|
+
"delete-metadata": ["did", "metadata", "remove"],
|
|
77
|
+
"list-rse-usage": ["rse", "show"],
|
|
78
|
+
"list-account-usage": ["account", "limit", "list"],
|
|
79
|
+
"list-account-limits": ["account", "limit", "list"],
|
|
80
|
+
"add-rule": ["rule", "add"],
|
|
81
|
+
"delete-rule": ["rule", "remove"],
|
|
82
|
+
"rule-info": ["rule", "show"],
|
|
83
|
+
"list-rules": ["rule", "list"],
|
|
84
|
+
"list-rules-history": ["rule", "history"],
|
|
85
|
+
"update-rule": ["rule", "update"],
|
|
86
|
+
"move-rule": ["rule", "move"],
|
|
87
|
+
"list-rses": ["rse", "list"],
|
|
88
|
+
"list-suspicious-replicas": ["replica", "state", "suspicious"],
|
|
89
|
+
"list-rse-attributes": ["rse", "attribute", "list"],
|
|
90
|
+
"touch": ["did", "update", "--touch"],
|
|
91
|
+
"add-lifetime-exception": ["lifetime-exception", "add"],
|
|
92
|
+
}
|
|
93
|
+
new_command = command_map.get(command)
|
|
94
|
+
|
|
95
|
+
return new_command
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__":
|
|
99
|
+
commands = ("account", "config", "did", "replica", "rse", "rule", "scope", "subscription", "ping", "whoami", "test-server", "lifetime-exception", "upload", "download")
|
|
100
|
+
|
|
101
|
+
parser = argparse.ArgumentParser(add_help=False)
|
|
102
|
+
# Check for legacy flag
|
|
103
|
+
parser.add_argument("--legacy", action="store_true")
|
|
104
|
+
# Check for commands in the new command list
|
|
105
|
+
parser.add_argument("-h", "--help", action="store_true")
|
|
106
|
+
parser.add_argument("--version", action="store_true")
|
|
107
|
+
|
|
108
|
+
args, _ = parser.parse_known_args()
|
|
109
|
+
|
|
110
|
+
logger = setup_logger(module_name=__name__)
|
|
111
|
+
|
|
112
|
+
if args.legacy:
|
|
113
|
+
make_warning(logger)
|
|
114
|
+
sys.argv.pop(sys.argv.index('--legacy'))
|
|
115
|
+
main_legacy()
|
|
116
|
+
|
|
117
|
+
elif (any(arg in commands for arg in sys.argv)) or args.help or args.version:
|
|
118
|
+
main() # pylint: disable=E1120
|
|
119
|
+
|
|
120
|
+
else:
|
|
121
|
+
make_warning(logger)
|
|
122
|
+
try:
|
|
123
|
+
main_legacy()
|
|
124
|
+
# Make a custom warning - show the new help menu when invalid commands are called.
|
|
125
|
+
except argparse.ArgumentError:
|
|
126
|
+
logger.error("Invalid argument(s) - %s " % sys.argv[1:])
|
|
127
|
+
command = map_legacy_command()
|
|
128
|
+
if command is not None:
|
|
129
|
+
sys.argv = ["rucio"] + command + ["-h"]
|
|
130
|
+
else:
|
|
131
|
+
sys.argv = ["rucio", "-h"]
|
|
132
|
+
|
|
133
|
+
main() # pylint: disable=E1120
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
#!python
|
|
2
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
import sys
|
|
17
|
+
|
|
18
|
+
from rucio.cli.bin_legacy.rucio_admin import main as main_legacy
|
|
19
|
+
from rucio.common.utils import setup_logger
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def make_warning():
|
|
23
|
+
logger = setup_logger(module_name=__name__)
|
|
24
|
+
|
|
25
|
+
base_warning = "This method is being deprecated."
|
|
26
|
+
args = [arg for arg in sys.argv if arg[0] != "-" or arg in ("-h", "--help", "--version")]
|
|
27
|
+
try:
|
|
28
|
+
first_command = args[1]
|
|
29
|
+
except IndexError:
|
|
30
|
+
first_command = None
|
|
31
|
+
try:
|
|
32
|
+
second_command = args[2]
|
|
33
|
+
except IndexError:
|
|
34
|
+
second_command = "-h"
|
|
35
|
+
|
|
36
|
+
if (first_command not in ("-h", "--version", "--help")) and (first_command is not None):
|
|
37
|
+
command_map = {
|
|
38
|
+
"data": {"import": "upload", "export": "download"},
|
|
39
|
+
"account": {
|
|
40
|
+
"-h": "account -h",
|
|
41
|
+
"add": "account add",
|
|
42
|
+
"delete": "account remove",
|
|
43
|
+
"info": "account show",
|
|
44
|
+
"update": "account update",
|
|
45
|
+
"set-limits": "account limit add",
|
|
46
|
+
"get-limits": "account limit list",
|
|
47
|
+
"delete-limits": "account limit remove",
|
|
48
|
+
"ban": "account update --ban",
|
|
49
|
+
"unban": "account update --unban",
|
|
50
|
+
"list-attributes": "account attribute list",
|
|
51
|
+
"add-attribute": "account attribute add",
|
|
52
|
+
"delete-attribute": "account attribute remove",
|
|
53
|
+
},
|
|
54
|
+
"identity": {"-h": "account identity -h", "add": "account identity add", "delete": "account identity remove"},
|
|
55
|
+
"rse": {
|
|
56
|
+
"-h": "rse -h",
|
|
57
|
+
"add": "rse add",
|
|
58
|
+
"list": "rse list",
|
|
59
|
+
"update": "rse update",
|
|
60
|
+
"info": "rse show",
|
|
61
|
+
"set-attribute": "rse attribute add",
|
|
62
|
+
"delete-attribute": "rse attribute remove",
|
|
63
|
+
"delete-distance": "rse distance remove",
|
|
64
|
+
"get-distance": "rse distance show",
|
|
65
|
+
"set-distance": "rse distance update",
|
|
66
|
+
"get-attribute": "rse attribute list",
|
|
67
|
+
"add-protocol": "rse protocol add",
|
|
68
|
+
"delete-protocol": "rse protocol remove",
|
|
69
|
+
"delete": "rse remove",
|
|
70
|
+
"add-qos-policy": "rse qos add",
|
|
71
|
+
"add-distance": "rse distance add",
|
|
72
|
+
"delete-qos-policy": "rse qos remove",
|
|
73
|
+
"list-qos-policies": "rse qos list",
|
|
74
|
+
"set-limit": "rse limit add",
|
|
75
|
+
"delete-limit": "rse limit remove",
|
|
76
|
+
},
|
|
77
|
+
"scope": {"-h": "scope -h", "add": "scope add", "list": "scope list"},
|
|
78
|
+
"config": {"-h": "config -h", "get": "config list", "set": "config add", "delete": "config remove"},
|
|
79
|
+
"subscription": {"-h": "subscription -h", "add": "subscription add", "list": "subscription", "update": "subscription update", "reevaluate": "subscription touch"},
|
|
80
|
+
"replicas": {"-h": "replica -h", "quarantine": "replica state update quarantine", "declare-bad": "replica state update bad", "declare-temporary-unavailable": "replica state update unavailable", "set-tombstone": "replica remove"},
|
|
81
|
+
}
|
|
82
|
+
try:
|
|
83
|
+
new_command = command_map[first_command]
|
|
84
|
+
new_command = new_command[second_command]
|
|
85
|
+
except KeyError:
|
|
86
|
+
new_command = "-h"
|
|
87
|
+
|
|
88
|
+
warning = f"{base_warning} Please replace your command with `rucio {new_command}`"
|
|
89
|
+
else:
|
|
90
|
+
warning = base_warning + " Please view rucio -h for an updated help menu."
|
|
91
|
+
|
|
92
|
+
logger.warning(warning)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if __name__ == "__main__":
|
|
96
|
+
make_warning()
|
|
97
|
+
main_legacy()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rucio-clients
|
|
3
|
-
Version:
|
|
3
|
+
Version: 37.0.0
|
|
4
4
|
Summary: Rucio Client Lite Package
|
|
5
5
|
Home-page: https://rucio.cern.ch/
|
|
6
6
|
Author: Rucio
|
|
@@ -19,25 +19,29 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
19
19
|
Requires-Python: >=3.9, <4
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: AUTHORS.rst
|
|
22
|
-
Requires-Dist: requests
|
|
23
|
-
Requires-Dist: urllib3
|
|
24
|
-
Requires-Dist: dogpile-cache
|
|
25
|
-
Requires-Dist:
|
|
26
|
-
Requires-Dist:
|
|
22
|
+
Requires-Dist: requests
|
|
23
|
+
Requires-Dist: urllib3
|
|
24
|
+
Requires-Dist: dogpile-cache<=1.2.2
|
|
25
|
+
Requires-Dist: packaging
|
|
26
|
+
Requires-Dist: tabulate
|
|
27
|
+
Requires-Dist: jsonschema
|
|
28
|
+
Requires-Dist: dataclasses
|
|
29
|
+
Requires-Dist: rich
|
|
30
|
+
Requires-Dist: typing_extensions
|
|
27
31
|
Provides-Extra: ssh
|
|
28
|
-
Requires-Dist: paramiko
|
|
32
|
+
Requires-Dist: paramiko; extra == "ssh"
|
|
29
33
|
Provides-Extra: kerberos
|
|
30
|
-
Requires-Dist: kerberos
|
|
31
|
-
Requires-Dist: pykerberos
|
|
32
|
-
Requires-Dist: requests-kerberos
|
|
34
|
+
Requires-Dist: kerberos; extra == "kerberos"
|
|
35
|
+
Requires-Dist: pykerberos; extra == "kerberos"
|
|
36
|
+
Requires-Dist: requests-kerberos; extra == "kerberos"
|
|
33
37
|
Provides-Extra: swift
|
|
34
|
-
Requires-Dist: python-swiftclient
|
|
38
|
+
Requires-Dist: python-swiftclient; extra == "swift"
|
|
35
39
|
Provides-Extra: argcomplete
|
|
36
|
-
Requires-Dist: argcomplete
|
|
40
|
+
Requires-Dist: argcomplete; extra == "argcomplete"
|
|
37
41
|
Provides-Extra: sftp
|
|
38
|
-
Requires-Dist: paramiko
|
|
42
|
+
Requires-Dist: paramiko; extra == "sftp"
|
|
39
43
|
Provides-Extra: dumper
|
|
40
|
-
Requires-Dist: python-magic
|
|
44
|
+
Requires-Dist: python-magic; extra == "dumper"
|
|
41
45
|
Dynamic: author
|
|
42
46
|
Dynamic: author-email
|
|
43
47
|
Dynamic: classifier
|