rucio 37.7.0__py3-none-any.whl → 38.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (122) hide show
  1. rucio/alembicrevision.py +1 -1
  2. rucio/cli/bin_legacy/rucio.py +51 -107
  3. rucio/cli/bin_legacy/rucio_admin.py +26 -26
  4. rucio/cli/command.py +1 -0
  5. rucio/cli/did.py +2 -2
  6. rucio/cli/opendata.py +132 -0
  7. rucio/cli/replica.py +15 -5
  8. rucio/cli/rule.py +7 -2
  9. rucio/cli/scope.py +3 -2
  10. rucio/cli/utils.py +28 -4
  11. rucio/client/baseclient.py +9 -1
  12. rucio/client/client.py +2 -0
  13. rucio/client/diracclient.py +73 -12
  14. rucio/client/opendataclient.py +249 -0
  15. rucio/client/subscriptionclient.py +30 -0
  16. rucio/client/uploadclient.py +10 -13
  17. rucio/common/constants.py +4 -1
  18. rucio/common/exception.py +55 -0
  19. rucio/common/plugins.py +45 -8
  20. rucio/common/schema/generic.py +5 -3
  21. rucio/common/schema/generic_multi_vo.py +4 -2
  22. rucio/common/types.py +8 -7
  23. rucio/common/utils.py +176 -11
  24. rucio/core/dirac.py +5 -5
  25. rucio/core/opendata.py +744 -0
  26. rucio/core/rule.py +63 -8
  27. rucio/core/transfer.py +1 -1
  28. rucio/daemons/hermes/hermes.py +26 -17
  29. rucio/db/sqla/constants.py +6 -0
  30. rucio/db/sqla/migrate_repo/versions/a62db546a1f1_opendata_initial_model.py +85 -0
  31. rucio/db/sqla/models.py +67 -0
  32. rucio/db/sqla/util.py +2 -2
  33. rucio/gateway/dirac.py +1 -1
  34. rucio/gateway/opendata.py +190 -0
  35. rucio/gateway/subscription.py +5 -3
  36. rucio/rse/protocols/protocol.py +9 -5
  37. rucio/rse/translation.py +17 -6
  38. rucio/transfertool/fts3.py +1 -0
  39. rucio/transfertool/fts3_plugins.py +6 -1
  40. rucio/vcsversion.py +4 -4
  41. rucio/web/rest/flaskapi/v1/common.py +34 -14
  42. rucio/web/rest/flaskapi/v1/config.py +1 -1
  43. rucio/web/rest/flaskapi/v1/dids.py +447 -160
  44. rucio/web/rest/flaskapi/v1/heartbeats.py +1 -1
  45. rucio/web/rest/flaskapi/v1/identities.py +1 -1
  46. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +1 -1
  47. rucio/web/rest/flaskapi/v1/locks.py +1 -1
  48. rucio/web/rest/flaskapi/v1/main.py +3 -8
  49. rucio/web/rest/flaskapi/v1/meta_conventions.py +1 -16
  50. rucio/web/rest/flaskapi/v1/nongrid_traces.py +1 -1
  51. rucio/web/rest/flaskapi/v1/opendata.py +391 -0
  52. rucio/web/rest/flaskapi/v1/opendata_public.py +146 -0
  53. rucio/web/rest/flaskapi/v1/requests.py +1 -1
  54. rucio/web/rest/flaskapi/v1/rses.py +1 -1
  55. rucio/web/rest/flaskapi/v1/rules.py +1 -1
  56. rucio/web/rest/flaskapi/v1/scopes.py +1 -1
  57. rucio/web/rest/flaskapi/v1/subscriptions.py +6 -9
  58. rucio/web/rest/flaskapi/v1/traces.py +1 -1
  59. rucio/web/rest/flaskapi/v1/vos.py +1 -1
  60. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/alembic.ini.template +1 -1
  61. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
  62. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/rucio.cfg.template +2 -2
  63. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/rucio_multi_vo.cfg.template +3 -3
  64. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/requirements.server.txt +6 -3
  65. rucio-38.0.0rc1.data/data/rucio/tools/reset_database.py +87 -0
  66. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio +2 -1
  67. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/METADATA +36 -36
  68. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/RECORD +120 -114
  69. rucio/client/fileclient.py +0 -57
  70. rucio-37.7.0.data/data/rucio/tools/reset_database.py +0 -40
  71. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/globus-config.yml.template +0 -0
  72. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/ldap.cfg.template +0 -0
  73. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  74. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  75. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  76. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  77. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  78. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  79. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  80. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  81. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/tools/bootstrap.py +0 -0
  82. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  83. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-abacus-account +0 -0
  84. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-abacus-collection-replica +0 -0
  85. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-abacus-rse +0 -0
  86. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-admin +0 -0
  87. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-atropos +0 -0
  88. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-auditor +0 -0
  89. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-automatix +0 -0
  90. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-bb8 +0 -0
  91. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-cache-client +0 -0
  92. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-cache-consumer +0 -0
  93. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-finisher +0 -0
  94. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-poller +0 -0
  95. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-preparer +0 -0
  96. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-receiver +0 -0
  97. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-stager +0 -0
  98. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-submitter +0 -0
  99. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-conveyor-throttler +0 -0
  100. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-dark-reaper +0 -0
  101. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-dumper +0 -0
  102. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-follower +0 -0
  103. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-hermes +0 -0
  104. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-judge-cleaner +0 -0
  105. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-judge-evaluator +0 -0
  106. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-judge-injector +0 -0
  107. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-judge-repairer +0 -0
  108. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-kronos +0 -0
  109. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-minos +0 -0
  110. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-minos-temporary-expiration +0 -0
  111. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-necromancer +0 -0
  112. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-oauth-manager +0 -0
  113. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-reaper +0 -0
  114. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-replica-recoverer +0 -0
  115. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-rse-decommissioner +0 -0
  116. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-storage-consistency-actions +0 -0
  117. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-transmogrifier +0 -0
  118. {rucio-37.7.0.data → rucio-38.0.0rc1.data}/scripts/rucio-undertaker +0 -0
  119. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/WHEEL +0 -0
  120. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/licenses/AUTHORS.rst +0 -0
  121. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/licenses/LICENSE +0 -0
  122. {rucio-37.7.0.dist-info → rucio-38.0.0rc1.dist-info}/top_level.txt +0 -0
rucio/rse/translation.py CHANGED
@@ -50,7 +50,7 @@ class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
50
50
  algorithm_name = "def"
51
51
  logger.debug("PFN2LFN: Falling back to %s algorithm.", 'default' if algorithm_name == 'def' else algorithm_name)
52
52
 
53
- self.parser = self.get_parser(algorithm_name)
53
+ self.parser = self.get_parser(algorithm_name, vo)
54
54
 
55
55
  @classmethod
56
56
  def _module_init_(cls) -> None:
@@ -60,8 +60,14 @@ class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
60
60
  cls.register(cls._default, "def")
61
61
 
62
62
  @classmethod
63
- def get_parser(cls, algorithm_name: str) -> 'Callable[..., Any]':
64
- return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
63
+ def get_parser(cls, algorithm_name: str, vo: str) -> 'Callable[..., Any]':
64
+ result = None
65
+ if algorithm_name == vo:
66
+ # default algorithm for VO
67
+ result = super()._get_default_algorithm(RSEDeterministicScopeTranslation._algorithm_type, vo)
68
+ if result is None:
69
+ result = super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
70
+ return result
65
71
 
66
72
  @classmethod
67
73
  def register(
@@ -111,7 +117,8 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
111
117
  self,
112
118
  rse: Optional[str] = None,
113
119
  rse_attributes: Optional["RSESettingsDict"] = None,
114
- protocol_attributes: Optional[dict[str, Any]] = None
120
+ protocol_attributes: Optional[dict[str, Any]] = None,
121
+ vo: str = DEFAULT_VO
115
122
  ):
116
123
  """
117
124
  Initialize a translator object from the RSE, its attributes, and the protocol-specific
@@ -125,6 +132,7 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
125
132
  self.rse = rse
126
133
  self.rse_attributes = rse_attributes if rse_attributes else {}
127
134
  self.protocol_attributes = protocol_attributes if protocol_attributes else {}
135
+ self.vo = vo
128
136
 
129
137
  @classmethod
130
138
  def supports(
@@ -251,9 +259,12 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
251
259
  :returns: RSE specific URI of the physical file
252
260
  """
253
261
  algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
254
- if algorithm == 'default':
262
+ algorithm_callable = None
263
+ if algorithm == 'default' or algorithm == RSEDeterministicTranslation._DEFAULT_LFN2PFN:
255
264
  algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
256
- algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
265
+ algorithm_callable = super()._get_default_algorithm(RSEDeterministicTranslation._algorithm_type, self.vo)
266
+ if algorithm_callable is None:
267
+ algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
257
268
  return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
258
269
 
259
270
 
@@ -1046,6 +1046,7 @@ class FTS3Transfertool(Transfertool):
1046
1046
  t_file['scitag'] = self.scitags_exp_id << 6 | activity_id
1047
1047
 
1048
1048
  if t_file['metadata']['dst_type'] == 'TAPE':
1049
+ t_file['metadata']['vo'] = rws.scope.vo
1049
1050
  for plugin in self.tape_metadata_plugins:
1050
1051
  t_file = deep_merge_dict(source=plugin.hints(t_file['metadata']), destination=t_file)
1051
1052
 
@@ -17,6 +17,7 @@ import sys
17
17
  from typing import TYPE_CHECKING, Any, Optional, TypeVar
18
18
 
19
19
  from rucio.common.config import config_get_int
20
+ from rucio.common.constants import DEFAULT_VO
20
21
  from rucio.common.exception import InvalidRequest
21
22
  from rucio.common.plugins import PolicyPackageAlgorithms
22
23
 
@@ -85,7 +86,11 @@ class FTS3TapeMetadataPlugin(PolicyPackageAlgorithms):
85
86
  """
86
87
  return {"collocation_hints": collocation_func(**hints)}
87
88
 
88
- def _default(self, *hints: dict) -> dict:
89
+ def _default(self, hint_dict: dict[str, Any]) -> dict:
90
+ vo = hint_dict['vo'] if 'vo' in hint_dict else DEFAULT_VO
91
+ default_algorithm = self._get_default_algorithm(self.ALGORITHM_NAME, vo=vo)
92
+ if default_algorithm is not None:
93
+ return default_algorithm(hint_dict)
89
94
  return {}
90
95
 
91
96
  def _verify_in_format(self, hint_dict: dict[str, Any]) -> None:
rucio/vcsversion.py CHANGED
@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
4
4
  '''
5
5
  VERSION_INFO = {
6
6
  'final': True,
7
- 'version': '37.7.0',
8
- 'branch_nick': 'release-37',
9
- 'revision_id': '61ed028cbd532ede88189a81bee971db566fcdec',
10
- 'revno': 13826
7
+ 'version': '38.0.0rc1',
8
+ 'branch_nick': 'master',
9
+ 'revision_id': '7f986b67665c21ac40a2b32732b2f263bafd2f6c',
10
+ 'revno': 13890
11
11
  }
@@ -20,11 +20,12 @@ import re
20
20
  from configparser import NoOptionError, NoSectionError
21
21
  from functools import wraps
22
22
  from time import time
23
- from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union
23
+ from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, cast
24
24
  from urllib.parse import unquote_plus
25
25
 
26
26
  import flask
27
27
  from flask.views import MethodView
28
+ from typing_extensions import ParamSpec
28
29
  from werkzeug.datastructures import Headers
29
30
  from werkzeug.exceptions import HTTPException
30
31
  from werkzeug.wrappers import Request, Response
@@ -183,25 +184,44 @@ def response_headers(response: ResponseTypeVar) -> ResponseTypeVar:
183
184
  return response
184
185
 
185
186
 
186
- def check_accept_header_wrapper_flask(supported_content_types: 'Iterable[str]'):
187
- """ Decorator to check if an endpoint supports the requested content type. """
187
+ P = ParamSpec('P')
188
+ R = TypeVar('R')
189
+
190
+
191
+ def check_accept_header_wrapper_flask(
192
+ supported_content_types: 'Iterable[str]'
193
+ ) -> 'Callable[[Callable[P, R]], Callable[P, R]]':
194
+ """Decorator that refuses requests with an unsupported *Accept* header."""
195
+
196
+ def wrapper(
197
+ f: 'Callable[P, R]'
198
+ ) -> 'Callable[P, R]':
199
+ """Decorate *f* with an *Accept*-header check and return the new callable."""
188
200
 
189
- def wrapper(f):
190
201
  @wraps(f)
191
- def decorated(*args, **kwargs):
202
+ def decorated(*args: 'P.args', **kwargs: 'P.kwargs') -> 'R':
203
+ """Run the header check, then delegate to *f* (or return 406)."""
204
+
205
+ # 1. no Accept header → accept everything
192
206
  if not flask.request.accept_mimetypes.provided:
193
- # accept anything, if Accept header is not provided
194
207
  return f(*args, **kwargs)
195
208
 
196
- for supported in supported_content_types:
197
- if supported in flask.request.accept_mimetypes:
198
- return f(*args, **kwargs)
209
+ # 2. at least one acceptable media‑type → call the view
210
+ if any(s in flask.request.accept_mimetypes for s in supported_content_types):
211
+ return f(*args, **kwargs)
199
212
 
200
- # none matched..
201
- return generate_http_error_flask(
202
- status_code=406,
203
- exc=UnsupportedRequestedContentType.__name__,
204
- exc_msg=f'The requested content type {flask.request.environ.get("HTTP_ACCEPT")} is not supported. Use {supported_content_types}.'
213
+ # 3. none matched → 406 response
214
+ return cast(
215
+ 'R',
216
+ generate_http_error_flask(
217
+ status_code=406,
218
+ exc=UnsupportedRequestedContentType.__name__,
219
+ exc_msg=(
220
+ f'The requested content type '
221
+ f'{flask.request.environ.get("HTTP_ACCEPT")} is not supported. '
222
+ f'Use {supported_content_types}.'
223
+ ),
224
+ ),
205
225
  )
206
226
 
207
227
  return decorated
@@ -281,7 +281,7 @@ class OptionSet(ErrorHandlingMethodView):
281
281
  return generate_http_error_flask(500, error, f"Could not set value '{value}' for section '{section}' option '{option}'")
282
282
 
283
283
 
284
- def blueprint():
284
+ def blueprint() -> AuthenticatedBlueprint:
285
285
  bp = AuthenticatedBlueprint('config', __name__, url_prefix='/config')
286
286
 
287
287
  option_set_view = OptionSet.as_view('option_set')