rucio 38.3.0__py3-none-any.whl → 38.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (149) hide show
  1. rucio/cli/bin_legacy/rucio.py +12 -7
  2. rucio/cli/bin_legacy/rucio_admin.py +9 -2
  3. rucio/cli/did.py +1 -1
  4. rucio/cli/opendata.py +19 -2
  5. rucio/cli/replica.py +6 -2
  6. rucio/cli/rule.py +0 -1
  7. rucio/cli/scope.py +9 -0
  8. rucio/cli/utils.py +11 -0
  9. rucio/client/accountclient.py +20 -19
  10. rucio/client/accountlimitclient.py +5 -4
  11. rucio/client/baseclient.py +25 -25
  12. rucio/client/configclient.py +7 -6
  13. rucio/client/credentialclient.py +2 -1
  14. rucio/client/didclient.py +33 -32
  15. rucio/client/diracclient.py +2 -1
  16. rucio/client/downloadclient.py +3 -1
  17. rucio/client/exportclient.py +2 -1
  18. rucio/client/importclient.py +2 -1
  19. rucio/client/lifetimeclient.py +3 -2
  20. rucio/client/lockclient.py +4 -3
  21. rucio/client/metaconventionsclient.py +5 -4
  22. rucio/client/opendataclient.py +8 -7
  23. rucio/client/pingclient.py +2 -1
  24. rucio/client/replicaclient.py +27 -26
  25. rucio/client/requestclient.py +8 -8
  26. rucio/client/rseclient.py +31 -28
  27. rucio/client/ruleclient.py +13 -12
  28. rucio/client/scopeclient.py +44 -4
  29. rucio/client/subscriptionclient.py +6 -5
  30. rucio/common/constants.py +18 -0
  31. rucio/common/didtype.py +18 -11
  32. rucio/common/exception.py +20 -0
  33. rucio/common/plugins.py +9 -7
  34. rucio/core/credential.py +19 -26
  35. rucio/core/did.py +1 -1
  36. rucio/core/did_meta_plugins/__init__.py +2 -1
  37. rucio/core/did_meta_plugins/did_column_meta.py +2 -10
  38. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +39 -25
  39. rucio/core/did_meta_plugins/elasticsearch_meta.py +3 -11
  40. rucio/core/did_meta_plugins/json_meta.py +2 -8
  41. rucio/core/did_meta_plugins/mongo_meta.py +3 -12
  42. rucio/core/did_meta_plugins/postgres_meta.py +7 -14
  43. rucio/core/dirac.py +1 -1
  44. rucio/core/opendata.py +150 -8
  45. rucio/core/rse.py +6 -2
  46. rucio/core/rule_grouping.py +3 -3
  47. rucio/core/scope.py +47 -7
  48. rucio/daemons/automatix/automatix.py +2 -0
  49. rucio/db/sqla/models.py +22 -0
  50. rucio/gateway/account.py +8 -7
  51. rucio/gateway/did.py +1 -1
  52. rucio/gateway/dirac.py +1 -1
  53. rucio/gateway/opendata.py +2 -2
  54. rucio/gateway/request.py +2 -117
  55. rucio/gateway/scope.py +35 -3
  56. rucio/rse/protocols/webdav.py +5 -2
  57. rucio/transfertool/fts3.py +0 -19
  58. rucio/vcsversion.py +3 -3
  59. rucio/web/rest/flaskapi/v1/accountlimits.py +4 -3
  60. rucio/web/rest/flaskapi/v1/accounts.py +26 -25
  61. rucio/web/rest/flaskapi/v1/archives.py +2 -2
  62. rucio/web/rest/flaskapi/v1/auth.py +15 -14
  63. rucio/web/rest/flaskapi/v1/common.py +4 -4
  64. rucio/web/rest/flaskapi/v1/config.py +6 -4
  65. rucio/web/rest/flaskapi/v1/credentials.py +3 -3
  66. rucio/web/rest/flaskapi/v1/dids.py +25 -24
  67. rucio/web/rest/flaskapi/v1/dirac.py +3 -2
  68. rucio/web/rest/flaskapi/v1/export.py +4 -2
  69. rucio/web/rest/flaskapi/v1/heartbeats.py +2 -1
  70. rucio/web/rest/flaskapi/v1/identities.py +5 -4
  71. rucio/web/rest/flaskapi/v1/import.py +3 -2
  72. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +3 -2
  73. rucio/web/rest/flaskapi/v1/locks.py +4 -3
  74. rucio/web/rest/flaskapi/v1/meta_conventions.py +4 -3
  75. rucio/web/rest/flaskapi/v1/metrics.py +2 -1
  76. rucio/web/rest/flaskapi/v1/nongrid_traces.py +2 -1
  77. rucio/web/rest/flaskapi/v1/opendata.py +28 -27
  78. rucio/web/rest/flaskapi/v1/opendata_public.py +12 -11
  79. rucio/web/rest/flaskapi/v1/ping.py +3 -2
  80. rucio/web/rest/flaskapi/v1/redirect.py +4 -3
  81. rucio/web/rest/flaskapi/v1/replicas.py +31 -31
  82. rucio/web/rest/flaskapi/v1/requests.py +7 -7
  83. rucio/web/rest/flaskapi/v1/rses.py +23 -16
  84. rucio/web/rest/flaskapi/v1/rules.py +9 -8
  85. rucio/web/rest/flaskapi/v1/scopes.py +66 -13
  86. rucio/web/rest/flaskapi/v1/subscriptions.py +9 -8
  87. rucio/web/rest/flaskapi/v1/traces.py +2 -1
  88. rucio/web/rest/flaskapi/v1/vos.py +4 -3
  89. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/rucio.cfg.template +2 -3
  90. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +2 -3
  91. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/METADATA +1 -1
  92. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/RECORD +149 -149
  93. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  94. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  95. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  96. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
  97. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  98. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  99. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  100. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  101. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  102. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  103. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  104. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  105. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/requirements.server.txt +0 -0
  106. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/tools/bootstrap.py +0 -0
  107. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
  108. {rucio-38.3.0.data → rucio-38.5.0.data}/data/rucio/tools/reset_database.py +0 -0
  109. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio +0 -0
  110. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-abacus-account +0 -0
  111. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-abacus-collection-replica +0 -0
  112. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-abacus-rse +0 -0
  113. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-admin +0 -0
  114. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-atropos +0 -0
  115. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-auditor +0 -0
  116. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-automatix +0 -0
  117. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-bb8 +0 -0
  118. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-cache-client +0 -0
  119. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-cache-consumer +0 -0
  120. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-finisher +0 -0
  121. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-poller +0 -0
  122. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-preparer +0 -0
  123. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-receiver +0 -0
  124. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-stager +0 -0
  125. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-submitter +0 -0
  126. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-conveyor-throttler +0 -0
  127. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-dark-reaper +0 -0
  128. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-dumper +0 -0
  129. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-follower +0 -0
  130. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-hermes +0 -0
  131. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-judge-cleaner +0 -0
  132. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-judge-evaluator +0 -0
  133. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-judge-injector +0 -0
  134. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-judge-repairer +0 -0
  135. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-kronos +0 -0
  136. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-minos +0 -0
  137. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
  138. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-necromancer +0 -0
  139. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-oauth-manager +0 -0
  140. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-reaper +0 -0
  141. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-replica-recoverer +0 -0
  142. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-rse-decommissioner +0 -0
  143. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-storage-consistency-actions +0 -0
  144. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-transmogrifier +0 -0
  145. {rucio-38.3.0.data → rucio-38.5.0.data}/scripts/rucio-undertaker +0 -0
  146. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/WHEEL +0 -0
  147. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/licenses/AUTHORS.rst +0 -0
  148. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/licenses/LICENSE +0 -0
  149. {rucio-38.3.0.dist-info → rucio-38.5.0.dist-info}/top_level.txt +0 -0
@@ -13,13 +13,18 @@
13
13
  # limitations under the License.
14
14
 
15
15
  from json import loads
16
+ from typing import TYPE_CHECKING, Any, Literal, Union
16
17
  from urllib.parse import quote_plus
17
18
 
18
19
  from requests.status_codes import codes
19
20
 
20
21
  from rucio.client.baseclient import BaseClient, choice
22
+ from rucio.common.constants import HTTPMethod
21
23
  from rucio.common.utils import build_url
22
24
 
25
+ if TYPE_CHECKING:
26
+ from collections.abc import Iterator
27
+
23
28
 
24
29
  class ScopeClient(BaseClient):
25
30
 
@@ -56,14 +61,14 @@ class ScopeClient(BaseClient):
56
61
 
57
62
  path = '/'.join([self.SCOPE_BASEURL, account, 'scopes', quote_plus(scope)])
58
63
  url = build_url(choice(self.list_hosts), path=path)
59
- r = self._send_request(url, type_='POST')
64
+ r = self._send_request(url, method=HTTPMethod.POST)
60
65
  if r.status_code == codes.created:
61
66
  return True
62
67
  else:
63
68
  exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
64
69
  raise exc_cls(exc_msg)
65
70
 
66
- def list_scopes(self) -> list[str]:
71
+ def list_scopes(self) -> "Union[list[str], Iterator[dict[Literal['scope', 'account'], Any]]]":
67
72
  """
68
73
  Sends the request to list all scopes.
69
74
 
@@ -74,7 +79,7 @@ class ScopeClient(BaseClient):
74
79
 
75
80
  path = '/'.join(['scopes/'])
76
81
  url = build_url(choice(self.list_hosts), path=path)
77
- r = self._send_request(url)
82
+ r = self._send_request(url, method=HTTPMethod.GET)
78
83
  if r.status_code == codes.ok:
79
84
  scopes = loads(r.text)
80
85
  return scopes
@@ -82,6 +87,41 @@ class ScopeClient(BaseClient):
82
87
  exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
83
88
  raise exc_cls(exc_msg)
84
89
 
90
+ def update_scope_ownership(self, account: str, scope: str) -> bool:
91
+ """
92
+ Change the ownership of a scope
93
+
94
+ Parameters
95
+ ----------
96
+ account :
97
+ New account to assign as scope owner
98
+ scope :
99
+ Scope to change ownership of
100
+
101
+ Returns
102
+ -------
103
+ bool
104
+ True if the operation was successful
105
+
106
+ Raises
107
+ ------
108
+ AccountNotFound
109
+ If account doesn't exist.
110
+ ScopeNotFound
111
+ If scope doesn't exist.
112
+ CannotAuthenticate, AccessDenied
113
+ Insufficient permission/incorrect credentials to change ownership.
114
+ """
115
+
116
+ path = '/'.join(['scopes', account, scope])
117
+ url = build_url(choice(self.list_hosts), path=path)
118
+ r = self._send_request(url, method=HTTPMethod.PUT)
119
+ if r.status_code == codes.ok:
120
+ return True
121
+ else:
122
+ exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
123
+ raise exc_cls(exc_msg)
124
+
85
125
  def list_scopes_for_account(self, account: str) -> list[str]:
86
126
  """
87
127
  Sends the request to list all scopes for a rucio account.
@@ -106,7 +146,7 @@ class ScopeClient(BaseClient):
106
146
  path = '/'.join([self.SCOPE_BASEURL, account, 'scopes/'])
107
147
  url = build_url(choice(self.list_hosts), path=path)
108
148
 
109
- r = self._send_request(url)
149
+ r = self._send_request(url, method=HTTPMethod.GET)
110
150
  if r.status_code == codes.ok:
111
151
  scopes = loads(r.text)
112
152
  return scopes
@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union
18
18
  from requests.status_codes import codes
19
19
 
20
20
  from rucio.client.baseclient import BaseClient, choice
21
+ from rucio.common.constants import HTTPMethod
21
22
  from rucio.common.utils import build_url
22
23
 
23
24
  if TYPE_CHECKING:
@@ -78,7 +79,7 @@ class SubscriptionClient(BaseClient):
78
79
  raise TypeError('replication_rules should be a list')
79
80
  data = dumps({'options': {'filter': filter_, 'replication_rules': replication_rules, 'comments': comments,
80
81
  'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
81
- result = self._send_request(url, type_='POST', data=data)
82
+ result = self._send_request(url, method=HTTPMethod.POST, data=data)
82
83
  if result.status_code == codes.created: # pylint: disable=no-member
83
84
  return result.text
84
85
  else:
@@ -120,7 +121,7 @@ class SubscriptionClient(BaseClient):
120
121
  else:
121
122
  path += '/'
122
123
  url = build_url(choice(self.list_hosts), path=path)
123
- result = self._send_request(url, type_='GET')
124
+ result = self._send_request(url, method=HTTPMethod.GET)
124
125
  if result.status_code == codes.ok: # pylint: disable=no-member
125
126
  return self._load_json_data(result)
126
127
  if result.status_code == codes.not_found:
@@ -173,7 +174,7 @@ class SubscriptionClient(BaseClient):
173
174
  raise TypeError('replication_rules should be a list')
174
175
  data = dumps({'options': {'filter': filter_, 'replication_rules': replication_rules, 'comments': comments,
175
176
  'lifetime': lifetime, 'retroactive': retroactive, 'dry_run': dry_run, 'priority': priority}})
176
- result = self._send_request(url, type_='PUT', data=data)
177
+ result = self._send_request(url, method=HTTPMethod.PUT, data=data)
177
178
  if result.status_code == codes.created: # pylint: disable=no-member
178
179
  return True
179
180
  else:
@@ -203,7 +204,7 @@ class SubscriptionClient(BaseClient):
203
204
  path = self.SUB_BASEURL + '/' + account + '/' + name # type: ignore
204
205
  url = build_url(choice(self.list_hosts), path=path)
205
206
  data = dumps({'options': {'state': 'I'}})
206
- result = self._send_request(url, type_='PUT', data=data)
207
+ result = self._send_request(url, method=HTTPMethod.PUT, data=data)
207
208
  if result.status_code == codes.created: # pylint: disable=no-member
208
209
  return True
209
210
  else:
@@ -228,7 +229,7 @@ class SubscriptionClient(BaseClient):
228
229
 
229
230
  path = '/'.join([self.SUB_BASEURL, account, name, 'rules'])
230
231
  url = build_url(choice(self.list_hosts), path=path)
231
- result = self._send_request(url, type_='GET')
232
+ result = self._send_request(url, method=HTTPMethod.GET)
232
233
  if result.status_code == codes.ok: # pylint: disable=no-member
233
234
  return self._load_json_data(result)
234
235
  else:
rucio/common/constants.py CHANGED
@@ -13,6 +13,7 @@
13
13
  # limitations under the License.
14
14
 
15
15
  import enum
16
+ import sys
16
17
  from collections import namedtuple
17
18
  from typing import Literal, get_args
18
19
 
@@ -224,3 +225,20 @@ OPENDATA_DID_STATE_LITERAL_LIST = list(get_args(OPENDATA_DID_STATE_LITERAL))
224
225
 
225
226
  POLICY_ALGORITHM_TYPES_LITERAL = Literal['non_deterministic_pfn', 'scope', 'lfn2pfn', 'pfn2lfn', 'fts3_tape_metadata_plugins', 'fts3_plugins_init', 'auto_approve']
226
227
  POLICY_ALGORITHM_TYPES = list(get_args(POLICY_ALGORITHM_TYPES_LITERAL))
228
+
229
+ # https://github.com/rucio/rucio/issues/7958
230
+ # When Python 3.11 is the minimum supported version, we can use the standard library enum and remove this logic
231
+ if sys.version_info >= (3, 11):
232
+ from http import HTTPMethod
233
+ else:
234
+ @enum.unique
235
+ class HTTPMethod(str, enum.Enum):
236
+ """HTTP verbs used in Rucio requests."""
237
+
238
+ HEAD = "HEAD"
239
+ OPTIONS = "OPTIONS"
240
+ PATCH = "PATCH"
241
+ GET = "GET"
242
+ POST = "POST"
243
+ PUT = "PUT"
244
+ DELETE = "DELETE"
rucio/common/didtype.py CHANGED
@@ -15,10 +15,12 @@
15
15
  """
16
16
  DID type to represent a DID and to simplify operations on it
17
17
  """
18
-
18
+ import logging
19
+ from configparser import NoSectionError
19
20
  from typing import Any, Union
20
21
 
21
- from rucio.common.exception import DIDError
22
+ from rucio.common.exception import ConfigNotFound, DIDError, InvalidAlgorithmName
23
+ from rucio.common.utils import extract_scope
22
24
 
23
25
 
24
26
  class DID:
@@ -126,15 +128,20 @@ class DID:
126
128
  Construct the DID from a string.
127
129
  :param did: string containing the DID information
128
130
  """
129
- did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
130
- if len(did_parts) == 1:
131
- self.name = did
132
- self._update_implicit_scope()
133
- if not self.has_scope():
134
- raise DIDError('Object construction from non-splitable string is ambigious')
135
- else:
136
- self.scope = did_parts[0]
137
- self.name = did_parts[1]
131
+ try:
132
+ self.scope, self.name = extract_scope(did)
133
+ except (ImportError, InvalidAlgorithmName, ConfigNotFound, NoSectionError) as e: # Only use when the policy can not be found
134
+ logging.debug("Failure using extract_scope policy for '%s': %s - Using fallback." % (did, type(e).__name__))
135
+ did_parts = did.split(DID.SCOPE_SEPARATOR, 1)
136
+ if len(did_parts) == 1:
137
+ self.name = did
138
+ self._update_implicit_scope()
139
+ if not self.has_scope():
140
+ error = f"Could not parse scope from did string {did} - fallback policy expects only one '{DID.SCOPE_SEPARATOR}'"
141
+ raise DIDError(error)
142
+ else:
143
+ self.scope = did_parts[0]
144
+ self.name = did_parts[1]
138
145
 
139
146
  def _did_from_dict(self, did: dict[str, str]) -> None:
140
147
  """
rucio/common/exception.py CHANGED
@@ -1271,3 +1271,23 @@ class InvalidPolicyPackageAlgorithmType(RucioException):
1271
1271
  super(InvalidPolicyPackageAlgorithmType, self).__init__(*args)
1272
1272
  self._message = f"Invalid policy package algorithm type '{param}'."
1273
1273
  self.error_code = 120
1274
+
1275
+
1276
+ class InvalidAccountType(RucioException):
1277
+ """
1278
+ Thrown when an account is created with an invalid type
1279
+ """
1280
+ def __init__(self, *args):
1281
+ super(InvalidAccountType, self).__init__(*args)
1282
+ self._message = "Cannot create an account with an invalid type."
1283
+ self.error_code = 121
1284
+
1285
+ class OpenDataDuplicateDOI(OpenDataError):
1286
+ """
1287
+ Throws when a data identifier with the same DOI already exists in the open data catalog.
1288
+ """
1289
+
1290
+ def __init__(self, doi: str, *args):
1291
+ super(OpenDataDuplicateDOI, self).__init__(*args)
1292
+ self._message = f"Data identifier with the same DOI ({doi}) already exists in the open data catalog."
1293
+ self.error_code = 122
rucio/common/plugins.py CHANGED
@@ -77,7 +77,7 @@ class PolicyPackageAlgorithms:
77
77
  """
78
78
  _ALGORITHMS: dict[POLICY_ALGORITHM_TYPES_LITERAL, dict[str, 'Callable[..., Any]']] = {}
79
79
  _loaded_policy_modules = False
80
- _default_algorithms: dict[str, 'Callable[..., Any]'] = {}
80
+ _default_algorithms: dict[str, Optional['Callable[..., Any]']] = {}
81
81
 
82
82
  def __init__(self) -> None:
83
83
  if not self._loaded_policy_modules:
@@ -105,17 +105,23 @@ class PolicyPackageAlgorithms:
105
105
  vo = ''
106
106
  package = cls._get_policy_package_name(vo)
107
107
  except (NoOptionError, NoSectionError):
108
+ cls._default_algorithms[type_for_vo] = default_algorithm
108
109
  return default_algorithm
109
110
 
110
111
  module_name = package + "." + algorithm_type
112
+ LOGGER.info('Attempting to find algorithm %s in default location %s...' % (algorithm_type, module_name))
111
113
  try:
112
114
  module = importlib.import_module(module_name)
113
115
 
114
116
  if hasattr(module, algorithm_type):
115
117
  default_algorithm = getattr(module, algorithm_type)
116
- cls._default_algorithms[type_for_vo] = default_algorithm
118
+ except ModuleNotFoundError:
119
+ LOGGER.info('Algorithm %s not found in default location %s' % (algorithm_type, module_name))
117
120
  except ImportError:
118
- LOGGER.info('Policy algorithm module %s could not be loaded' % module_name)
121
+ LOGGER.info('Algorithm %s found in default location %s, but could not be loaded' % (algorithm_type, module_name))
122
+ # if the default algorithm is not present, this will store None and we will
123
+ # not attempt to load the same algorithm again
124
+ cls._default_algorithms[type_for_vo] = default_algorithm
119
125
  return default_algorithm
120
126
 
121
127
  @classmethod
@@ -212,10 +218,6 @@ class PolicyPackageAlgorithms:
212
218
  if hasattr(module, 'get_algorithms'):
213
219
  all_algorithms = module.get_algorithms()
214
220
 
215
- # for backward compatibility, rename 'surl' to 'non_deterministic_pfn' here
216
- if 'surl' in all_algorithms:
217
- all_algorithms['non_deterministic_pfn'] = all_algorithms['surl']
218
-
219
221
  # check that the names are correctly prefixed for multi-VO
220
222
  if vo:
221
223
  for _, algorithms in all_algorithms.items():
rucio/core/credential.py CHANGED
@@ -27,7 +27,7 @@ from google.oauth2.service_account import Credentials
27
27
 
28
28
  from rucio.common.cache import MemcacheRegion
29
29
  from rucio.common.config import config_get, get_rse_credentials
30
- from rucio.common.constants import RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS, RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, SUPPORTED_SIGN_URL_SERVICES, SUPPORTED_SIGN_URL_SERVICES_LITERAL, RseAttr
30
+ from rucio.common.constants import RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS, RSE_BASE_SUPPORTED_PROTOCOL_OPERATIONS_LITERAL, SUPPORTED_SIGN_URL_SERVICES, SUPPORTED_SIGN_URL_SERVICES_LITERAL, HTTPMethod, RseAttr
31
31
  from rucio.common.exception import UnsupportedOperation
32
32
  from rucio.core.monitor import MetricManager
33
33
  from rucio.core.rse import get_rse_attribute
@@ -51,7 +51,7 @@ def get_signed_url(
51
51
  The signed URL will be valid for 1 hour but can be overridden.
52
52
 
53
53
  :param rse_id: The ID of the RSE that the URL points to.
54
- :param service: The service to authorise, either 'gcs', 's3' or 'swift'.
54
+ :param service: The service to authorize, either 'gcs', 's3' or 'swift'.
55
55
  :param operation: The operation to sign, either 'read', 'write', or 'delete'.
56
56
  :param url: The URL to sign.
57
57
  :param lifetime: Lifetime of the signed URL in seconds.
@@ -69,6 +69,8 @@ def get_signed_url(
69
69
  if url is None or url == '':
70
70
  raise UnsupportedOperation('URL must not be empty')
71
71
 
72
+ operations_map = {'read': HTTPMethod.GET.value, 'write': HTTPMethod.PUT.value, 'delete': HTTPMethod.DELETE.value}
73
+
72
74
  if lifetime:
73
75
  if not isinstance(lifetime, int):
74
76
  try:
@@ -88,25 +90,21 @@ def get_signed_url(
88
90
  if lifetime is None:
89
91
  lifetime = 0
90
92
  else:
91
- # GCS is timezone-sensitive, don't use UTC
92
- # has to be converted to Unixtime
93
+ # GCS is timezone-sensitive, don't use UTC. Has to be converted to Unix time
93
94
  lifetime_datetime = datetime.datetime.now() + datetime.timedelta(seconds=lifetime)
94
95
  lifetime = int(time.mktime(lifetime_datetime.timetuple()))
95
96
 
96
97
  # sign the path only
97
98
  path = components.path
98
99
 
99
- # Map operations
100
- operations = {'read': 'GET', 'write': 'PUT', 'delete': 'DELETE'}
101
-
102
- # assemble message to sign
103
- to_sign = "%s\n\n\n%s\n%s" % (operations[operation], lifetime, path)
100
+ # assemble a message to sign
101
+ to_sign = "%s\n\n\n%s\n%s" % (operations_map[operation], lifetime, path)
104
102
 
105
103
  # create URL-capable signature
106
104
  # first character is always a '=', remove it
107
105
  signature = urlencode({'': base64.b64encode(CREDS_GCS.sign_bytes(to_sign))})[1:]
108
106
 
109
- # assemble final signed URL
107
+ # assemble the final signed URL
110
108
  signed_url = (
111
109
  f'https://{host}{path}'
112
110
  f'?GoogleAccessId={CREDS_GCS.service_account_email}'
@@ -127,18 +125,18 @@ def get_signed_url(
127
125
  # split URL to get hostname, bucket and key
128
126
  components = urlparse(url)
129
127
  host = components.netloc
130
- pathcomponents = components.path.split('/')
128
+ path_components = components.path.split('/')
131
129
  if s3_url_style == "path":
132
- if len(pathcomponents) < 3:
130
+ if len(path_components) < 3:
133
131
  raise UnsupportedOperation('Not a valid Path-Style S3 URL')
134
- bucket = pathcomponents[1]
135
- key = '/'.join(pathcomponents[2:])
132
+ bucket = path_components[1]
133
+ key = '/'.join(path_components[2:])
136
134
  elif s3_url_style == "host":
137
- hostcomponents = host.split('.')
138
- bucket = hostcomponents[0]
139
- if len(pathcomponents) < 2:
135
+ host_components = host.split('.')
136
+ bucket = host_components[0]
137
+ if len(path_components) < 2:
140
138
  raise UnsupportedOperation('Not a valid Host-Style S3 URL')
141
- key = '/'.join(pathcomponents[1:])
139
+ key = '/'.join(path_components[1:])
142
140
  else:
143
141
  raise UnsupportedOperation('Not a valid RSE S3 URL style (allowed values: path|host)')
144
142
 
@@ -185,7 +183,7 @@ def get_signed_url(
185
183
  s3op, Params={'Bucket': bucket, 'Key': key}, ExpiresIn=lifetime)
186
184
 
187
185
  else: # service == 'swift'
188
- # split URL to get hostname and path
186
+ # split URL to get the hostname and path
189
187
  components = urlparse(url)
190
188
  host = components.netloc
191
189
 
@@ -194,7 +192,7 @@ def get_signed_url(
194
192
  if colon >= 0:
195
193
  host = host[:colon]
196
194
 
197
- # use RSE ID to look up key
195
+ # use RSE ID to look up the key
198
196
  cred_name = rse_id
199
197
 
200
198
  # look up tempurl signing key
@@ -205,12 +203,7 @@ def get_signed_url(
205
203
  REGION.set('swift-%s' % cred_name, cred)
206
204
  tempurl_key = cred['tempurl_key']
207
205
 
208
- if operation == 'read':
209
- swiftop = 'GET'
210
- elif operation == 'write':
211
- swiftop = 'PUT'
212
- else:
213
- swiftop = 'DELETE'
206
+ swiftop = operations_map[operation]
214
207
 
215
208
  expires = int(time.time() + lifetime) # type: ignore (lifetime could be None)
216
209
 
rucio/core/did.py CHANGED
@@ -724,7 +724,7 @@ def __add_collections_to_container(
724
724
  for row in session.execute(stmt):
725
725
 
726
726
  if row.did_scope is None:
727
- raise exception.DataIdentifierNotFound("Data identifier '%(scope)s:%(name)s' not found" % row)
727
+ raise exception.DataIdentifierNotFound(f"Data identifier '{row.scope}:{row.name}' not found")
728
728
 
729
729
  if row.did_type == DIDType.FILE:
730
730
  raise exception.UnsupportedOperation("Adding a file (%s:%s) to a container (%s:%s) is forbidden" % (row.scope, row.name, parent_did.scope, parent_did.name))
@@ -96,8 +96,9 @@ def get_metadata(scope, name, plugin="DID_COLUMN", *, session: "Session"):
96
96
  return all_metadata
97
97
  else:
98
98
  for metadata_plugin in METADATA_PLUGIN_MODULES:
99
- if metadata_plugin.get_plugin_name().lower() == plugin.lower():
99
+ if metadata_plugin.is_named(plugin):
100
100
  return metadata_plugin.get_metadata(scope, name, session=session)
101
+
101
102
  raise exception.UnsupportedMetadataPlugin(f'Metadata plugin "{plugin}" is not enabled on the server.')
102
103
 
103
104
 
@@ -46,7 +46,8 @@ class DidColumnMeta(DidMetaPlugin):
46
46
  def __init__(self) -> None:
47
47
  """Initialize the DID column metadata plugin."""
48
48
  super(DidColumnMeta, self).__init__()
49
- self.plugin_name = "DID_COLUMN"
49
+
50
+ self._plugin_name = "DID_COLUMN"
50
51
 
51
52
  @read_session
52
53
  def get_metadata(
@@ -477,12 +478,3 @@ class DidColumnMeta(DidMetaPlugin):
477
478
  hardcoded_keys = list(set(all_did_table_columns) - set(exclude_did_table_columns)) + additional_keys
478
479
 
479
480
  return key in hardcoded_keys
480
-
481
- def get_plugin_name(
482
- self
483
- ) -> str:
484
- """
485
- Return a unique identifier for this plugin.
486
- :returns: The name of the plugin.
487
- """
488
- return self.plugin_name
@@ -13,13 +13,11 @@
13
13
  # limitations under the License.
14
14
 
15
15
  from abc import ABCMeta, abstractmethod
16
- from typing import TYPE_CHECKING, Literal
17
-
18
- from rucio.db.sqla.session import transactional_session
16
+ from typing import TYPE_CHECKING
19
17
 
20
18
  if TYPE_CHECKING:
21
19
  from collections.abc import Iterator
22
- from typing import Any, Optional, Union
20
+ from typing import Any, Literal
23
21
 
24
22
  from sqlalchemy.orm import Session
25
23
 
@@ -35,7 +33,23 @@ class DidMetaPlugin(metaclass=ABCMeta):
35
33
  """
36
34
  Initializes the plugin
37
35
  """
38
- pass
36
+ self._plugin_name = None
37
+
38
+ @property
39
+ def name(self) -> str:
40
+ """
41
+ The getter method for the plugin's name.
42
+
43
+ :returns: The standardized (casefolded) name of this plugin.
44
+ :raises AttributeError: If '_plugin_name' is not defined in the subclass.
45
+ """
46
+ if self._plugin_name:
47
+ return self._plugin_name.casefold()
48
+ raise AttributeError("Subclasses of DidMetaPlugin must define the '_plugin_name' attribute.")
49
+
50
+ def is_named(self, plugin_name: str) -> bool:
51
+ """Return whether the plugin matches the provided name using case-insensitive comparison."""
52
+ return self.name == plugin_name.casefold()
39
53
 
40
54
  @abstractmethod
41
55
  def get_metadata(
@@ -43,12 +57,12 @@ class DidMetaPlugin(metaclass=ABCMeta):
43
57
  scope: "InternalScope",
44
58
  name: str,
45
59
  *,
46
- session: "Optional[Session]" = None
60
+ session: "Session | None" = None
47
61
  ) -> "Any":
48
62
  """
49
63
  Get data identifier metadata
50
64
 
51
- :param scope: The scope name.
65
+ :param scope: The scope of the DID.
52
66
  :param name: The data identifier name.
53
67
  :param session: The database session in use.
54
68
  """
@@ -63,22 +77,21 @@ class DidMetaPlugin(metaclass=ABCMeta):
63
77
  value: str,
64
78
  recursive: bool = False,
65
79
  *,
66
- session: "Optional[Session]" = None
80
+ session: "Session | None" = None
67
81
  ) -> None:
68
82
  """
69
83
  Add metadata to data identifier.
70
84
 
71
- :param scope: The scope name.
85
+ :param scope: The scope of the DID.
72
86
  :param name: The data identifier name.
73
87
  :param key: the key.
74
88
  :param value: the value.
75
- :param did: The data identifier info.
76
- :param recursive: Option to propagate the metadata change to content.
89
+ :param recursive: Instruction to propagate the metadata change recursively to content (False by default).
77
90
  :param session: The database session in use.
78
91
  """
79
92
  pass
80
93
 
81
- @transactional_session
94
+ @abstractmethod
82
95
  def set_metadata_bulk(
83
96
  self,
84
97
  scope: "InternalScope",
@@ -86,16 +99,16 @@ class DidMetaPlugin(metaclass=ABCMeta):
86
99
  meta: dict[str, "Any"],
87
100
  recursive: bool = False,
88
101
  *,
89
- session: "Optional[Session]" = None
102
+ session: "Session | None" = None
90
103
  ) -> None:
91
104
  """
92
105
  Add metadata to data identifier in bulk.
93
106
 
94
- :param scope: The scope name.
107
+ :param scope: The scope of the DID.
95
108
  :param name: The data identifier name.
96
109
  :param meta: all key-values to set.
97
110
  :type meta: dict
98
- :param recursive: Option to propagate the metadata change to content.
111
+ :param recursive: Instruction to propagate the metadata change recursively to content (False by default).
99
112
  :param session: The database session in use.
100
113
  """
101
114
  for key, value in meta.items():
@@ -108,14 +121,14 @@ class DidMetaPlugin(metaclass=ABCMeta):
108
121
  name: str,
109
122
  key: str,
110
123
  *,
111
- session: "Optional[Session]" = None
124
+ session: "Session | None" = None
112
125
  ) -> None:
113
126
  """
114
127
  Deletes the metadata stored for the given key.
115
128
 
116
129
  :param scope: The scope of the DID.
117
130
  :param name: The name of the DID.
118
- :param key: Key of the metadata.
131
+ :param key: The key to be deleted.
119
132
  :param session: The database session in use.
120
133
  """
121
134
  pass
@@ -125,19 +138,19 @@ class DidMetaPlugin(metaclass=ABCMeta):
125
138
  self,
126
139
  scope: "InternalScope",
127
140
  filters: dict[str, "Any"],
128
- did_type: Literal['all', 'collection', 'dataset', 'container', 'file'] = 'collection',
141
+ did_type: "Literal['all', 'collection', 'dataset', 'container', 'file']" = 'collection',
129
142
  ignore_case: bool = False,
130
- limit: "Optional[int]" = None,
131
- offset: "Optional[int]" = None,
143
+ limit: "int | None" = None,
144
+ offset: "int | None" = None,
132
145
  long: bool = False,
133
146
  recursive: bool = False,
134
147
  *,
135
- session: "Optional[Session]" = None
136
- ) -> "Iterator[Union[str, dict[str, Any]]]":
148
+ session: "Session | None" = None
149
+ ) -> "Iterator[str | dict[str, Any]]":
137
150
  """
138
151
  Search data identifiers
139
152
 
140
- :param scope: the scope name.
153
+ :param scope: The scope of the DID.
141
154
  :param filters: dictionary of attributes by which the results should be filtered.
142
155
  :param did_type: the type of the DID: all(container, dataset, file), collection(dataset or container), dataset, container, file.
143
156
  :param ignore_case: ignore case distinctions.
@@ -154,12 +167,13 @@ class DidMetaPlugin(metaclass=ABCMeta):
154
167
  self,
155
168
  key: str,
156
169
  *,
157
- session: "Optional[Session]" = None
170
+ session: "Session | None" = None
158
171
  ) -> bool:
159
172
  """
160
173
  Returns whether key is managed by this plugin or not.
174
+
161
175
  :param key: Key of the metadata.
162
176
  :param session: The database session in use.
163
- :returns (Boolean)
177
+ :returns: (Boolean)
164
178
  """
165
179
  pass
@@ -91,7 +91,7 @@ class ElasticDidMeta(DidMetaPlugin):
91
91
  })
92
92
 
93
93
  self.client = Elasticsearch(**self.es_config)
94
- self.plugin_name = "ELASTIC"
94
+ self._plugin_name = "ELASTIC"
95
95
 
96
96
  def drop_index(self) -> None:
97
97
  self.client.indices.delete(index=self.index)
@@ -188,7 +188,7 @@ class ElasticDidMeta(DidMetaPlugin):
188
188
  raise exception.RucioException(err)
189
189
 
190
190
  if recursive:
191
- raise exception.UnsupportedOperation(f"'{self.plugin_name.lower()}' metadata module does not currently support recursive inserts of metadata")
191
+ raise exception.UnsupportedOperation(f"'{self.name}' metadata module does not currently support recursive inserts of metadata")
192
192
 
193
193
  def delete_metadata(
194
194
  self,
@@ -330,7 +330,7 @@ class ElasticDidMeta(DidMetaPlugin):
330
330
  self.client.close_point_in_time(body={"id": pit_id})
331
331
 
332
332
  if recursive:
333
- raise exception.UnsupportedOperation(f"'{self.plugin_name.lower()}' metadata module does not currently support recursive searches")
333
+ raise exception.UnsupportedOperation(f"'{self.name}' metadata module does not currently support recursive searches")
334
334
 
335
335
  def on_delete(
336
336
  self,
@@ -397,11 +397,3 @@ class ElasticDidMeta(DidMetaPlugin):
397
397
  session: "Optional[Session]" = None
398
398
  ) -> bool:
399
399
  return True
400
-
401
- def get_plugin_name(self) -> str:
402
- """
403
- Returns a unique identifier for this plugin. This can be later used for filtering down results to this plugin only.
404
-
405
- :returns: The name of the plugin
406
- """
407
- return self.plugin_name