rucio 37.0.0rc4__py3-none-any.whl → 37.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/cli/account.py +14 -14
- rucio/cli/command.py +9 -9
- rucio/cli/config.py +3 -3
- rucio/cli/did.py +13 -13
- rucio/cli/lifetime_exception.py +1 -1
- rucio/cli/replica.py +3 -3
- rucio/cli/rse.py +18 -18
- rucio/cli/rule.py +5 -5
- rucio/cli/scope.py +2 -2
- rucio/cli/subscription.py +4 -4
- rucio/client/baseclient.py +0 -3
- rucio/client/lifetimeclient.py +46 -13
- rucio/common/config.py +0 -26
- rucio/common/utils.py +14 -17
- rucio/core/account_limit.py +56 -79
- rucio/core/did_meta_plugins/filter_engine.py +1 -3
- rucio/core/rse_selector.py +3 -3
- rucio/core/rule_grouping.py +0 -1
- rucio/gateway/account_limit.py +40 -76
- rucio/transfertool/fts3.py +1 -1
- rucio/vcsversion.py +4 -4
- rucio/web/rest/flaskapi/v1/accounts.py +3 -9
- rucio/web/rest/flaskapi/v1/auth.py +3 -3
- rucio/web/rest/flaskapi/v1/common.py +10 -4
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-replica-recoverer +1 -1
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/METADATA +1 -1
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/RECORD +85 -85
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/requirements.server.txt +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-abacus-account +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-admin +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-atropos +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-auditor +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-automatix +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-bb8 +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-cache-client +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-receiver +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-dumper +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-follower +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-hermes +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-judge-injector +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-kronos +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-minos +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-necromancer +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-reaper +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-transmogrifier +0 -0
- {rucio-37.0.0rc4.data → rucio-37.1.0.data}/scripts/rucio-undertaker +0 -0
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/WHEEL +0 -0
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-37.0.0rc4.dist-info → rucio-37.1.0.dist-info}/top_level.txt +0 -0
rucio/client/lifetimeclient.py
CHANGED
|
@@ -39,10 +39,32 @@ class LifetimeClient(BaseClient):
|
|
|
39
39
|
states: Optional['Sequence[LifetimeExceptionsState]'] = None
|
|
40
40
|
) -> 'Iterator[dict[str, Any]]':
|
|
41
41
|
"""
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
Lists lifetime model exceptions that allow extending data lifetimes beyond their configured policies.
|
|
43
|
+
|
|
44
|
+
The lifetime model exceptions are used to override the default lifecycle policies for data identifiers
|
|
45
|
+
(files, datasets, containers, or archives) that need to be kept longer than usual. These exceptions
|
|
46
|
+
can be filtered by their ID or approval state (this feature is not available yet).
|
|
47
|
+
|
|
48
|
+
:param exception_id: The unique identifier of a specific exception. If provided, returns only that exception.
|
|
49
|
+
:param states: Filter exceptions by their states. Possible values are:
|
|
50
|
+
- `A` (APPROVED): Exception was approved
|
|
51
|
+
- `R` (REJECTED): Exception was rejected
|
|
52
|
+
- `W` (WAITING): Exception is waiting for approval by an admin (or other authorized account)
|
|
53
|
+
|
|
54
|
+
:returns:
|
|
55
|
+
An iterator of dictionaries containing the exception details:
|
|
56
|
+
- `id`: The unique identifier of the exception
|
|
57
|
+
- `scope`: The scope of the data identifier
|
|
58
|
+
- `name`: The name of the data identifier
|
|
59
|
+
- `did_type`: Type of the data identifier:
|
|
60
|
+
`F` (file), `D` (dataset), `C` (container), `A` (archive),
|
|
61
|
+
`X` (deleted file), `Y` (deleted dataset), `Z` (deleted container)
|
|
62
|
+
- `account`: The account that requested the exception
|
|
63
|
+
- `pattern`: Pattern used for matching data identifiers
|
|
64
|
+
- `comments`: User provided comments explaining the exception
|
|
65
|
+
- `state`: Current state of the exception
|
|
66
|
+
- `created_at`: When the exception was created (returned as timestamp string)
|
|
67
|
+
- `expires_at`: When the exception expires (returned as timestamp string)
|
|
46
68
|
"""
|
|
47
69
|
|
|
48
70
|
path = self.LIFETIME_BASEURL + '/'
|
|
@@ -70,16 +92,27 @@ class LifetimeClient(BaseClient):
|
|
|
70
92
|
expires_at: 'datetime'
|
|
71
93
|
) -> dict[str, Any]:
|
|
72
94
|
"""
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
:param
|
|
80
|
-
|
|
81
|
-
|
|
95
|
+
Creates a lifetime model exception request to extend the expiration date of data identifiers (DIDs).
|
|
96
|
+
|
|
97
|
+
These exceptions allow requesting extensions to DIDs' lifetimes, subject to approval and configured
|
|
98
|
+
maximum extension periods. The request includes details about which DIDs should have extended
|
|
99
|
+
lifetimes, who is requesting it, and why it's needed.
|
|
100
|
+
|
|
101
|
+
:param dids: List of dictionaries containing the data identifiers to be excepted.
|
|
102
|
+
Each dictionary must contain:
|
|
103
|
+
- `scope`: The scope of the data identifier
|
|
104
|
+
- `name`: The name of the data identifier
|
|
105
|
+
:param account: The account requesting the exception
|
|
106
|
+
:param pattern: Associated pattern for the exception request
|
|
107
|
+
:param comments: Justification for why the exception is needed (e.g. "Needed for my XYZ analysis..")
|
|
108
|
+
:param expires_at: When the exception should expire (datetime object)
|
|
109
|
+
|
|
110
|
+
:returns: A dictionary containing:
|
|
111
|
+
- `exceptions`: Dictionary mapping exception IDs to lists of DIDs that were successfully added
|
|
112
|
+
- `unknown`: List of DIDs that could not be found
|
|
113
|
+
- `not_affected`: List of DIDs that did not qualify for an exception
|
|
82
114
|
"""
|
|
115
|
+
|
|
83
116
|
path = self.LIFETIME_BASEURL + '/'
|
|
84
117
|
url = build_url(choice(self.list_hosts), path=path)
|
|
85
118
|
data = {'dids': dids, 'account': account, 'pattern': pattern, 'comments': comments, 'expires_at': expires_at}
|
rucio/common/config.py
CHANGED
|
@@ -31,8 +31,6 @@ if TYPE_CHECKING:
|
|
|
31
31
|
|
|
32
32
|
from sqlalchemy.orm import Session
|
|
33
33
|
|
|
34
|
-
LEGACY_SECTION_NAME = {}
|
|
35
|
-
LEGACY_OPTION_NAME = {}
|
|
36
34
|
|
|
37
35
|
|
|
38
36
|
def convert_to_any_type(value: str) -> Union[bool, int, float, str]:
|
|
@@ -194,12 +192,6 @@ def config_get(
|
|
|
194
192
|
try:
|
|
195
193
|
return convert_type_fnc(get_config().get(section, option))
|
|
196
194
|
except (configparser.NoOptionError, configparser.NoSectionError, ConfigNotFound) as err:
|
|
197
|
-
try:
|
|
198
|
-
legacy_config = get_legacy_config(section, option)
|
|
199
|
-
if legacy_config is not None:
|
|
200
|
-
return convert_type_fnc(legacy_config)
|
|
201
|
-
except ConfigNotFound:
|
|
202
|
-
pass
|
|
203
195
|
|
|
204
196
|
from rucio.common.client import is_client
|
|
205
197
|
client_mode = is_client()
|
|
@@ -220,24 +212,6 @@ def config_get(
|
|
|
220
212
|
return default
|
|
221
213
|
|
|
222
214
|
|
|
223
|
-
def get_legacy_config(section: str, option: str):
|
|
224
|
-
"""
|
|
225
|
-
Returns a legacy config value, if it is present.
|
|
226
|
-
|
|
227
|
-
:param section: The section of the new config.
|
|
228
|
-
:param option: The option of the new config.
|
|
229
|
-
:returns: The string value of the legacy option if one is found, None otherwise.
|
|
230
|
-
"""
|
|
231
|
-
|
|
232
|
-
section = LEGACY_SECTION_NAME.get(section, section)
|
|
233
|
-
option = LEGACY_OPTION_NAME.get(option, option)
|
|
234
|
-
|
|
235
|
-
if config_has_option(section, option):
|
|
236
|
-
return get_config().get(section, option)
|
|
237
|
-
|
|
238
|
-
return None
|
|
239
|
-
|
|
240
|
-
|
|
241
215
|
def config_has_section(section: str) -> bool:
|
|
242
216
|
"""
|
|
243
217
|
Indicates whether the named section is present in the configuration. The DEFAULT section is not acknowledged.
|
rucio/common/utils.py
CHANGED
|
@@ -852,27 +852,24 @@ def get_bytes_value_from_string(input_string: str) -> Union[bool, int]:
|
|
|
852
852
|
:param input_string: String containing a value and an unit
|
|
853
853
|
:return: Integer value representing the value in bytes
|
|
854
854
|
"""
|
|
855
|
-
|
|
855
|
+
unit_multipliers = {
|
|
856
|
+
'b': 1,
|
|
857
|
+
'kb': 10**3,
|
|
858
|
+
'mb': 10**6,
|
|
859
|
+
'gb': 10**9,
|
|
860
|
+
'tb': 10**12,
|
|
861
|
+
'pb': 10**15,
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
result = re.findall(r'^([0-9]+)([A-Za-z]+)$', input_string)
|
|
856
865
|
if result:
|
|
857
866
|
value = int(result[0][0])
|
|
858
867
|
unit = result[0][1].lower()
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
elif unit == 'kb':
|
|
862
|
-
value = value * 1000
|
|
863
|
-
elif unit == 'mb':
|
|
864
|
-
value = value * 1000000
|
|
865
|
-
elif unit == 'gb':
|
|
866
|
-
value = value * 1000000000
|
|
867
|
-
elif unit == 'tb':
|
|
868
|
-
value = value * 1000000000000
|
|
869
|
-
elif unit == 'pb':
|
|
870
|
-
value = value * 1000000000000000
|
|
871
|
-
else:
|
|
868
|
+
multiplier = unit_multipliers.get(unit)
|
|
869
|
+
if multiplier is None:
|
|
872
870
|
return False
|
|
873
|
-
return value
|
|
874
|
-
|
|
875
|
-
return False
|
|
871
|
+
return value * multiplier
|
|
872
|
+
return False
|
|
876
873
|
|
|
877
874
|
|
|
878
875
|
def parse_did_filter_from_string(input_string: str) -> tuple[dict[str, Any], str]:
|
rucio/core/account_limit.py
CHANGED
|
@@ -18,13 +18,16 @@ from sqlalchemy.exc import NoResultFound
|
|
|
18
18
|
from sqlalchemy.sql import func, literal, select
|
|
19
19
|
from sqlalchemy.sql.expression import and_, or_
|
|
20
20
|
|
|
21
|
-
from rucio.
|
|
21
|
+
from rucio.common.exception import AccountNotFound
|
|
22
|
+
from rucio.core.account import account_exists, get_all_rse_usages_per_account
|
|
22
23
|
from rucio.core.rse import get_rse_name
|
|
23
24
|
from rucio.core.rse_expression_parser import parse_expression
|
|
24
25
|
from rucio.db.sqla import models
|
|
25
26
|
from rucio.db.sqla.session import read_session, transactional_session
|
|
26
27
|
|
|
27
28
|
if TYPE_CHECKING:
|
|
29
|
+
from collections.abc import Iterable
|
|
30
|
+
|
|
28
31
|
from sqlalchemy.orm import Session
|
|
29
32
|
|
|
30
33
|
from rucio.common.types import InternalAccount, RSEAccountUsageDict, RSEGlobalAccountUsageDict, RSELocalAccountUsageDict, RSEResolvedGlobalAccountLimitDict
|
|
@@ -91,26 +94,39 @@ def get_rse_account_usage(rse_id: str, *, session: "Session") -> list["RSEAccoun
|
|
|
91
94
|
|
|
92
95
|
|
|
93
96
|
@read_session
|
|
94
|
-
def
|
|
97
|
+
def get_global_account_limit(account: Optional["InternalAccount"] = None, rse_expression: Optional[str] = None, *,
|
|
98
|
+
session: "Session") -> Optional[Union[int, float, dict[str, "RSEResolvedGlobalAccountLimitDict"]]]:
|
|
95
99
|
"""
|
|
96
|
-
Returns the global account
|
|
100
|
+
Returns the global account limit for the given account and RSE expression.
|
|
101
|
+
If no RSE expression is provided, returns all limits for the given account.
|
|
97
102
|
|
|
98
|
-
:
|
|
99
|
-
|
|
100
|
-
:
|
|
103
|
+
Note: The account parameter must always be specified when fetching global account limit for an account.
|
|
104
|
+
|
|
105
|
+
:param account: Account to check the limit for (required unless fetching all limits for all accounts is supported).
|
|
106
|
+
:param rse_expression: Specific RSE expression to check the limit for (optional).
|
|
107
|
+
:param session: Database session in use.
|
|
108
|
+
:return: Limit in Bytes for a single RSE expression, or a dictionary of all limits {'MOCK': {'resolved_rses': ['MOCK'], 'limit': 10, 'resolved_rse_ids': [123]}}.
|
|
101
109
|
"""
|
|
102
110
|
if account:
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
111
|
+
if not account_exists(account, session=session):
|
|
112
|
+
raise AccountNotFound(f"Account {account} does not exist")
|
|
113
|
+
if rse_expression:
|
|
114
|
+
# Fetch limit for a single RSE expression
|
|
115
|
+
try:
|
|
116
|
+
stmt = select(models.AccountGlobalLimit).where(
|
|
117
|
+
and_(models.AccountGlobalLimit.account == account,
|
|
118
|
+
models.AccountGlobalLimit.rse_expression == rse_expression)
|
|
119
|
+
)
|
|
120
|
+
global_account_limit = session.execute(stmt).scalar_one()
|
|
121
|
+
return float("inf") if global_account_limit.bytes == -1 else global_account_limit.bytes
|
|
122
|
+
except NoResultFound:
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
# Fetch all global limits for the account (or all accounts if no account specified)
|
|
126
|
+
stmt = select(models.AccountGlobalLimit)
|
|
127
|
+
if account:
|
|
128
|
+
stmt = stmt.where(models.AccountGlobalLimit.account == account)
|
|
129
|
+
global_account_limits = session.execute(stmt).scalars().all()
|
|
114
130
|
|
|
115
131
|
resolved_global_account_limits = {}
|
|
116
132
|
for limit in global_account_limits:
|
|
@@ -118,9 +134,7 @@ def get_global_account_limits(account: Optional["InternalAccount"] = None, *, se
|
|
|
118
134
|
resolved_rses = parse_expression(limit['rse_expression'], filter_={'vo': account.vo}, session=session)
|
|
119
135
|
else:
|
|
120
136
|
resolved_rses = parse_expression(limit['rse_expression'], session=session)
|
|
121
|
-
limit_in_bytes = limit['bytes']
|
|
122
|
-
if limit_in_bytes == -1:
|
|
123
|
-
limit_in_bytes = float('inf')
|
|
137
|
+
limit_in_bytes = float('inf') if limit['bytes'] == -1 else limit['bytes']
|
|
124
138
|
resolved_global_account_limits[limit['rse_expression']] = {
|
|
125
139
|
'resolved_rses': [resolved_rse['rse'] for resolved_rse in resolved_rses],
|
|
126
140
|
'resolved_rse_ids': [resolved_rse['id'] for resolved_rse in resolved_rses],
|
|
@@ -130,69 +144,32 @@ def get_global_account_limits(account: Optional["InternalAccount"] = None, *, se
|
|
|
130
144
|
|
|
131
145
|
|
|
132
146
|
@read_session
|
|
133
|
-
def
|
|
147
|
+
def get_local_account_limit(account: "InternalAccount", rse_ids: Optional[Union[str, "Iterable[str]"]] = None, *, session: "Session") -> Optional[Union[int, float, dict[str, int]]]:
|
|
134
148
|
"""
|
|
135
|
-
Returns the
|
|
136
|
-
|
|
137
|
-
:param account: Account to check the limit for.
|
|
138
|
-
:param rse_expression: RSE expression to check the limit for.
|
|
139
|
-
:param session: Database session in use.
|
|
140
|
-
:return: Limit in Bytes.
|
|
141
|
-
"""
|
|
142
|
-
try:
|
|
143
|
-
stmt = select(
|
|
144
|
-
models.AccountGlobalLimit
|
|
145
|
-
).where(
|
|
146
|
-
and_(models.AccountGlobalLimit.account == account,
|
|
147
|
-
models.AccountGlobalLimit.rse_expression == rse_expression)
|
|
148
|
-
)
|
|
149
|
-
global_account_limit = session.execute(stmt).scalar_one()
|
|
150
|
-
if global_account_limit.bytes == -1:
|
|
151
|
-
return float("inf")
|
|
152
|
-
else:
|
|
153
|
-
return global_account_limit.bytes
|
|
154
|
-
except NoResultFound:
|
|
155
|
-
return None
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
@read_session
|
|
159
|
-
def get_local_account_limit(account: "InternalAccount", rse_id: str, *, session: "Session") -> Union[int, float, None]:
|
|
160
|
-
"""
|
|
161
|
-
Returns the account limit for the account on the rse.
|
|
162
|
-
|
|
163
|
-
:param account: Account to check the limit for.
|
|
164
|
-
:param rse_id: RSE id to check the limit for.
|
|
165
|
-
:param session: Database session in use.
|
|
166
|
-
:return: Limit in Bytes.
|
|
167
|
-
"""
|
|
168
|
-
try:
|
|
169
|
-
stmt = select(
|
|
170
|
-
models.AccountLimit
|
|
171
|
-
).where(
|
|
172
|
-
and_(models.AccountLimit.account == account,
|
|
173
|
-
models.AccountLimit.rse_id == rse_id)
|
|
174
|
-
)
|
|
175
|
-
account_limit = session.execute(stmt).scalar_one()
|
|
176
|
-
if account_limit.bytes == -1:
|
|
177
|
-
return float("inf")
|
|
178
|
-
else:
|
|
179
|
-
return account_limit.bytes
|
|
180
|
-
except NoResultFound:
|
|
181
|
-
return None
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
@read_session
|
|
185
|
-
def get_local_account_limits(account: "InternalAccount", rse_ids: Optional[list[str]] = None, *, session: "Session") -> dict[str, int]:
|
|
186
|
-
"""
|
|
187
|
-
Returns the account limits for the account on the list of rses.
|
|
149
|
+
Returns the local account limit for a given RSE or list of RSEs.
|
|
188
150
|
|
|
189
151
|
:param account: Account to check the limit for.
|
|
190
|
-
:param rse_ids:
|
|
152
|
+
:param rse_ids: Single RSE id or an iterable of RSE ids to check the limit for.
|
|
191
153
|
:param session: Database session in use.
|
|
192
|
-
:return:
|
|
154
|
+
:return: Limit in Bytes (int/float) for a single RSE or
|
|
155
|
+
Dictionary {'rse_id': bytes, ...} for multiple RSEs.
|
|
193
156
|
"""
|
|
157
|
+
if not account_exists(account, session=session):
|
|
158
|
+
raise AccountNotFound(f"Account {account} does not exist")
|
|
159
|
+
if isinstance(rse_ids, str): # Single RSE case
|
|
160
|
+
try:
|
|
161
|
+
stmt = select(models.AccountLimit).where(
|
|
162
|
+
and_(models.AccountLimit.account == account, models.AccountLimit.rse_id == rse_ids)
|
|
163
|
+
)
|
|
164
|
+
account_limit = session.execute(stmt).scalar_one()
|
|
165
|
+
return float("inf") if account_limit.bytes == -1 else account_limit.bytes
|
|
166
|
+
except NoResultFound:
|
|
167
|
+
return None
|
|
194
168
|
|
|
169
|
+
# Multiple RSE case or no RSE specified
|
|
195
170
|
account_limits = {}
|
|
171
|
+
|
|
172
|
+
# If rse_ids is a list of RSEs
|
|
196
173
|
if rse_ids:
|
|
197
174
|
rse_id_clauses = []
|
|
198
175
|
for rse_id in rse_ids:
|
|
@@ -339,14 +316,14 @@ def get_local_account_usage(account: "InternalAccount", rse_id: Optional[str] =
|
|
|
339
316
|
)
|
|
340
317
|
if not rse_id:
|
|
341
318
|
# All RSESs
|
|
342
|
-
limits =
|
|
319
|
+
limits = get_local_account_limit(account=account, session=session)
|
|
343
320
|
counters = {c.rse_id: c for c in session.execute(stmt).scalars().all()}
|
|
344
321
|
else:
|
|
345
322
|
# One RSE
|
|
346
323
|
stmt.where(
|
|
347
324
|
models.AccountUsage.rse_id == rse_id
|
|
348
325
|
)
|
|
349
|
-
limits =
|
|
326
|
+
limits = get_local_account_limit(account=account, rse_ids=[rse_id], session=session)
|
|
350
327
|
counters = {c.rse_id: c for c in session.execute(stmt).scalars().all()}
|
|
351
328
|
result_list = []
|
|
352
329
|
|
|
@@ -385,7 +362,7 @@ def get_global_account_usage(account: "InternalAccount", rse_expression: Optiona
|
|
|
385
362
|
result_list = []
|
|
386
363
|
if not rse_expression:
|
|
387
364
|
# All RSE Expressions
|
|
388
|
-
limits =
|
|
365
|
+
limits = get_global_account_limit(account=account, session=session)
|
|
389
366
|
all_rse_usages = {usage['rse_id']: (usage['bytes'], usage['files']) for usage in get_all_rse_usages_per_account(account=account, session=session)}
|
|
390
367
|
for rse_expression, limit in limits.items():
|
|
391
368
|
usage = 0
|
|
@@ -102,9 +102,7 @@ class FilterEngine:
|
|
|
102
102
|
filters, _ = parse_did_filter_from_string_fe(filters, omit_name=True)
|
|
103
103
|
elif isinstance(filters, dict):
|
|
104
104
|
filters = [filters]
|
|
105
|
-
elif isinstance(filters, list):
|
|
106
|
-
filters = filters
|
|
107
|
-
else:
|
|
105
|
+
elif not isinstance(filters, list):
|
|
108
106
|
raise exception.DIDFilterSyntaxError("Input filters are of an unrecognised type.")
|
|
109
107
|
|
|
110
108
|
filters = self._make_input_backwards_compatible(filters=filters)
|
rucio/core/rse_selector.py
CHANGED
|
@@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Optional
|
|
|
17
17
|
|
|
18
18
|
from rucio.common.exception import InsufficientAccountLimit, InsufficientTargetRSEs, InvalidRuleWeight, RSEOverQuota
|
|
19
19
|
from rucio.core.account import get_all_rse_usages_per_account, get_usage, has_account_attribute
|
|
20
|
-
from rucio.core.account_limit import
|
|
20
|
+
from rucio.core.account_limit import get_global_account_limit, get_local_account_limit
|
|
21
21
|
from rucio.core.rse import get_rse_limits, has_rse_attribute, list_rse_attributes
|
|
22
22
|
from rucio.core.rse_counter import get_counter as get_rse_counter
|
|
23
23
|
from rucio.core.rse_expression_parser import parse_expression
|
|
@@ -86,7 +86,7 @@ class RSESelector:
|
|
|
86
86
|
rse['space_left'] = float('inf')
|
|
87
87
|
rses_with_enough_quota.append(rse)
|
|
88
88
|
else:
|
|
89
|
-
global_quota_limit =
|
|
89
|
+
global_quota_limit = get_global_account_limit(account=account, session=session)
|
|
90
90
|
all_rse_usages = {usage['rse_id']: usage['bytes'] for usage in get_all_rse_usages_per_account(account=account, session=session)}
|
|
91
91
|
for rse in self.rses:
|
|
92
92
|
if rse['mock_rse']:
|
|
@@ -96,7 +96,7 @@ class RSESelector:
|
|
|
96
96
|
else:
|
|
97
97
|
# check local quota
|
|
98
98
|
local_quota_left = None
|
|
99
|
-
quota_limit = get_local_account_limit(account=account,
|
|
99
|
+
quota_limit = get_local_account_limit(account=account, rse_ids=rse['rse_id'], session=session)
|
|
100
100
|
if quota_limit is None:
|
|
101
101
|
local_quota_left = 0
|
|
102
102
|
else:
|
rucio/core/rule_grouping.py
CHANGED
|
@@ -1617,4 +1617,3 @@ def apply_rule(did, rule, rses, source_rses, rseselector, *, session: "Session",
|
|
|
1617
1617
|
account_counter.increase(rse_id=rse_id, account=rule.account, files=account_counters_files[rse_id], bytes_=account_counters_bytes[rse_id], session=session)
|
|
1618
1618
|
session.flush()
|
|
1619
1619
|
|
|
1620
|
-
return
|
rucio/gateway/account_limit.py
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
-
from typing import TYPE_CHECKING, Any, Union
|
|
15
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
16
16
|
|
|
17
17
|
import rucio.common.exception
|
|
18
18
|
import rucio.gateway.permission
|
|
@@ -47,110 +47,74 @@ def get_rse_account_usage(
|
|
|
47
47
|
return [gateway_update_return_dict(d, session=session) for d in account_limit_core.get_rse_account_usage(rse_id=rse_id, session=session)]
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
@read_session
|
|
51
|
-
def get_local_account_limits(
|
|
52
|
-
account: str,
|
|
53
|
-
vo: str = 'def',
|
|
54
|
-
*,
|
|
55
|
-
session: "Session"
|
|
56
|
-
) -> dict[str, Any]:
|
|
57
|
-
"""
|
|
58
|
-
Lists the limitation names/values for the specified account name.
|
|
59
|
-
|
|
60
|
-
REST API: http://<host>:<port>/rucio/account/<account>/limits
|
|
61
|
-
|
|
62
|
-
:param account: The account name.
|
|
63
|
-
:param vo: The VO to act on.
|
|
64
|
-
:param session: The database session in use.
|
|
65
|
-
|
|
66
|
-
:returns: The account limits.
|
|
67
|
-
"""
|
|
68
|
-
|
|
69
|
-
internal_account = InternalAccount(account, vo=vo)
|
|
70
|
-
|
|
71
|
-
rse_instead_id = {}
|
|
72
|
-
for elem in account_limit_core.get_local_account_limits(account=internal_account, session=session).items():
|
|
73
|
-
rse_instead_id[get_rse_name(rse_id=elem[0], session=session)] = elem[1]
|
|
74
|
-
return rse_instead_id
|
|
75
|
-
|
|
76
|
-
|
|
77
50
|
@read_session
|
|
78
51
|
def get_local_account_limit(
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
52
|
+
account: str,
|
|
53
|
+
rse: Optional[str] = None,
|
|
54
|
+
vo: str = 'def',
|
|
55
|
+
*,
|
|
56
|
+
session: "Session"
|
|
84
57
|
) -> dict[str, Union[int, float, None]]:
|
|
85
58
|
"""
|
|
86
|
-
Lists the limitation names/values for the specified account name
|
|
59
|
+
Lists the limitation names/values for the specified account name.
|
|
60
|
+
If an RSE is provided, it returns the limit for that specific RSE.
|
|
61
|
+
Otherwise, it returns all account limits.
|
|
87
62
|
|
|
88
63
|
REST API: http://<host>:<port>/rucio/account/<account>/limits
|
|
89
64
|
|
|
90
|
-
:param account:
|
|
91
|
-
:param rse:
|
|
92
|
-
:param vo:
|
|
93
|
-
:param session:
|
|
65
|
+
:param account: The account name.
|
|
66
|
+
:param rse: The RSE name (optional).
|
|
67
|
+
:param vo: The VO to act on.
|
|
68
|
+
:param session: The database session in use.
|
|
94
69
|
|
|
95
|
-
:returns:
|
|
70
|
+
:returns: A dictionary of account limits with RSE names as keys and limits as values.
|
|
96
71
|
"""
|
|
97
72
|
|
|
98
73
|
internal_account = InternalAccount(account, vo=vo)
|
|
99
74
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
@read_session
|
|
105
|
-
def get_global_account_limits(
|
|
106
|
-
account: str,
|
|
107
|
-
vo: str = 'def',
|
|
108
|
-
*,
|
|
109
|
-
session: "Session"
|
|
110
|
-
) -> dict[str, RSEResolvedGlobalAccountLimitDict]:
|
|
111
|
-
"""
|
|
112
|
-
Lists the limitation names/values for the specified account name.
|
|
113
|
-
|
|
114
|
-
REST API: http://<host>:<port>/rucio/account/<account>/limits
|
|
115
|
-
|
|
116
|
-
:param account: The account name.
|
|
117
|
-
:param vo: The VO to act on.
|
|
118
|
-
:param session: The database session in use.
|
|
119
|
-
|
|
120
|
-
:returns: The account limits.
|
|
121
|
-
"""
|
|
122
|
-
if account:
|
|
123
|
-
internal_account = InternalAccount(account, vo=vo)
|
|
75
|
+
if rse:
|
|
76
|
+
# Single RSE lookup
|
|
77
|
+
rse_id = get_rse_id(rse=rse, vo=vo, session=session)
|
|
78
|
+
return {rse: account_limit_core.get_local_account_limit(account=internal_account, rse_ids=rse_id, session=session)}
|
|
124
79
|
else:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
80
|
+
# Fetch all RSE limits
|
|
81
|
+
limits = account_limit_core.get_local_account_limit(account=internal_account, rse_ids=None, session=session)
|
|
82
|
+
return {get_rse_name(rse_id=rse_id, session=session): limit for rse_id, limit in limits.items()}
|
|
128
83
|
|
|
129
84
|
|
|
130
85
|
@read_session
|
|
131
86
|
def get_global_account_limit(
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
) -> dict[str, dict[str, RSEResolvedGlobalAccountLimitDict]]:
|
|
87
|
+
account: str,
|
|
88
|
+
rse_expression: Optional[str] = None,
|
|
89
|
+
vo: str = 'def',
|
|
90
|
+
*,
|
|
91
|
+
session: "Session"
|
|
92
|
+
) -> Union[dict[str, RSEResolvedGlobalAccountLimitDict], dict[str, dict[str, RSEResolvedGlobalAccountLimitDict]]]:
|
|
138
93
|
"""
|
|
139
|
-
Lists the limitation names/values for the specified account
|
|
94
|
+
Lists the global account limitation names/values for the specified account.
|
|
95
|
+
If an RSE expression is provided, fetches the limit for that expression.
|
|
96
|
+
Otherwise, fetches all global limits for the account.
|
|
140
97
|
|
|
141
98
|
REST API: http://<host>:<port>/rucio/account/<account>/limits
|
|
142
99
|
|
|
143
100
|
:param account: The account name.
|
|
144
|
-
:param rse_expression: The
|
|
101
|
+
:param rse_expression: The RSE expression (optional; if not provided, all limits will be fetched).
|
|
145
102
|
:param vo: The VO to act on.
|
|
146
103
|
:param session: The database session in use.
|
|
147
104
|
|
|
148
|
-
:returns:
|
|
105
|
+
:returns:
|
|
106
|
+
- If `rse_expression` is provided: `{rse_expression: {...}}`
|
|
107
|
+
- If `rse_expression` is not provided: `{...}` (dictionary of all limits).
|
|
149
108
|
"""
|
|
150
109
|
|
|
151
110
|
internal_account = InternalAccount(account, vo=vo)
|
|
152
111
|
|
|
153
|
-
|
|
112
|
+
if rse_expression:
|
|
113
|
+
return {rse_expression: account_limit_core.get_global_account_limit(
|
|
114
|
+
account=internal_account, rse_expression=rse_expression, session=session
|
|
115
|
+
)}
|
|
116
|
+
|
|
117
|
+
return account_limit_core.get_global_account_limit(account=internal_account, session=session)
|
|
154
118
|
|
|
155
119
|
|
|
156
120
|
@transactional_session
|
rucio/transfertool/fts3.py
CHANGED
|
@@ -607,7 +607,7 @@ class Fts3TransferStatusReport(TransferStatusReport):
|
|
|
607
607
|
'src-type': self._file_metadata.get('src_type'),
|
|
608
608
|
'src-rse': self._src_rse,
|
|
609
609
|
'src-url': self._src_url,
|
|
610
|
-
'dst-type': self._file_metadata.get('
|
|
610
|
+
'dst-type': self._file_metadata.get('dst_type'),
|
|
611
611
|
'dst-rse': self._file_metadata.get('dst_rse'),
|
|
612
612
|
'dst-url': self._dst_url,
|
|
613
613
|
'started_at': self.started_at,
|
rucio/vcsversion.py
CHANGED
|
@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
|
|
|
4
4
|
'''
|
|
5
5
|
VERSION_INFO = {
|
|
6
6
|
'final': True,
|
|
7
|
-
'version': '37.0
|
|
8
|
-
'branch_nick': '
|
|
9
|
-
'revision_id': '
|
|
10
|
-
'revno':
|
|
7
|
+
'version': '37.1.0',
|
|
8
|
+
'branch_nick': 'release-37',
|
|
9
|
+
'revision_id': 'f6fa712c32b14e1d104349a1c2596cdcb4d8d92d',
|
|
10
|
+
'revno': 13640
|
|
11
11
|
}
|
|
@@ -21,7 +21,7 @@ from flask import Flask, Response, jsonify, redirect, request
|
|
|
21
21
|
from rucio.common.exception import AccessDenied, AccountNotFound, CounterNotFound, Duplicate, IdentityError, InvalidObject, RSENotFound, RuleNotFound, ScopeNotFound
|
|
22
22
|
from rucio.common.utils import APIEncoder, render_json
|
|
23
23
|
from rucio.gateway.account import add_account, add_account_attribute, del_account, del_account_attribute, get_account_info, get_usage_history, list_account_attributes, list_accounts, list_identities, update_account
|
|
24
|
-
from rucio.gateway.account_limit import get_global_account_limit,
|
|
24
|
+
from rucio.gateway.account_limit import get_global_account_limit, get_global_account_usage, get_local_account_limit, get_local_account_usage
|
|
25
25
|
from rucio.gateway.identity import add_account_identity, del_account_identity
|
|
26
26
|
from rucio.gateway.rule import list_replication_rules
|
|
27
27
|
from rucio.gateway.scope import add_scope, get_scopes
|
|
@@ -561,10 +561,7 @@ class LocalAccountLimits(ErrorHandlingMethodView):
|
|
|
561
561
|
description: Not Acceptable
|
|
562
562
|
"""
|
|
563
563
|
try:
|
|
564
|
-
|
|
565
|
-
limits = get_local_account_limit(account=account, rse=rse, vo=request.environ.get('vo'))
|
|
566
|
-
else:
|
|
567
|
-
limits = get_local_account_limits(account=account, vo=request.environ.get('vo'))
|
|
564
|
+
limits = get_local_account_limit(account=account, rse=rse, vo=request.environ.get('vo'))
|
|
568
565
|
except RSENotFound as error:
|
|
569
566
|
return generate_http_error_flask(404, error)
|
|
570
567
|
|
|
@@ -609,10 +606,7 @@ class GlobalAccountLimits(ErrorHandlingMethodView):
|
|
|
609
606
|
description: Not Acceptable
|
|
610
607
|
"""
|
|
611
608
|
try:
|
|
612
|
-
|
|
613
|
-
limits = get_global_account_limit(account=account, rse_expression=rse_expression, vo=request.environ.get('vo'))
|
|
614
|
-
else:
|
|
615
|
-
limits = get_global_account_limits(account=account, vo=request.environ.get('vo'))
|
|
609
|
+
limits = get_global_account_limit(account=account, rse_expression=rse_expression, vo=request.environ.get('vo'))
|
|
616
610
|
except RSENotFound as error:
|
|
617
611
|
return generate_http_error_flask(404, error)
|
|
618
612
|
|
|
@@ -22,7 +22,7 @@ from flask import Blueprint, Flask, Response, redirect, render_template, request
|
|
|
22
22
|
from werkzeug.datastructures import Headers
|
|
23
23
|
|
|
24
24
|
from rucio.common.config import config_get
|
|
25
|
-
from rucio.common.exception import AccessDenied, CannotAuthenticate, CannotAuthorize, IdentityError, IdentityNotFound
|
|
25
|
+
from rucio.common.exception import AccessDenied, CannotAuthenticate, CannotAuthorize, ConfigurationError, IdentityError, IdentityNotFound, InvalidRequest
|
|
26
26
|
from rucio.common.extra import import_extras
|
|
27
27
|
from rucio.common.utils import date_to_str
|
|
28
28
|
from rucio.core.authentication import strip_x509_proxy_attributes
|
|
@@ -697,7 +697,7 @@ class TokenOIDC(ErrorHandlingMethodView):
|
|
|
697
697
|
# response.set_cookie('rucio-auth-token-created-at', value=str(time.time()))
|
|
698
698
|
return response
|
|
699
699
|
else:
|
|
700
|
-
return
|
|
700
|
+
return generate_http_error_flask(status_code=400, exc=InvalidRequest.__name__, exc_msg="", headers=headers)
|
|
701
701
|
|
|
702
702
|
|
|
703
703
|
class RefreshOIDC(ErrorHandlingMethodView):
|
|
@@ -1444,7 +1444,7 @@ class SAML(ErrorHandlingMethodView):
|
|
|
1444
1444
|
headers.set('Pragma', 'no-cache')
|
|
1445
1445
|
|
|
1446
1446
|
if not EXTRA_MODULES['onelogin']:
|
|
1447
|
-
return "SAML not configured on the server side.",
|
|
1447
|
+
return generate_http_error_flask(status_code=400, exc=ConfigurationError.__name__, exc_msg="SAML not configured on the server side.", headers=headers)
|
|
1448
1448
|
|
|
1449
1449
|
saml_nameid = request.cookies.get('saml-nameid', default=None)
|
|
1450
1450
|
vo = extract_vo(request.headers)
|