rucio 37.3.0__py3-none-any.whl → 37.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/cli/rule.py +1 -1
- rucio/client/accountclient.py +205 -60
- rucio/client/accountlimitclient.py +84 -25
- rucio/client/baseclient.py +85 -48
- rucio/client/client.py +49 -41
- rucio/client/configclient.py +36 -13
- rucio/client/credentialclient.py +16 -6
- rucio/client/didclient.py +321 -133
- rucio/client/diracclient.py +13 -6
- rucio/client/downloadclient.py +435 -165
- rucio/client/exportclient.py +8 -2
- rucio/client/fileclient.py +10 -3
- rucio/client/importclient.py +4 -1
- rucio/client/lifetimeclient.py +48 -31
- rucio/client/lockclient.py +22 -7
- rucio/client/metaconventionsclient.py +59 -21
- rucio/client/pingclient.py +3 -1
- rucio/client/replicaclient.py +213 -96
- rucio/client/requestclient.py +124 -16
- rucio/client/rseclient.py +385 -160
- rucio/client/ruleclient.py +147 -51
- rucio/client/scopeclient.py +35 -10
- rucio/client/subscriptionclient.py +60 -27
- rucio/client/touchclient.py +16 -7
- rucio/common/constants.py +14 -17
- rucio/common/utils.py +18 -2
- rucio/core/permission/generic.py +40 -1
- rucio/core/replica.py +6 -6
- rucio/core/request.py +2 -2
- rucio/core/rule.py +5 -3
- rucio/core/transfer.py +4 -5
- rucio/daemons/conveyor/throttler.py +2 -1
- rucio/daemons/judge/evaluator.py +1 -1
- rucio/db/sqla/constants.py +3 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -1
- rucio/db/sqla/models.py +1 -1
- rucio/gateway/replica.py +129 -41
- rucio/gateway/request.py +177 -103
- rucio/gateway/subscription.py +90 -108
- rucio/rse/rsemanager.py +2 -2
- rucio/vcsversion.py +3 -3
- rucio/web/rest/flaskapi/v1/accountlimits.py +22 -22
- rucio/web/rest/flaskapi/v1/accounts.py +157 -157
- rucio/web/rest/flaskapi/v1/archives.py +10 -10
- rucio/web/rest/flaskapi/v1/auth.py +106 -106
- rucio/web/rest/flaskapi/v1/config.py +37 -37
- rucio/web/rest/flaskapi/v1/credentials.py +25 -25
- rucio/web/rest/flaskapi/v1/dids.py +381 -381
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +6 -6
- rucio/web/rest/flaskapi/v1/heartbeats.py +14 -14
- rucio/web/rest/flaskapi/v1/identities.py +25 -25
- rucio/web/rest/flaskapi/v1/import.py +19 -19
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +54 -54
- rucio/web/rest/flaskapi/v1/locks.py +60 -60
- rucio/web/rest/flaskapi/v1/meta_conventions.py +29 -29
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +4 -4
- rucio/web/rest/flaskapi/v1/redirect.py +17 -17
- rucio/web/rest/flaskapi/v1/replicas.py +282 -282
- rucio/web/rest/flaskapi/v1/requests.py +424 -229
- rucio/web/rest/flaskapi/v1/rses.py +427 -427
- rucio/web/rest/flaskapi/v1/rules.py +129 -129
- rucio/web/rest/flaskapi/v1/scopes.py +21 -21
- rucio/web/rest/flaskapi/v1/subscriptions.py +122 -122
- rucio/web/rest/flaskapi/v1/traces.py +18 -18
- rucio/web/rest/flaskapi/v1/vos.py +32 -32
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/rucio.cfg.template +0 -1
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -1
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/requirements.server.txt +1 -1
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/METADATA +1 -1
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/RECORD +128 -128
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/WHEEL +1 -1
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-abacus-account +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-admin +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-atropos +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-auditor +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-automatix +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-bb8 +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-cache-client +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-receiver +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-dumper +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-follower +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-hermes +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-judge-injector +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-kronos +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-minos +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-minos-temporary-expiration +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-necromancer +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-reaper +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-transmogrifier +0 -0
- {rucio-37.3.0.data → rucio-37.5.0.data}/scripts/rucio-undertaker +0 -0
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/licenses/AUTHORS.rst +0 -0
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-37.3.0.dist-info → rucio-37.5.0.dist-info}/top_level.txt +0 -0
rucio/core/replica.py
CHANGED
|
@@ -42,7 +42,7 @@ from rucio.common import exception
|
|
|
42
42
|
from rucio.common.cache import MemcacheRegion
|
|
43
43
|
from rucio.common.config import config_get, config_get_bool
|
|
44
44
|
from rucio.common.constants import RseAttr, SuspiciousAvailability
|
|
45
|
-
from rucio.common.types import InternalAccount, InternalScope, LFNDict, is_str_list
|
|
45
|
+
from rucio.common.types import InternalAccount, InternalScope, IPDict, LFNDict, is_str_list
|
|
46
46
|
from rucio.common.utils import add_url_query, chunks, clean_pfns, str_to_date
|
|
47
47
|
from rucio.core.credential import get_signed_url
|
|
48
48
|
from rucio.core.message import add_messages
|
|
@@ -402,7 +402,7 @@ def __declare_bad_file_replicas(
|
|
|
402
402
|
if f"{pfn} Unknown replica" not in unknown_replicas:
|
|
403
403
|
unknown_replicas.append('%s %s' % (pfn, 'Unknown replica'))
|
|
404
404
|
elif scope or name:
|
|
405
|
-
unknown_replicas.append(f"{(scope,name)} Unknown replica")
|
|
405
|
+
unknown_replicas.append(f"{(scope, name)} Unknown replica")
|
|
406
406
|
|
|
407
407
|
if status == BadFilesStatus.BAD:
|
|
408
408
|
# For BAD file, we modify the replica state, not for suspicious
|
|
@@ -943,7 +943,7 @@ def _build_list_replicas_pfn(
|
|
|
943
943
|
path: str,
|
|
944
944
|
sign_urls: bool,
|
|
945
945
|
signature_lifetime: Optional[int],
|
|
946
|
-
client_location: Optional[
|
|
946
|
+
client_location: Optional[IPDict],
|
|
947
947
|
logger: "LoggerFunction" = logging.log,
|
|
948
948
|
*,
|
|
949
949
|
session: "Session",
|
|
@@ -1024,7 +1024,7 @@ def _list_replicas(
|
|
|
1024
1024
|
show_pfns: bool,
|
|
1025
1025
|
schemes: Optional[list[str]],
|
|
1026
1026
|
files_wo_replica: "Iterable[dict[str, Any]]",
|
|
1027
|
-
client_location: Optional[
|
|
1027
|
+
client_location: Optional[IPDict],
|
|
1028
1028
|
domain: Optional[str],
|
|
1029
1029
|
sign_urls: bool,
|
|
1030
1030
|
signature_lifetime: Optional[int],
|
|
@@ -1200,7 +1200,7 @@ def list_replicas(
|
|
|
1200
1200
|
all_states: bool = False,
|
|
1201
1201
|
pfns: bool = True,
|
|
1202
1202
|
rse_expression: Optional[str] = None,
|
|
1203
|
-
client_location: Optional[
|
|
1203
|
+
client_location: Optional[IPDict] = None,
|
|
1204
1204
|
domain: Optional[str] = None,
|
|
1205
1205
|
sign_urls: bool = False,
|
|
1206
1206
|
signature_lifetime: "Optional[int]" = None,
|
|
@@ -4176,7 +4176,7 @@ def get_replicas_state(
|
|
|
4176
4176
|
|
|
4177
4177
|
@read_session
|
|
4178
4178
|
def get_suspicious_files(
|
|
4179
|
-
rse_expression: str,
|
|
4179
|
+
rse_expression: Optional[str],
|
|
4180
4180
|
available_elsewhere: int,
|
|
4181
4181
|
filter_: Optional[dict[str, Any]] = None,
|
|
4182
4182
|
logger: "LoggerFunction" = logging.log,
|
rucio/core/request.py
CHANGED
|
@@ -31,7 +31,7 @@ from sqlalchemy.orm import aliased
|
|
|
31
31
|
from sqlalchemy.sql.expression import asc, false, func, null, true
|
|
32
32
|
|
|
33
33
|
from rucio.common.config import config_get_bool, config_get_int
|
|
34
|
-
from rucio.common.constants import RseAttr
|
|
34
|
+
from rucio.common.constants import RseAttr, TransferLimitDirection
|
|
35
35
|
from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RucioException, UnsupportedOperation
|
|
36
36
|
from rucio.common.types import FilterDict, InternalAccount, InternalScope, LoggerFunction, RequestDict
|
|
37
37
|
from rucio.common.utils import chunks, generate_uuid
|
|
@@ -41,7 +41,7 @@ from rucio.core.monitor import MetricManager
|
|
|
41
41
|
from rucio.core.rse import RseCollection, RseData, get_rse_attribute, get_rse_name, get_rse_vo
|
|
42
42
|
from rucio.core.rse_expression_parser import parse_expression
|
|
43
43
|
from rucio.db.sqla import filter_thread_work, models
|
|
44
|
-
from rucio.db.sqla.constants import LockState, ReplicaState, RequestErrMsg, RequestState, RequestType
|
|
44
|
+
from rucio.db.sqla.constants import LockState, ReplicaState, RequestErrMsg, RequestState, RequestType
|
|
45
45
|
from rucio.db.sqla.session import read_session, stream_session, transactional_session
|
|
46
46
|
from rucio.db.sqla.util import temp_table_mngr
|
|
47
47
|
|
rucio/core/rule.py
CHANGED
|
@@ -2056,7 +2056,8 @@ def re_evaluate_did(
|
|
|
2056
2056
|
name: str,
|
|
2057
2057
|
rule_evaluation_action: DIDReEvaluation,
|
|
2058
2058
|
*,
|
|
2059
|
-
session: "Session"
|
|
2059
|
+
session: "Session",
|
|
2060
|
+
logger: LoggerFunction = logging.log
|
|
2060
2061
|
) -> None:
|
|
2061
2062
|
"""
|
|
2062
2063
|
Re-Evaluates a did.
|
|
@@ -2065,6 +2066,7 @@ def re_evaluate_did(
|
|
|
2065
2066
|
:param name: The name of the did to be re-evaluated.
|
|
2066
2067
|
:param rule_evaluation_action: The Rule evaluation action.
|
|
2067
2068
|
:param session: The database session in use.
|
|
2069
|
+
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
2068
2070
|
:raises: DataIdentifierNotFound
|
|
2069
2071
|
"""
|
|
2070
2072
|
|
|
@@ -2080,9 +2082,9 @@ def re_evaluate_did(
|
|
|
2080
2082
|
raise DataIdentifierNotFound() from exc
|
|
2081
2083
|
|
|
2082
2084
|
if rule_evaluation_action == DIDReEvaluation.ATTACH:
|
|
2083
|
-
__evaluate_did_attach(did, session=session)
|
|
2085
|
+
__evaluate_did_attach(did, session=session, logger=logger)
|
|
2084
2086
|
else:
|
|
2085
|
-
__evaluate_did_detach(did, session=session)
|
|
2087
|
+
__evaluate_did_detach(did, session=session, logger=logger)
|
|
2086
2088
|
|
|
2087
2089
|
# Update size and length of did
|
|
2088
2090
|
if session.bind.dialect.name == 'oracle':
|
rucio/core/transfer.py
CHANGED
|
@@ -27,11 +27,10 @@ from dogpile.cache.api import NoValue
|
|
|
27
27
|
from sqlalchemy import select, update
|
|
28
28
|
from sqlalchemy.exc import IntegrityError
|
|
29
29
|
|
|
30
|
-
from rucio.common import constants
|
|
31
30
|
from rucio.common.config import config_get, config_get_list
|
|
32
|
-
from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr
|
|
31
|
+
from rucio.common.constants import SUPPORTED_PROTOCOLS, RseAttr, TransferLimitDirection
|
|
33
32
|
from rucio.common.exception import InvalidRSEExpression, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation
|
|
34
|
-
from rucio.common.utils import construct_non_deterministic_pfn
|
|
33
|
+
from rucio.common.utils import construct_non_deterministic_pfn, get_transfer_schemas
|
|
35
34
|
from rucio.core import did
|
|
36
35
|
from rucio.core import message as message_core
|
|
37
36
|
from rucio.core import request as request_core
|
|
@@ -40,7 +39,7 @@ from rucio.core.monitor import MetricManager
|
|
|
40
39
|
from rucio.core.request import DirectTransfer, RequestSource, RequestWithSources, TransferDestination, transition_request_state
|
|
41
40
|
from rucio.core.rse_expression_parser import parse_expression
|
|
42
41
|
from rucio.db.sqla import models
|
|
43
|
-
from rucio.db.sqla.constants import DIDType, RequestState, RequestType
|
|
42
|
+
from rucio.db.sqla.constants import DIDType, RequestState, RequestType
|
|
44
43
|
from rucio.db.sqla.session import read_session, stream_session, transactional_session
|
|
45
44
|
from rucio.rse import rsemanager as rsemgr
|
|
46
45
|
from rucio.transfertool.bittorrent import BittorrentTransfertool
|
|
@@ -1331,7 +1330,7 @@ def __add_compatible_schemes(schemes, allowed_schemes):
|
|
|
1331
1330
|
for scheme in schemes:
|
|
1332
1331
|
if scheme in allowed_schemes:
|
|
1333
1332
|
return_schemes.append(scheme)
|
|
1334
|
-
for scheme_map_scheme in
|
|
1333
|
+
for scheme_map_scheme in get_transfer_schemas().get(scheme, []):
|
|
1335
1334
|
if scheme_map_scheme not in allowed_schemes:
|
|
1336
1335
|
continue
|
|
1337
1336
|
else:
|
|
@@ -26,13 +26,14 @@ from sqlalchemy import null
|
|
|
26
26
|
|
|
27
27
|
import rucio.db.sqla.util
|
|
28
28
|
from rucio.common import exception
|
|
29
|
+
from rucio.common.constants import TransferLimitDirection
|
|
29
30
|
from rucio.common.logging import setup_logging
|
|
30
31
|
from rucio.core.monitor import MetricManager
|
|
31
32
|
from rucio.core.request import get_request_stats, re_sync_all_transfer_limits, release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, reset_stale_waiting_requests, set_transfer_limit_stats
|
|
32
33
|
from rucio.core.rse import RseCollection, RseData
|
|
33
34
|
from rucio.core.transfer import applicable_rse_transfer_limits
|
|
34
35
|
from rucio.daemons.common import ProducerConsumerDaemon, db_workqueue
|
|
35
|
-
from rucio.db.sqla.constants import RequestState
|
|
36
|
+
from rucio.db.sqla.constants import RequestState
|
|
36
37
|
|
|
37
38
|
if TYPE_CHECKING:
|
|
38
39
|
from collections.abc import Iterator
|
rucio/daemons/judge/evaluator.py
CHANGED
|
@@ -122,7 +122,7 @@ def run_once(
|
|
|
122
122
|
|
|
123
123
|
try:
|
|
124
124
|
start_time = time.time()
|
|
125
|
-
re_evaluate_did(scope=did.scope, name=did.name, rule_evaluation_action=did.rule_evaluation_action)
|
|
125
|
+
re_evaluate_did(scope=did.scope, name=did.name, rule_evaluation_action=did.rule_evaluation_action, logger=logger)
|
|
126
126
|
logger(logging.DEBUG, 'evaluation of %s:%s took %f', did.scope, did.name, time.time() - start_time)
|
|
127
127
|
delete_updated_did(id_=did.id)
|
|
128
128
|
done_dids[did_tag].append(did.rule_evaluation_action)
|
rucio/db/sqla/constants.py
CHANGED
|
@@ -196,9 +196,9 @@ class SubscriptionState(Enum):
|
|
|
196
196
|
BROKEN = 'B'
|
|
197
197
|
|
|
198
198
|
|
|
199
|
-
class TransferLimitDirection(Enum):
|
|
200
|
-
SOURCE = 'S'
|
|
201
|
-
DESTINATION = 'D'
|
|
199
|
+
#class TransferLimitDirection(Enum):
|
|
200
|
+
# SOURCE = 'S'
|
|
201
|
+
# DESTINATION = 'D'
|
|
202
202
|
|
|
203
203
|
|
|
204
204
|
class DatabaseOperationType(Enum):
|
|
@@ -19,7 +19,7 @@ import sqlalchemy as sa
|
|
|
19
19
|
from alembic import context
|
|
20
20
|
from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
|
|
21
21
|
|
|
22
|
-
from rucio.
|
|
22
|
+
from rucio.common.constants import TransferLimitDirection
|
|
23
23
|
from rucio.db.sqla.types import GUID
|
|
24
24
|
|
|
25
25
|
# Alembic revision identifiers
|
rucio/db/sqla/models.py
CHANGED
|
@@ -29,6 +29,7 @@ from sqlalchemy.types import LargeBinary
|
|
|
29
29
|
# and it must be renamed to avoid conflicts with the policy package schema modules
|
|
30
30
|
from rucio.common import schema as common_schema
|
|
31
31
|
from rucio.common import utils
|
|
32
|
+
from rucio.common.constants import TransferLimitDirection
|
|
32
33
|
from rucio.common.types import InternalAccount, InternalScope # noqa: TCH001 (types are needed by SQLAlchemy)
|
|
33
34
|
from rucio.db.sqla.constants import (
|
|
34
35
|
AccountStatus,
|
|
@@ -51,7 +52,6 @@ from rucio.db.sqla.constants import (
|
|
|
51
52
|
RuleState,
|
|
52
53
|
ScopeStatus,
|
|
53
54
|
SubscriptionState,
|
|
54
|
-
TransferLimitDirection,
|
|
55
55
|
)
|
|
56
56
|
from rucio.db.sqla.session import BASE
|
|
57
57
|
from rucio.db.sqla.types import GUID, JSON, BooleanString, InternalAccountString, InternalScopeString
|
rucio/gateway/replica.py
CHANGED
|
@@ -13,12 +13,12 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import datetime
|
|
16
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
16
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
|
|
17
17
|
|
|
18
18
|
from rucio.common import exception
|
|
19
19
|
from rucio.common.constants import SuspiciousAvailability
|
|
20
20
|
from rucio.common.schema import validate_schema
|
|
21
|
-
from rucio.common.types import InternalAccount, InternalScope
|
|
21
|
+
from rucio.common.types import InternalAccount, InternalScope, IPDict
|
|
22
22
|
from rucio.common.utils import gateway_update_return_dict, invert_dict
|
|
23
23
|
from rucio.core import replica
|
|
24
24
|
from rucio.core.rse import get_rse_id, get_rse_name
|
|
@@ -27,10 +27,15 @@ from rucio.db.sqla.session import db_session
|
|
|
27
27
|
from rucio.gateway import permission
|
|
28
28
|
|
|
29
29
|
if TYPE_CHECKING:
|
|
30
|
-
from collections.abc import Iterator
|
|
30
|
+
from collections.abc import Iterable, Iterator
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def get_bad_replicas_summary(
|
|
33
|
+
def get_bad_replicas_summary(
|
|
34
|
+
rse_expression: Optional[str] = None,
|
|
35
|
+
from_date: Optional[datetime.datetime] = None,
|
|
36
|
+
to_date: Optional[datetime.date] = None,
|
|
37
|
+
vo: str = 'def'
|
|
38
|
+
) -> list[dict[str, Any]]:
|
|
34
39
|
"""
|
|
35
40
|
List the bad file replicas summary. Method used by the rucio-ui.
|
|
36
41
|
:param rse_expression: The RSE expression.
|
|
@@ -71,7 +76,13 @@ def list_bad_replicas_status(
|
|
|
71
76
|
return [gateway_update_return_dict(r, session=session) for r in replicas]
|
|
72
77
|
|
|
73
78
|
|
|
74
|
-
def declare_bad_file_replicas(
|
|
79
|
+
def declare_bad_file_replicas(
|
|
80
|
+
replicas: Union[list[str], list[dict[str, Any]]],
|
|
81
|
+
reason: str,
|
|
82
|
+
issuer: str,
|
|
83
|
+
vo: str = 'def',
|
|
84
|
+
force: bool = False
|
|
85
|
+
) -> dict[str, Any]:
|
|
75
86
|
"""
|
|
76
87
|
Declare a list of bad replicas.
|
|
77
88
|
|
|
@@ -105,6 +116,10 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False):
|
|
|
105
116
|
raise exception.ReplicaNotFound("Not all replicas found")
|
|
106
117
|
rse_ids_to_check = set(rses_for_replicas.keys())
|
|
107
118
|
else:
|
|
119
|
+
# replicas is a list[dict] in this path,
|
|
120
|
+
# but the static code analyzer does not see it due to as_pfns logic above,
|
|
121
|
+
# so cast is used instead
|
|
122
|
+
replicas = cast("list[dict[str, Any]]", replicas)
|
|
108
123
|
replicas_lst = []
|
|
109
124
|
for r in replicas:
|
|
110
125
|
if "name" not in r or "scope" not in r or ("rse" not in r and "rse_id" not in r):
|
|
@@ -153,7 +168,12 @@ def declare_bad_file_replicas(replicas, reason, issuer, vo='def', force=False):
|
|
|
153
168
|
return out
|
|
154
169
|
|
|
155
170
|
|
|
156
|
-
def declare_suspicious_file_replicas(
|
|
171
|
+
def declare_suspicious_file_replicas(
|
|
172
|
+
pfns: list[Union[str, dict[str, Any]]],
|
|
173
|
+
reason: str,
|
|
174
|
+
issuer: str,
|
|
175
|
+
vo: str = 'def'
|
|
176
|
+
) -> dict[str, list[str]]:
|
|
157
177
|
"""
|
|
158
178
|
Declare a list of bad replicas.
|
|
159
179
|
|
|
@@ -169,9 +189,9 @@ def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def'):
|
|
|
169
189
|
if not auth_result.allowed:
|
|
170
190
|
raise exception.AccessDenied('Account %s can not declare suspicious replicas. %s' % (issuer, auth_result.message))
|
|
171
191
|
|
|
172
|
-
|
|
192
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
173
193
|
|
|
174
|
-
replicas = replica.declare_bad_file_replicas(pfns, reason=reason, issuer=
|
|
194
|
+
replicas = replica.declare_bad_file_replicas(pfns, reason=reason, issuer=issuer_account, status=BadFilesStatus.SUSPICIOUS, session=session)
|
|
175
195
|
|
|
176
196
|
for k in list(replicas):
|
|
177
197
|
try:
|
|
@@ -183,7 +203,11 @@ def declare_suspicious_file_replicas(pfns, reason, issuer, vo='def'):
|
|
|
183
203
|
return replicas
|
|
184
204
|
|
|
185
205
|
|
|
186
|
-
def get_did_from_pfns(
|
|
206
|
+
def get_did_from_pfns(
|
|
207
|
+
pfns: "Iterable[str]",
|
|
208
|
+
rse: str,
|
|
209
|
+
vo: str = 'def'
|
|
210
|
+
) -> 'Iterator[dict[str, dict[str, Any]]]':
|
|
187
211
|
"""
|
|
188
212
|
Get the DIDs associated to a PFN on one given RSE
|
|
189
213
|
|
|
@@ -202,12 +226,24 @@ def get_did_from_pfns(pfns, rse, vo='def'):
|
|
|
202
226
|
yield r
|
|
203
227
|
|
|
204
228
|
|
|
205
|
-
def list_replicas(
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
229
|
+
def list_replicas(
|
|
230
|
+
dids: "Iterable[dict[str, Any]]",
|
|
231
|
+
schemes: Optional[list[str]] = None,
|
|
232
|
+
unavailable: bool = False,
|
|
233
|
+
request_id: Optional[str] = None,
|
|
234
|
+
ignore_availability: bool = True,
|
|
235
|
+
all_states: bool = False,
|
|
236
|
+
rse_expression: Optional[str] = None,
|
|
237
|
+
client_location: Optional[IPDict] = None,
|
|
238
|
+
domain: Optional[str] = None,
|
|
239
|
+
signature_lifetime: Optional[int] = None,
|
|
240
|
+
resolve_archives: bool = True,
|
|
241
|
+
resolve_parents: bool = False,
|
|
242
|
+
nrandom: Optional[int] = None,
|
|
243
|
+
updated_after: Optional[datetime.datetime] = None,
|
|
244
|
+
issuer: Optional[str] = None,
|
|
245
|
+
vo: str = 'def'
|
|
246
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
211
247
|
"""
|
|
212
248
|
List file replicas for a list of data identifiers.
|
|
213
249
|
|
|
@@ -261,7 +297,13 @@ def list_replicas(dids, schemes=None, unavailable=False, request_id=None,
|
|
|
261
297
|
yield rep
|
|
262
298
|
|
|
263
299
|
|
|
264
|
-
def add_replicas(
|
|
300
|
+
def add_replicas(
|
|
301
|
+
rse: str,
|
|
302
|
+
files: "Iterable[dict[str, Any]]",
|
|
303
|
+
issuer: str,
|
|
304
|
+
ignore_availability: bool = False,
|
|
305
|
+
vo: str = 'def'
|
|
306
|
+
) -> None:
|
|
265
307
|
"""
|
|
266
308
|
Bulk add file replicas.
|
|
267
309
|
|
|
@@ -270,8 +312,6 @@ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
|
|
|
270
312
|
:param issuer: The issuer account.
|
|
271
313
|
:param ignore_availability: Ignore blocked RSEs.
|
|
272
314
|
:param vo: The VO to act on.
|
|
273
|
-
|
|
274
|
-
:returns: True is successful, False otherwise
|
|
275
315
|
"""
|
|
276
316
|
for v_file in files:
|
|
277
317
|
v_file.update({"type": "FILE"}) # Make sure DIDs are identified as files for checking
|
|
@@ -287,16 +327,22 @@ def add_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
|
|
|
287
327
|
if not permission.has_permission(issuer=issuer, vo=vo, action='skip_availability_check', kwargs=kwargs, session=session):
|
|
288
328
|
ignore_availability = False
|
|
289
329
|
|
|
290
|
-
|
|
330
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
291
331
|
for f in files:
|
|
292
332
|
f['scope'] = InternalScope(f['scope'], vo=vo)
|
|
293
333
|
if 'account' in f:
|
|
294
334
|
f['account'] = InternalAccount(f['account'], vo=vo)
|
|
295
335
|
|
|
296
|
-
replica.add_replicas(rse_id=rse_id, files=files, account=
|
|
336
|
+
replica.add_replicas(rse_id=rse_id, files=files, account=issuer_account, ignore_availability=ignore_availability, session=session)
|
|
297
337
|
|
|
298
338
|
|
|
299
|
-
def delete_replicas(
|
|
339
|
+
def delete_replicas(
|
|
340
|
+
rse: str,
|
|
341
|
+
files: "Iterable[dict[str, Any]]",
|
|
342
|
+
issuer: str,
|
|
343
|
+
ignore_availability: bool = False,
|
|
344
|
+
vo: str = 'def'
|
|
345
|
+
) -> None:
|
|
300
346
|
"""
|
|
301
347
|
Bulk delete file replicas.
|
|
302
348
|
|
|
@@ -305,8 +351,6 @@ def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
|
|
|
305
351
|
:param issuer: The issuer account.
|
|
306
352
|
:param ignore_availability: Ignore blocked RSEs.
|
|
307
353
|
:param vo: The VO to act on.
|
|
308
|
-
|
|
309
|
-
:returns: True is successful, False otherwise
|
|
310
354
|
"""
|
|
311
355
|
validate_schema(name='r_dids', obj=files, vo=vo)
|
|
312
356
|
|
|
@@ -326,7 +370,12 @@ def delete_replicas(rse, files, issuer, ignore_availability=False, vo='def'):
|
|
|
326
370
|
replica.delete_replicas(rse_id=rse_id, files=files, ignore_availability=ignore_availability, session=session)
|
|
327
371
|
|
|
328
372
|
|
|
329
|
-
def update_replicas_states(
|
|
373
|
+
def update_replicas_states(
|
|
374
|
+
rse: str,
|
|
375
|
+
files: "Iterable[dict[str, Any]]",
|
|
376
|
+
issuer: str,
|
|
377
|
+
vo: str = 'def'
|
|
378
|
+
) -> None:
|
|
330
379
|
"""
|
|
331
380
|
Update File replica information and state.
|
|
332
381
|
|
|
@@ -355,7 +404,12 @@ def update_replicas_states(rse, files, issuer, vo='def'):
|
|
|
355
404
|
replica.update_replicas_states(replicas=replicas, session=session)
|
|
356
405
|
|
|
357
406
|
|
|
358
|
-
def list_dataset_replicas(
|
|
407
|
+
def list_dataset_replicas(
|
|
408
|
+
scope: str,
|
|
409
|
+
name: str,
|
|
410
|
+
deep: bool = False,
|
|
411
|
+
vo: str = 'def'
|
|
412
|
+
) -> "Iterator[dict[str, Any]]":
|
|
359
413
|
"""
|
|
360
414
|
:param scope: The scope of the dataset.
|
|
361
415
|
:param name: The name of the dataset.
|
|
@@ -365,17 +419,20 @@ def list_dataset_replicas(scope, name, deep=False, vo='def'):
|
|
|
365
419
|
:returns: A list of dict dataset replicas
|
|
366
420
|
"""
|
|
367
421
|
|
|
368
|
-
|
|
422
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
369
423
|
|
|
370
424
|
with db_session(DatabaseOperationType.READ) as session:
|
|
371
|
-
replicas = replica.list_dataset_replicas(scope=
|
|
425
|
+
replicas = replica.list_dataset_replicas(scope=internal_scope, name=name, deep=deep, session=session)
|
|
372
426
|
|
|
373
427
|
for r in replicas:
|
|
374
428
|
r['scope'] = r['scope'].external
|
|
375
429
|
yield r
|
|
376
430
|
|
|
377
431
|
|
|
378
|
-
def list_dataset_replicas_bulk(
|
|
432
|
+
def list_dataset_replicas_bulk(
|
|
433
|
+
dids: 'Iterable[dict[str, Any]]',
|
|
434
|
+
vo: str = 'def'
|
|
435
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
379
436
|
"""
|
|
380
437
|
:param dids: The list of did dictionaries with scope and name.
|
|
381
438
|
:param vo: The VO to act on.
|
|
@@ -403,7 +460,12 @@ def list_dataset_replicas_bulk(dids, vo='def'):
|
|
|
403
460
|
yield gateway_update_return_dict(r, session=session)
|
|
404
461
|
|
|
405
462
|
|
|
406
|
-
def list_dataset_replicas_vp(
|
|
463
|
+
def list_dataset_replicas_vp(
|
|
464
|
+
scope: str,
|
|
465
|
+
name: str,
|
|
466
|
+
deep: bool = False,
|
|
467
|
+
vo: str = 'def'
|
|
468
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
407
469
|
"""
|
|
408
470
|
:param scope: The scope of the dataset.
|
|
409
471
|
:param name: The name of the dataset.
|
|
@@ -415,10 +477,10 @@ def list_dataset_replicas_vp(scope, name, deep=False, vo='def'):
|
|
|
415
477
|
NOTICE: This is an RnD function and might change or go away at any time.
|
|
416
478
|
"""
|
|
417
479
|
|
|
418
|
-
|
|
480
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
419
481
|
|
|
420
482
|
with db_session(DatabaseOperationType.READ) as session:
|
|
421
|
-
for r in replica.list_dataset_replicas_vp(scope=
|
|
483
|
+
for r in replica.list_dataset_replicas_vp(scope=internal_scope, name=name, deep=deep, session=session):
|
|
422
484
|
yield gateway_update_return_dict(r, session=session)
|
|
423
485
|
|
|
424
486
|
|
|
@@ -443,7 +505,14 @@ def list_datasets_per_rse(rse: str, filters: Optional[dict[str, Any]] = None, li
|
|
|
443
505
|
yield gateway_update_return_dict(r, session=session)
|
|
444
506
|
|
|
445
507
|
|
|
446
|
-
def add_bad_pfns(
|
|
508
|
+
def add_bad_pfns(
|
|
509
|
+
pfns: "Iterable[str]",
|
|
510
|
+
issuer: str,
|
|
511
|
+
state: BadFilesStatus,
|
|
512
|
+
reason: Optional[str] = None,
|
|
513
|
+
expires_at: Optional[datetime.datetime] = None,
|
|
514
|
+
vo: str = 'def'
|
|
515
|
+
) -> Literal[True]:
|
|
447
516
|
"""
|
|
448
517
|
Add bad PFNs.
|
|
449
518
|
|
|
@@ -466,12 +535,20 @@ def add_bad_pfns(pfns, issuer, state, reason=None, expires_at=None, vo='def'):
|
|
|
466
535
|
if expires_at and datetime.datetime.utcnow() <= expires_at and expires_at > datetime.datetime.utcnow() + datetime.timedelta(days=30):
|
|
467
536
|
raise exception.InputValidationError('The given duration of %s days exceeds the maximum duration of 30 days.' % (expires_at - datetime.datetime.utcnow()).days)
|
|
468
537
|
|
|
469
|
-
|
|
538
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
470
539
|
|
|
471
|
-
return replica.add_bad_pfns(pfns=pfns, account=
|
|
540
|
+
return replica.add_bad_pfns(pfns=pfns, account=issuer_account, state=state, reason=reason, expires_at=expires_at, session=session)
|
|
472
541
|
|
|
473
542
|
|
|
474
|
-
def add_bad_dids(
|
|
543
|
+
def add_bad_dids(
|
|
544
|
+
dids: "Iterable[dict[str, Any]]",
|
|
545
|
+
rse: str,
|
|
546
|
+
issuer: str,
|
|
547
|
+
state: BadFilesStatus,
|
|
548
|
+
reason: Optional[str] = None,
|
|
549
|
+
expires_at: Optional[datetime.datetime] = None,
|
|
550
|
+
vo: str = 'def'
|
|
551
|
+
) -> list[str]:
|
|
475
552
|
"""
|
|
476
553
|
Add bad replica entries for DIDs.
|
|
477
554
|
|
|
@@ -492,13 +569,18 @@ def add_bad_dids(dids, rse, issuer, state, reason=None, expires_at=None, vo='def
|
|
|
492
569
|
if not auth_result.allowed:
|
|
493
570
|
raise exception.AccessDenied('Account %s can not declare bad PFN or DIDs. %s' % (issuer, auth_result.message))
|
|
494
571
|
|
|
495
|
-
|
|
572
|
+
issuer_account = InternalAccount(issuer, vo=vo)
|
|
496
573
|
rse_id = get_rse_id(rse=rse, session=session)
|
|
497
574
|
|
|
498
|
-
return replica.add_bad_dids(dids=dids, rse_id=rse_id, reason=reason, issuer=
|
|
575
|
+
return replica.add_bad_dids(dids=dids, rse_id=rse_id, reason=reason, issuer=issuer_account, state=state, session=session)
|
|
499
576
|
|
|
500
577
|
|
|
501
|
-
def get_suspicious_files(
|
|
578
|
+
def get_suspicious_files(
|
|
579
|
+
rse_expression: Optional[str],
|
|
580
|
+
younger_than: Optional[datetime.datetime] = None,
|
|
581
|
+
nattempts: Optional[int] = None,
|
|
582
|
+
vo: str = 'def'
|
|
583
|
+
) -> list[dict[str, Any]]:
|
|
502
584
|
"""
|
|
503
585
|
List the list of suspicious files on a list of RSEs
|
|
504
586
|
:param rse_expression: The RSE expression where the suspicious files are located
|
|
@@ -513,7 +595,13 @@ def get_suspicious_files(rse_expression, younger_than=None, nattempts=None, vo='
|
|
|
513
595
|
return [gateway_update_return_dict(r, session=session) for r in replicas]
|
|
514
596
|
|
|
515
597
|
|
|
516
|
-
def set_tombstone(
|
|
598
|
+
def set_tombstone(
|
|
599
|
+
rse: str,
|
|
600
|
+
scope: str,
|
|
601
|
+
name: str,
|
|
602
|
+
issuer: str,
|
|
603
|
+
vo: str = 'def'
|
|
604
|
+
) -> None:
|
|
517
605
|
"""
|
|
518
606
|
Sets a tombstone on one replica.
|
|
519
607
|
|
|
@@ -531,5 +619,5 @@ def set_tombstone(rse, scope, name, issuer, vo='def'):
|
|
|
531
619
|
if not auth_result.allowed:
|
|
532
620
|
raise exception.AccessDenied('Account %s can not set tombstones. %s' % (issuer, auth_result.message))
|
|
533
621
|
|
|
534
|
-
|
|
535
|
-
replica.set_tombstone(rse_id,
|
|
622
|
+
internal_scope = InternalScope(scope, vo=vo)
|
|
623
|
+
replica.set_tombstone(rse_id, internal_scope, name, session=session)
|