rucio 35.7.0__py3-none-any.whl → 37.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/{daemons/c3po/collectors → cli}/__init__.py +1 -0
- rucio/cli/account.py +216 -0
- rucio-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/dumper/__init__.py +45 -38
- rucio/common/dumper/consistency.py +75 -30
- rucio/common/dumper/data_models.py +63 -19
- rucio/common/dumper/path_parsing.py +19 -8
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/core/account_limit.py +14 -12
- rucio/core/authentication.py +2 -2
- rucio/core/config.py +23 -42
- rucio/core/credential.py +14 -15
- rucio/core/did.py +5 -1
- rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
- rucio/core/did_meta_plugins/filter_engine.py +62 -3
- rucio/core/did_meta_plugins/json_meta.py +2 -2
- rucio/core/did_meta_plugins/mongo_meta.py +43 -30
- rucio/core/did_meta_plugins/postgres_meta.py +75 -39
- rucio/core/identity.py +6 -5
- rucio/core/importer.py +4 -3
- rucio/core/lifetime_exception.py +2 -2
- rucio/core/lock.py +8 -7
- rucio/core/message.py +6 -0
- rucio/core/monitor.py +30 -29
- rucio/core/naming_convention.py +2 -2
- rucio/core/nongrid_trace.py +2 -2
- rucio/core/oidc.py +11 -9
- rucio/core/permission/__init__.py +79 -37
- rucio/core/permission/generic.py +1 -7
- rucio/core/permission/generic_multi_vo.py +1 -7
- rucio/core/quarantined_replica.py +4 -3
- rucio/core/replica.py +464 -139
- rucio/core/replica_sorter.py +55 -59
- rucio/core/request.py +34 -32
- rucio/core/rse.py +301 -97
- rucio/core/rse_counter.py +1 -2
- rucio/core/rse_expression_parser.py +7 -7
- rucio/core/rse_selector.py +9 -7
- rucio/core/rule.py +41 -40
- rucio/core/rule_grouping.py +42 -40
- rucio/core/scope.py +5 -4
- rucio/core/subscription.py +26 -28
- rucio/core/topology.py +11 -11
- rucio/core/trace.py +2 -2
- rucio/core/transfer.py +29 -15
- rucio/core/volatile_replica.py +4 -3
- rucio/daemons/atropos/atropos.py +1 -1
- rucio/daemons/auditor/__init__.py +2 -2
- rucio/daemons/auditor/srmdumps.py +6 -6
- rucio/daemons/automatix/automatix.py +32 -21
- rucio/daemons/badreplicas/necromancer.py +2 -2
- rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
- rucio/daemons/bb8/t2_background_rebalance.py +1 -1
- rucio/daemons/common.py +15 -25
- rucio/daemons/conveyor/finisher.py +2 -2
- rucio/daemons/conveyor/poller.py +18 -28
- rucio/daemons/conveyor/receiver.py +2 -2
- rucio/daemons/conveyor/stager.py +1 -0
- rucio/daemons/conveyor/submitter.py +3 -3
- rucio/daemons/hermes/hermes.py +91 -30
- rucio/daemons/judge/evaluator.py +2 -2
- rucio/daemons/oauthmanager/oauthmanager.py +3 -3
- rucio/daemons/reaper/dark_reaper.py +7 -3
- rucio/daemons/reaper/reaper.py +12 -16
- rucio/daemons/rsedecommissioner/config.py +1 -1
- rucio/daemons/rsedecommissioner/profiles/generic.py +5 -4
- rucio/daemons/rsedecommissioner/profiles/types.py +7 -6
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +1 -1
- rucio/daemons/storage/consistency/actions.py +8 -6
- rucio/daemons/tracer/kronos.py +4 -4
- rucio/db/sqla/constants.py +5 -0
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +4 -4
- rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
- rucio/db/sqla/models.py +157 -154
- rucio/db/sqla/session.py +58 -27
- rucio/db/sqla/types.py +2 -2
- rucio/db/sqla/util.py +2 -2
- rucio/gateway/account.py +18 -12
- rucio/gateway/account_limit.py +137 -60
- rucio/gateway/authentication.py +18 -12
- rucio/gateway/config.py +30 -20
- rucio/gateway/credential.py +9 -10
- rucio/gateway/did.py +70 -53
- rucio/gateway/dirac.py +6 -4
- rucio/gateway/exporter.py +3 -2
- rucio/gateway/heartbeat.py +6 -4
- rucio/gateway/identity.py +36 -51
- rucio/gateway/importer.py +3 -2
- rucio/gateway/lifetime_exception.py +3 -2
- rucio/gateway/meta_conventions.py +17 -6
- rucio/gateway/permission.py +4 -1
- rucio/gateway/quarantined_replica.py +3 -2
- rucio/gateway/replica.py +31 -22
- rucio/gateway/request.py +27 -18
- rucio/gateway/rse.py +69 -37
- rucio/gateway/rule.py +46 -26
- rucio/gateway/scope.py +3 -2
- rucio/gateway/subscription.py +14 -11
- rucio/gateway/vo.py +12 -8
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/tests/common.py +23 -13
- rucio/tests/common_server.py +26 -9
- rucio/transfertool/bittorrent.py +15 -14
- rucio/transfertool/bittorrent_driver.py +5 -7
- rucio/transfertool/bittorrent_driver_qbittorrent.py +9 -8
- rucio/transfertool/fts3.py +20 -16
- rucio/transfertool/mock.py +2 -3
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- rucio/web/rest/flaskapi/v1/accounts.py +17 -3
- rucio/web/rest/flaskapi/v1/auth.py +5 -5
- rucio/web/rest/flaskapi/v1/credentials.py +3 -2
- rucio/web/rest/flaskapi/v1/dids.py +21 -15
- rucio/web/rest/flaskapi/v1/identities.py +33 -9
- rucio/web/rest/flaskapi/v1/redirect.py +5 -4
- rucio/web/rest/flaskapi/v1/replicas.py +12 -8
- rucio/web/rest/flaskapi/v1/rses.py +15 -4
- rucio/web/rest/flaskapi/v1/traces.py +56 -19
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/alembic.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio.cfg.template +3 -19
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +1 -18
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/requirements.server.txt +97 -68
- rucio-37.0.0.data/scripts/rucio +133 -0
- rucio-37.0.0.data/scripts/rucio-admin +97 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-atropos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-auditor +2 -1
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-automatix +2 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-cache-client +17 -10
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-receiver +1 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-kronos +1 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-minos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-minos-temporary-expiration +2 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-necromancer +2 -2
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-reaper +6 -6
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-transmogrifier +2 -2
- rucio-37.0.0.dist-info/METADATA +92 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/RECORD +237 -243
- {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/core/permission/atlas.py +0 -1348
- rucio/core/permission/belleii.py +0 -1077
- rucio/core/permission/escape.py +0 -1078
- rucio/daemons/c3po/algorithms/__init__.py +0 -13
- rucio/daemons/c3po/algorithms/simple.py +0 -134
- rucio/daemons/c3po/algorithms/t2_free_space.py +0 -128
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +0 -130
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +0 -294
- rucio/daemons/c3po/c3po.py +0 -371
- rucio/daemons/c3po/collectors/agis.py +0 -108
- rucio/daemons/c3po/collectors/free_space.py +0 -81
- rucio/daemons/c3po/collectors/jedi_did.py +0 -57
- rucio/daemons/c3po/collectors/mock_did.py +0 -51
- rucio/daemons/c3po/collectors/network_metrics.py +0 -71
- rucio/daemons/c3po/collectors/workload.py +0 -112
- rucio/daemons/c3po/utils/__init__.py +0 -13
- rucio/daemons/c3po/utils/dataset_cache.py +0 -50
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +0 -56
- rucio/daemons/c3po/utils/expiring_list.py +0 -62
- rucio/daemons/c3po/utils/popularity.py +0 -85
- rucio/daemons/c3po/utils/timeseries.py +0 -89
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio-35.7.0.data/scripts/rucio-c3po +0 -85
- rucio-35.7.0.dist-info/METADATA +0 -72
- /rucio/{daemons/c3po → cli/bin_legacy}/__init__.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-account +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-bb8 +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-dumper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-follower +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-hermes +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-injector +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-35.7.0.data → rucio-37.0.0.data}/scripts/rucio-undertaker +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/WHEEL +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0.dist-info}/top_level.txt +0 -0
rucio/core/replica.py
CHANGED
|
@@ -26,7 +26,7 @@ from json import dumps
|
|
|
26
26
|
from re import match
|
|
27
27
|
from struct import unpack
|
|
28
28
|
from traceback import format_exc
|
|
29
|
-
from typing import TYPE_CHECKING
|
|
29
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, Union
|
|
30
30
|
|
|
31
31
|
import requests
|
|
32
32
|
from dogpile.cache.api import NO_VALUE
|
|
@@ -34,15 +34,15 @@ from sqlalchemy import and_, delete, exists, func, insert, not_, or_, union, upd
|
|
|
34
34
|
from sqlalchemy.exc import DatabaseError, IntegrityError
|
|
35
35
|
from sqlalchemy.orm import aliased
|
|
36
36
|
from sqlalchemy.orm.exc import FlushError, NoResultFound
|
|
37
|
-
from sqlalchemy.sql.expression import case, false, literal, literal_column, null, select, text, true
|
|
37
|
+
from sqlalchemy.sql.expression import ColumnElement, case, false, literal, literal_column, null, select, text, true
|
|
38
38
|
|
|
39
39
|
import rucio.core.did
|
|
40
40
|
import rucio.core.lock
|
|
41
41
|
from rucio.common import exception
|
|
42
|
-
from rucio.common.cache import
|
|
42
|
+
from rucio.common.cache import MemcacheRegion
|
|
43
43
|
from rucio.common.config import config_get, config_get_bool
|
|
44
44
|
from rucio.common.constants import RseAttr, SuspiciousAvailability
|
|
45
|
-
from rucio.common.types import InternalScope
|
|
45
|
+
from rucio.common.types import InternalAccount, InternalScope, LFNDict, is_str_list
|
|
46
46
|
from rucio.common.utils import add_url_query, chunks, clean_pfns, str_to_date
|
|
47
47
|
from rucio.core.credential import get_signed_url
|
|
48
48
|
from rucio.core.message import add_messages
|
|
@@ -57,15 +57,16 @@ from rucio.db.sqla.util import temp_table_mngr
|
|
|
57
57
|
from rucio.rse import rsemanager as rsemgr
|
|
58
58
|
|
|
59
59
|
if TYPE_CHECKING:
|
|
60
|
-
from collections.abc import Iterable, Iterator, Sequence
|
|
61
|
-
from typing import Any, Optional
|
|
60
|
+
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
62
61
|
|
|
62
|
+
from sqlalchemy.engine import Row
|
|
63
63
|
from sqlalchemy.orm import Session
|
|
64
|
+
from sqlalchemy.sql.selectable import Select, Subquery
|
|
64
65
|
|
|
65
66
|
from rucio.common.types import LoggerFunction
|
|
66
67
|
from rucio.rse.protocols.protocol import RSEProtocol
|
|
67
68
|
|
|
68
|
-
REGION =
|
|
69
|
+
REGION = MemcacheRegion(expiration_time=60)
|
|
69
70
|
METRICS = MetricManager(module=__name__)
|
|
70
71
|
|
|
71
72
|
|
|
@@ -74,7 +75,14 @@ Association = namedtuple('Association', ['scope', 'name', 'child_scope', 'child_
|
|
|
74
75
|
|
|
75
76
|
|
|
76
77
|
@read_session
|
|
77
|
-
def get_bad_replicas_summary(
|
|
78
|
+
def get_bad_replicas_summary(
|
|
79
|
+
rse_expression: Optional[str] = None,
|
|
80
|
+
from_date: Optional[datetime] = None,
|
|
81
|
+
to_date: Optional[datetime] = None,
|
|
82
|
+
filter_: Optional[dict[str, Any]] = None,
|
|
83
|
+
*,
|
|
84
|
+
session: "Session"
|
|
85
|
+
) -> list[dict[str, Any]]:
|
|
78
86
|
"""
|
|
79
87
|
List the bad file replicas summary. Method used by the rucio-ui.
|
|
80
88
|
:param rse_expression: The RSE expression.
|
|
@@ -94,11 +102,11 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
|
|
|
94
102
|
for rse in list_rses(filters=filter_, session=session):
|
|
95
103
|
rse_clause.append(models.BadReplica.rse_id == rse['id'])
|
|
96
104
|
|
|
97
|
-
if session.bind.dialect.name == 'oracle':
|
|
105
|
+
if session.bind.dialect.name == 'oracle': # type: ignore
|
|
98
106
|
to_days = func.trunc(models.BadReplica.created_at, 'DD')
|
|
99
|
-
elif session.bind.dialect.name == 'mysql':
|
|
107
|
+
elif session.bind.dialect.name == 'mysql': # type: ignore
|
|
100
108
|
to_days = func.date(models.BadReplica.created_at)
|
|
101
|
-
elif session.bind.dialect.name == 'postgresql':
|
|
109
|
+
elif session.bind.dialect.name == 'postgresql': # type: ignore
|
|
102
110
|
to_days = func.date_trunc('day', models.BadReplica.created_at)
|
|
103
111
|
else:
|
|
104
112
|
to_days = func.strftime(models.BadReplica.created_at, '%Y-%m-%d')
|
|
@@ -137,7 +145,22 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
|
|
|
137
145
|
|
|
138
146
|
|
|
139
147
|
@read_session
|
|
140
|
-
def __exist_replicas(
|
|
148
|
+
def __exist_replicas(
|
|
149
|
+
rse_id: str,
|
|
150
|
+
replicas: list[tuple[Optional[str], Optional[str], Optional[str]]],
|
|
151
|
+
*,
|
|
152
|
+
session: "Session"
|
|
153
|
+
) -> list[
|
|
154
|
+
tuple
|
|
155
|
+
[
|
|
156
|
+
str,
|
|
157
|
+
str,
|
|
158
|
+
str,
|
|
159
|
+
bool,
|
|
160
|
+
bool,
|
|
161
|
+
Optional[int]
|
|
162
|
+
]
|
|
163
|
+
]:
|
|
141
164
|
"""
|
|
142
165
|
Internal method to check if a replica exists at a given site.
|
|
143
166
|
:param rse_id: The RSE id.
|
|
@@ -153,8 +176,8 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
|
|
|
153
176
|
"""
|
|
154
177
|
|
|
155
178
|
return_list = []
|
|
156
|
-
path_clause = []
|
|
157
|
-
did_clause = []
|
|
179
|
+
path_clause: list[ColumnElement[bool]] = []
|
|
180
|
+
did_clause: list[ColumnElement[bool]] = []
|
|
158
181
|
for scope, name, path in replicas:
|
|
159
182
|
if path:
|
|
160
183
|
path_clause.append(models.RSEFileAssociation.path == path)
|
|
@@ -219,7 +242,17 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
|
|
|
219
242
|
|
|
220
243
|
|
|
221
244
|
@read_session
|
|
222
|
-
def list_bad_replicas_status(
|
|
245
|
+
def list_bad_replicas_status(
|
|
246
|
+
state: BadFilesStatus = BadFilesStatus.BAD,
|
|
247
|
+
rse_id: Optional[str] = None,
|
|
248
|
+
younger_than: Optional[datetime] = None,
|
|
249
|
+
older_than: Optional[datetime] = None,
|
|
250
|
+
limit: Optional[int] = None,
|
|
251
|
+
list_pfns: Optional[bool] = False,
|
|
252
|
+
vo: str = 'def',
|
|
253
|
+
*,
|
|
254
|
+
session: "Session"
|
|
255
|
+
) -> list[dict[str, Any]]:
|
|
223
256
|
"""
|
|
224
257
|
List the bad file replicas history states. Method used by the rucio-ui.
|
|
225
258
|
:param state: The state of the file (SUSPICIOUS or BAD).
|
|
@@ -272,7 +305,18 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
|
|
|
272
305
|
|
|
273
306
|
|
|
274
307
|
@transactional_session
|
|
275
|
-
def __declare_bad_file_replicas(
|
|
308
|
+
def __declare_bad_file_replicas(
|
|
309
|
+
pfns: list[Union[str, dict[str, Any]]],
|
|
310
|
+
rse_id: str,
|
|
311
|
+
reason: str,
|
|
312
|
+
issuer: InternalAccount,
|
|
313
|
+
status: BadFilesStatus = BadFilesStatus.BAD,
|
|
314
|
+
scheme: str = 'srm',
|
|
315
|
+
force: bool = False,
|
|
316
|
+
logger: "LoggerFunction" = logging.log,
|
|
317
|
+
*,
|
|
318
|
+
session: "Session"
|
|
319
|
+
) -> list[str]:
|
|
276
320
|
"""
|
|
277
321
|
Declare a list of bad replicas.
|
|
278
322
|
|
|
@@ -285,11 +329,11 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
285
329
|
:param force: boolean, if declaring BAD replica, ignore existing replica status in the bad_replicas table. Default: False
|
|
286
330
|
:param session: The database session in use.
|
|
287
331
|
"""
|
|
288
|
-
unknown_replicas = []
|
|
289
|
-
replicas = []
|
|
290
|
-
path_pfn_dict = {}
|
|
332
|
+
unknown_replicas: list[str] = []
|
|
333
|
+
replicas: list[dict[str, Any]] = []
|
|
334
|
+
path_pfn_dict: dict[str, str] = {}
|
|
291
335
|
|
|
292
|
-
if len(pfns) > 0 and
|
|
336
|
+
if len(pfns) > 0 and is_str_list(pfns):
|
|
293
337
|
# If pfns is a list of PFNs, the scope and names need to be extracted from the path
|
|
294
338
|
rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
|
|
295
339
|
proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
|
|
@@ -319,8 +363,8 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
319
363
|
else:
|
|
320
364
|
# If pfns is a list of replicas, just use scope, name and rse_id
|
|
321
365
|
for pfn in pfns:
|
|
322
|
-
replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status})
|
|
323
|
-
logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path")
|
|
366
|
+
replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status}) # type: ignore
|
|
367
|
+
logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path") # type: ignore
|
|
324
368
|
|
|
325
369
|
replicas_list = []
|
|
326
370
|
for replica in replicas:
|
|
@@ -381,7 +425,15 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
381
425
|
|
|
382
426
|
|
|
383
427
|
@transactional_session
|
|
384
|
-
def add_bad_dids(
|
|
428
|
+
def add_bad_dids(
|
|
429
|
+
dids: "Iterable[dict[str, Any]]",
|
|
430
|
+
rse_id: str,
|
|
431
|
+
reason: str,
|
|
432
|
+
issuer: InternalAccount,
|
|
433
|
+
state: BadFilesStatus = BadFilesStatus.BAD,
|
|
434
|
+
*,
|
|
435
|
+
session: "Session"
|
|
436
|
+
) -> list[str]:
|
|
385
437
|
"""
|
|
386
438
|
Declare a list of bad replicas.
|
|
387
439
|
|
|
@@ -438,8 +490,15 @@ def add_bad_dids(dids, rse_id, reason, issuer, state=BadFilesStatus.BAD, *, sess
|
|
|
438
490
|
|
|
439
491
|
|
|
440
492
|
@transactional_session
|
|
441
|
-
def declare_bad_file_replicas(
|
|
442
|
-
|
|
493
|
+
def declare_bad_file_replicas(
|
|
494
|
+
replicas: list[Union[str, dict[str, Any]]],
|
|
495
|
+
reason: str,
|
|
496
|
+
issuer: InternalAccount,
|
|
497
|
+
status: BadFilesStatus = BadFilesStatus.BAD,
|
|
498
|
+
force: bool = False,
|
|
499
|
+
*,
|
|
500
|
+
session: "Session"
|
|
501
|
+
) -> dict[str, list[str]]:
|
|
443
502
|
"""
|
|
444
503
|
Declare a list of bad replicas.
|
|
445
504
|
|
|
@@ -451,7 +510,7 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
|
|
|
451
510
|
:param session: The database session in use.
|
|
452
511
|
:returns: Dictionary {rse_id -> [replicas failed to declare with errors]}
|
|
453
512
|
"""
|
|
454
|
-
unknown_replicas = {}
|
|
513
|
+
unknown_replicas: dict[str, list[str]] = {}
|
|
455
514
|
if replicas:
|
|
456
515
|
type_ = type(replicas[0])
|
|
457
516
|
files_to_declare = {}
|
|
@@ -459,11 +518,11 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
|
|
|
459
518
|
for replica in replicas:
|
|
460
519
|
if not isinstance(replica, type_):
|
|
461
520
|
raise exception.InvalidType('Replicas must be specified either as a list of string or a list of dicts')
|
|
462
|
-
if type_
|
|
521
|
+
if type_ is str:
|
|
463
522
|
scheme, files_to_declare, unknown_replicas = get_pfn_to_rse(replicas, vo=issuer.vo, session=session)
|
|
464
523
|
else:
|
|
465
524
|
for replica in replicas:
|
|
466
|
-
rse_id = replica['rse_id']
|
|
525
|
+
rse_id = replica['rse_id'] # type: ignore
|
|
467
526
|
files_to_declare.setdefault(rse_id, []).append(replica)
|
|
468
527
|
for rse_id in files_to_declare:
|
|
469
528
|
notdeclared = __declare_bad_file_replicas(files_to_declare[rse_id], rse_id, reason, issuer,
|
|
@@ -475,7 +534,12 @@ def declare_bad_file_replicas(replicas: list, reason: str, issuer, status=BadFil
|
|
|
475
534
|
|
|
476
535
|
|
|
477
536
|
@read_session
|
|
478
|
-
def get_pfn_to_rse(
|
|
537
|
+
def get_pfn_to_rse(
|
|
538
|
+
pfns: "Iterable[str]",
|
|
539
|
+
vo: str = 'def',
|
|
540
|
+
*,
|
|
541
|
+
session: "Session"
|
|
542
|
+
) -> tuple[Optional[str], dict[str, Any], dict[str, list[str]]]:
|
|
479
543
|
"""
|
|
480
544
|
Get the RSE associated to a list of PFNs.
|
|
481
545
|
|
|
@@ -552,7 +616,10 @@ def get_pfn_to_rse(pfns, vo='def', *, session: "Session"):
|
|
|
552
616
|
|
|
553
617
|
|
|
554
618
|
@read_session
|
|
555
|
-
def get_bad_replicas_backlog(
|
|
619
|
+
def get_bad_replicas_backlog(
|
|
620
|
+
*,
|
|
621
|
+
session: "Session"
|
|
622
|
+
) -> dict[str, int]:
|
|
556
623
|
"""
|
|
557
624
|
Get the replica backlog by RSE.
|
|
558
625
|
|
|
@@ -587,7 +654,14 @@ def get_bad_replicas_backlog(*, session: "Session"):
|
|
|
587
654
|
|
|
588
655
|
|
|
589
656
|
@read_session
|
|
590
|
-
def list_bad_replicas(
|
|
657
|
+
def list_bad_replicas(
|
|
658
|
+
limit: int = 10000,
|
|
659
|
+
thread: Optional[int] = None,
|
|
660
|
+
total_threads: Optional[int] = None,
|
|
661
|
+
rses: Optional['Iterable[dict[str, Any]]'] = None,
|
|
662
|
+
*,
|
|
663
|
+
session: "Session"
|
|
664
|
+
) -> list[dict[str, Any]]:
|
|
591
665
|
"""
|
|
592
666
|
List RSE File replicas with no locks.
|
|
593
667
|
|
|
@@ -634,7 +708,13 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, *
|
|
|
634
708
|
|
|
635
709
|
|
|
636
710
|
@stream_session
|
|
637
|
-
def get_did_from_pfns(
|
|
711
|
+
def get_did_from_pfns(
|
|
712
|
+
pfns: "Iterable[str]",
|
|
713
|
+
rse_id: Optional[str] = None,
|
|
714
|
+
vo: str = 'def',
|
|
715
|
+
*,
|
|
716
|
+
session: "Session"
|
|
717
|
+
) -> 'Iterator[dict[str, dict[str, Any]]]':
|
|
638
718
|
"""
|
|
639
719
|
Get the DIDs associated to a PFN on one given RSE
|
|
640
720
|
|
|
@@ -656,7 +736,7 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
|
|
|
656
736
|
pfns = dict_rse[rse_id]
|
|
657
737
|
rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
|
|
658
738
|
pfndict = {}
|
|
659
|
-
proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
|
|
739
|
+
proto: RSEProtocol = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
|
|
660
740
|
if rse_info['deterministic']:
|
|
661
741
|
scope_proto = rsemgr.get_scope_protocol(vo=vo)
|
|
662
742
|
parsed_pfn = proto.parse_pfns(pfns=pfns)
|
|
@@ -685,7 +765,10 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
|
|
|
685
765
|
yield {pfndict[pfn]: {'scope': scope, 'name': name}}
|
|
686
766
|
|
|
687
767
|
|
|
688
|
-
def _pick_n_random(
|
|
768
|
+
def _pick_n_random(
|
|
769
|
+
nrandom: int,
|
|
770
|
+
generator: 'Iterable[Any]'
|
|
771
|
+
) -> 'Iterator[Any]':
|
|
689
772
|
"""
|
|
690
773
|
Select n random elements from the generator
|
|
691
774
|
"""
|
|
@@ -720,7 +803,11 @@ def _pick_n_random(nrandom, generator):
|
|
|
720
803
|
yield r
|
|
721
804
|
|
|
722
805
|
|
|
723
|
-
def _list_files_wo_replicas(
|
|
806
|
+
def _list_files_wo_replicas(
|
|
807
|
+
files_wo_replica: "Iterable[dict[str, Any]]",
|
|
808
|
+
*,
|
|
809
|
+
session: "Session"
|
|
810
|
+
) -> 'Iterator[tuple[str, str, int, str, str]]':
|
|
724
811
|
if files_wo_replica:
|
|
725
812
|
file_wo_clause = []
|
|
726
813
|
for file in sorted(files_wo_replica, key=lambda f: (f['scope'], f['name'])):
|
|
@@ -744,7 +831,7 @@ def _list_files_wo_replicas(files_wo_replica, *, session: "Session"):
|
|
|
744
831
|
yield scope, name, bytes_, md5, adler32
|
|
745
832
|
|
|
746
833
|
|
|
747
|
-
def get_vp_endpoint():
|
|
834
|
+
def get_vp_endpoint() -> str:
|
|
748
835
|
"""
|
|
749
836
|
VP endpoint is the Virtual Placement server.
|
|
750
837
|
Once VP is integrated in Rucio it won't be needed.
|
|
@@ -753,7 +840,11 @@ def get_vp_endpoint():
|
|
|
753
840
|
return vp_endpoint
|
|
754
841
|
|
|
755
842
|
|
|
756
|
-
def get_multi_cache_prefix(
|
|
843
|
+
def get_multi_cache_prefix(
|
|
844
|
+
cache_site: str,
|
|
845
|
+
filename: str,
|
|
846
|
+
logger: "LoggerFunction" = logging.log
|
|
847
|
+
) -> str:
|
|
757
848
|
"""
|
|
758
849
|
for a givent cache site and filename, return address of the cache node that
|
|
759
850
|
should be prefixed.
|
|
@@ -780,10 +871,10 @@ def get_multi_cache_prefix(cache_site, filename, logger=logging.log):
|
|
|
780
871
|
logger(logging.WARNING, 'In get_multi_cache_prefix, could not access {}. Excaption:{}'.format(vp_endpoint, re))
|
|
781
872
|
return ''
|
|
782
873
|
|
|
783
|
-
if cache_site not in x_caches:
|
|
874
|
+
if cache_site not in x_caches: # type: ignore
|
|
784
875
|
return ''
|
|
785
876
|
|
|
786
|
-
xcache_site = x_caches[cache_site]
|
|
877
|
+
xcache_site = x_caches[cache_site] # type: ignore
|
|
787
878
|
h = float(
|
|
788
879
|
unpack('Q', sha256(filename.encode('utf-8')).digest()[:8])[0]) / 2**64
|
|
789
880
|
for irange in xcache_site['ranges']:
|
|
@@ -795,8 +886,8 @@ def get_multi_cache_prefix(cache_site, filename, logger=logging.log):
|
|
|
795
886
|
def _get_list_replicas_protocols(
|
|
796
887
|
rse_id: str,
|
|
797
888
|
domain: str,
|
|
798
|
-
schemes:
|
|
799
|
-
additional_schemes: "
|
|
889
|
+
schemes: Optional[list[str]],
|
|
890
|
+
additional_schemes: "Iterable[str]",
|
|
800
891
|
session: "Session"
|
|
801
892
|
) -> "list[tuple[str, RSEProtocol, int]]":
|
|
802
893
|
"""
|
|
@@ -851,9 +942,9 @@ def _build_list_replicas_pfn(
|
|
|
851
942
|
protocol: "RSEProtocol",
|
|
852
943
|
path: str,
|
|
853
944
|
sign_urls: bool,
|
|
854
|
-
signature_lifetime: int,
|
|
855
|
-
client_location:
|
|
856
|
-
logger=logging.log,
|
|
945
|
+
signature_lifetime: Optional[int],
|
|
946
|
+
client_location: Optional[dict[str, Any]],
|
|
947
|
+
logger: "LoggerFunction" = logging.log,
|
|
857
948
|
*,
|
|
858
949
|
session: "Session",
|
|
859
950
|
) -> str:
|
|
@@ -862,9 +953,12 @@ def _build_list_replicas_pfn(
|
|
|
862
953
|
If needed, sign the PFN url
|
|
863
954
|
If relevant, add the server-side root proxy to the pfn url
|
|
864
955
|
"""
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
956
|
+
lfn: LFNDict = {
|
|
957
|
+
'scope': scope.external, # type: ignore (scope.external might be None)
|
|
958
|
+
'name': name,
|
|
959
|
+
'path': path
|
|
960
|
+
}
|
|
961
|
+
pfn: str = list(protocol.lfns2pfns(lfns=lfn).values())[0]
|
|
868
962
|
|
|
869
963
|
# do we need to sign the URLs?
|
|
870
964
|
if sign_urls and protocol.attributes['scheme'] == 'https':
|
|
@@ -908,7 +1002,7 @@ def _build_list_replicas_pfn(
|
|
|
908
1002
|
# don't forget to mangle gfal-style davs URL into generic https URL
|
|
909
1003
|
pfn = f"root://{root_proxy_internal}//{pfn.replace('davs://', 'https://')}"
|
|
910
1004
|
|
|
911
|
-
simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE)
|
|
1005
|
+
simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE, session=session)
|
|
912
1006
|
|
|
913
1007
|
if simulate_multirange is not None:
|
|
914
1008
|
try:
|
|
@@ -925,8 +1019,21 @@ def _build_list_replicas_pfn(
|
|
|
925
1019
|
return pfn
|
|
926
1020
|
|
|
927
1021
|
|
|
928
|
-
def _list_replicas(
|
|
929
|
-
|
|
1022
|
+
def _list_replicas(
|
|
1023
|
+
replicas: "Iterable[tuple]",
|
|
1024
|
+
show_pfns: bool,
|
|
1025
|
+
schemes: Optional[list[str]],
|
|
1026
|
+
files_wo_replica: "Iterable[dict[str, Any]]",
|
|
1027
|
+
client_location: Optional[dict[str, Any]],
|
|
1028
|
+
domain: Optional[str],
|
|
1029
|
+
sign_urls: bool,
|
|
1030
|
+
signature_lifetime: Optional[int],
|
|
1031
|
+
resolve_parents: bool,
|
|
1032
|
+
filters: dict[str, Any],
|
|
1033
|
+
by_rse_name: bool,
|
|
1034
|
+
*,
|
|
1035
|
+
session: "Session"
|
|
1036
|
+
) -> "Iterator[dict[str, Any]]":
|
|
930
1037
|
|
|
931
1038
|
# the `domain` variable name will be re-used throughout the function with different values
|
|
932
1039
|
input_domain = domain
|
|
@@ -948,7 +1055,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
|
|
|
948
1055
|
pfns = {}
|
|
949
1056
|
for scope, name, archive_scope, archive_name, bytes_, md5, adler32, path, state, rse_id, rse, rse_type, volatile in replica_group:
|
|
950
1057
|
if isinstance(archive_scope, str):
|
|
951
|
-
archive_scope = InternalScope(archive_scope,
|
|
1058
|
+
archive_scope = InternalScope(archive_scope, from_external=False)
|
|
952
1059
|
|
|
953
1060
|
is_archive = bool(archive_scope and archive_name)
|
|
954
1061
|
|
|
@@ -1005,7 +1112,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
|
|
|
1005
1112
|
try:
|
|
1006
1113
|
path = pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)]
|
|
1007
1114
|
except KeyError: # No cache entry scope:name found for this protocol
|
|
1008
|
-
path = protocol._get_path(t_scope, t_name)
|
|
1115
|
+
path = protocol._get_path(t_scope, t_name) # type: ignore (t_scope is InternalScope instead of str)
|
|
1009
1116
|
pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)] = path
|
|
1010
1117
|
|
|
1011
1118
|
try:
|
|
@@ -1086,24 +1193,24 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
|
|
|
1086
1193
|
@stream_session
|
|
1087
1194
|
def list_replicas(
|
|
1088
1195
|
dids: "Sequence[dict[str, Any]]",
|
|
1089
|
-
schemes:
|
|
1196
|
+
schemes: Optional[list[str]] = None,
|
|
1090
1197
|
unavailable: bool = False,
|
|
1091
|
-
request_id:
|
|
1198
|
+
request_id: Optional[str] = None,
|
|
1092
1199
|
ignore_availability: bool = True,
|
|
1093
1200
|
all_states: bool = False,
|
|
1094
1201
|
pfns: bool = True,
|
|
1095
|
-
rse_expression:
|
|
1096
|
-
client_location:
|
|
1097
|
-
domain:
|
|
1202
|
+
rse_expression: Optional[str] = None,
|
|
1203
|
+
client_location: Optional[dict[str, Any]] = None,
|
|
1204
|
+
domain: Optional[str] = None,
|
|
1098
1205
|
sign_urls: bool = False,
|
|
1099
1206
|
signature_lifetime: "Optional[int]" = None,
|
|
1100
1207
|
resolve_archives: bool = True,
|
|
1101
1208
|
resolve_parents: bool = False,
|
|
1102
|
-
nrandom:
|
|
1103
|
-
updated_after:
|
|
1209
|
+
nrandom: Optional[int] = None,
|
|
1210
|
+
updated_after: Optional[datetime] = None,
|
|
1104
1211
|
by_rse_name: bool = False,
|
|
1105
1212
|
*, session: "Session",
|
|
1106
|
-
):
|
|
1213
|
+
) -> 'Iterator':
|
|
1107
1214
|
"""
|
|
1108
1215
|
List file replicas for a list of data identifiers (DIDs).
|
|
1109
1216
|
|
|
@@ -1185,7 +1292,11 @@ def list_replicas(
|
|
|
1185
1292
|
|
|
1186
1293
|
return stmt.subquery()
|
|
1187
1294
|
|
|
1188
|
-
def _resolve_collection_files(
|
|
1295
|
+
def _resolve_collection_files(
|
|
1296
|
+
temp_table: Any,
|
|
1297
|
+
*,
|
|
1298
|
+
session: "Session"
|
|
1299
|
+
) -> tuple[int, Any]:
|
|
1189
1300
|
"""
|
|
1190
1301
|
Find all FILE dids contained in collections from temp_table and return them in a newly
|
|
1191
1302
|
created temporary table.
|
|
@@ -1202,7 +1313,10 @@ def list_replicas(
|
|
|
1202
1313
|
|
|
1203
1314
|
return session.execute(stmt).rowcount, resolved_files_temp_table
|
|
1204
1315
|
|
|
1205
|
-
def _list_replicas_for_collection_files_stmt(
|
|
1316
|
+
def _list_replicas_for_collection_files_stmt(
|
|
1317
|
+
temp_table: Any,
|
|
1318
|
+
replicas_subquery: "Subquery"
|
|
1319
|
+
) -> "Select":
|
|
1206
1320
|
"""
|
|
1207
1321
|
Build a query for listing replicas of files resolved from containers/datasets
|
|
1208
1322
|
|
|
@@ -1229,7 +1343,10 @@ def list_replicas(
|
|
|
1229
1343
|
replicas_subquery.c.name == temp_table.name),
|
|
1230
1344
|
)
|
|
1231
1345
|
|
|
1232
|
-
def _list_replicas_for_constituents_stmt(
|
|
1346
|
+
def _list_replicas_for_constituents_stmt(
|
|
1347
|
+
temp_table: Any,
|
|
1348
|
+
replicas_subquery: "Subquery"
|
|
1349
|
+
) -> "Select":
|
|
1233
1350
|
"""
|
|
1234
1351
|
Build a query for listing replicas of archives containing the files(constituents) given as input.
|
|
1235
1352
|
i.e. for a file scope:file.log which exists in scope:archive.tar.gz, it will return the replicas
|
|
@@ -1266,7 +1383,10 @@ def list_replicas(
|
|
|
1266
1383
|
replicas_subquery.c.name == models.ConstituentAssociation.name),
|
|
1267
1384
|
)
|
|
1268
1385
|
|
|
1269
|
-
def _list_replicas_for_input_files_stmt(
|
|
1386
|
+
def _list_replicas_for_input_files_stmt(
|
|
1387
|
+
temp_table: Any,
|
|
1388
|
+
replicas_subquery: "Subquery"
|
|
1389
|
+
) -> "Select":
|
|
1270
1390
|
"""
|
|
1271
1391
|
Builds a query which list the replicas of FILEs from users input, but ignores
|
|
1272
1392
|
collections in the same input.
|
|
@@ -1300,7 +1420,11 @@ def list_replicas(
|
|
|
1300
1420
|
replicas_subquery.c.name == temp_table.name),
|
|
1301
1421
|
)
|
|
1302
1422
|
|
|
1303
|
-
def _inspect_dids(
|
|
1423
|
+
def _inspect_dids(
|
|
1424
|
+
temp_table: Any,
|
|
1425
|
+
*,
|
|
1426
|
+
session: "Session"
|
|
1427
|
+
) -> tuple[int, int, int]:
|
|
1304
1428
|
"""
|
|
1305
1429
|
Find how many files, collections and constituents are among the dids in the temp_table
|
|
1306
1430
|
"""
|
|
@@ -1328,7 +1452,7 @@ def list_replicas(
|
|
|
1328
1452
|
else:
|
|
1329
1453
|
filter_ = {'vo': 'def'}
|
|
1330
1454
|
|
|
1331
|
-
dids = {(did['scope'], did['name']): did for did in dids} # Deduplicate input
|
|
1455
|
+
dids = {(did['scope'], did['name']): did for did in dids} # type: ignore (Deduplicate input)
|
|
1332
1456
|
if not dids:
|
|
1333
1457
|
return
|
|
1334
1458
|
|
|
@@ -1382,17 +1506,17 @@ def list_replicas(
|
|
|
1382
1506
|
# (for example: scheme; or client_location/domain). We don't have any guarantee that
|
|
1383
1507
|
# those, python, filters will not drop the replicas which we just selected randomly.
|
|
1384
1508
|
stmt = select(
|
|
1385
|
-
resolved_files_temp_table.scope.label('scope'),
|
|
1386
|
-
resolved_files_temp_table.name.label('name'),
|
|
1509
|
+
resolved_files_temp_table.scope.label('scope'), # type: ignore (resolved_files_temp_table might be None)
|
|
1510
|
+
resolved_files_temp_table.name.label('name'), # type: ignore (resolved_files_temp_table might be None)
|
|
1387
1511
|
).where(
|
|
1388
1512
|
exists(
|
|
1389
1513
|
select(1)
|
|
1390
1514
|
).where(
|
|
1391
|
-
replicas_subquery.c.scope == resolved_files_temp_table.scope,
|
|
1392
|
-
replicas_subquery.c.name == resolved_files_temp_table.name
|
|
1515
|
+
replicas_subquery.c.scope == resolved_files_temp_table.scope, # type: ignore (resolved_files_temp_table might be None)
|
|
1516
|
+
replicas_subquery.c.name == resolved_files_temp_table.name # type: ignore (resolved_files_temp_table might be None)
|
|
1393
1517
|
)
|
|
1394
1518
|
).order_by(
|
|
1395
|
-
literal_column('dbms_random.value') if session.bind.dialect.name == 'oracle' else func.random()
|
|
1519
|
+
literal_column('dbms_random.value') if session.bind.dialect.name == 'oracle' else func.random() # type: ignore
|
|
1396
1520
|
).limit(
|
|
1397
1521
|
# slightly overshoot to reduce the probability that python-side filtering will
|
|
1398
1522
|
# leave us with less than nrandom replicas.
|
|
@@ -1416,7 +1540,7 @@ def list_replicas(
|
|
|
1416
1540
|
random_replicas = list(
|
|
1417
1541
|
_pick_n_random(
|
|
1418
1542
|
nrandom,
|
|
1419
|
-
_list_replicas(replica_tuples, pfns, schemes, [], client_location, domain,
|
|
1543
|
+
_list_replicas(replica_tuples, pfns, schemes, [], client_location, domain, # type: ignore (replica_tuples, pending SQLA2.1: https://github.com/rucio/rucio/discussions/6615)
|
|
1420
1544
|
sign_urls, signature_lifetime, resolve_parents, filter_, by_rse_name, session=session)
|
|
1421
1545
|
)
|
|
1422
1546
|
)
|
|
@@ -1432,7 +1556,7 @@ def list_replicas(
|
|
|
1432
1556
|
stmt = replica_sources[0].order_by('scope', 'name')
|
|
1433
1557
|
replica_tuples = session.execute(stmt)
|
|
1434
1558
|
else:
|
|
1435
|
-
if session.bind.dialect.name == 'mysql':
|
|
1559
|
+
if session.bind.dialect.name == 'mysql': # type: ignore
|
|
1436
1560
|
# On mysql, perform both queries independently and merge their result in python.
|
|
1437
1561
|
# The union query fails with "Can't reopen table"
|
|
1438
1562
|
replica_tuples = heapq.merge(
|
|
@@ -1444,14 +1568,20 @@ def list_replicas(
|
|
|
1444
1568
|
replica_tuples = session.execute(stmt)
|
|
1445
1569
|
|
|
1446
1570
|
yield from _pick_n_random(
|
|
1447
|
-
nrandom,
|
|
1448
|
-
_list_replicas(replica_tuples, pfns, schemes, [], client_location, domain,
|
|
1571
|
+
nrandom, # type: ignore (nrandom is not None)
|
|
1572
|
+
_list_replicas(replica_tuples, pfns, schemes, [], client_location, domain, # type: ignore (replica_tuples, pending SQLA2.1: https://github.com/rucio/rucio/discussions/6615)
|
|
1449
1573
|
sign_urls, signature_lifetime, resolve_parents, filter_, by_rse_name, session=session)
|
|
1450
1574
|
)
|
|
1451
1575
|
|
|
1452
1576
|
|
|
1453
1577
|
@transactional_session
|
|
1454
|
-
def __bulk_add_new_file_dids(
|
|
1578
|
+
def __bulk_add_new_file_dids(
|
|
1579
|
+
files: "Iterable[dict[str, Any]]",
|
|
1580
|
+
account: InternalAccount,
|
|
1581
|
+
dataset_meta: Optional["Mapping[str, Any]"] = None,
|
|
1582
|
+
*,
|
|
1583
|
+
session: "Session"
|
|
1584
|
+
) -> Literal[True]:
|
|
1455
1585
|
"""
|
|
1456
1586
|
Bulk add new dids.
|
|
1457
1587
|
|
|
@@ -1498,7 +1628,13 @@ def __bulk_add_new_file_dids(files, account, dataset_meta=None, *, session: "Ses
|
|
|
1498
1628
|
|
|
1499
1629
|
|
|
1500
1630
|
@transactional_session
|
|
1501
|
-
def __bulk_add_file_dids(
|
|
1631
|
+
def __bulk_add_file_dids(
|
|
1632
|
+
files: "Iterable[dict[str, Any]]",
|
|
1633
|
+
account: InternalAccount,
|
|
1634
|
+
dataset_meta: Optional["Mapping[str, Any]"] = None,
|
|
1635
|
+
*,
|
|
1636
|
+
session: "Session"
|
|
1637
|
+
) -> list[dict[str, Any]]:
|
|
1502
1638
|
"""
|
|
1503
1639
|
Bulk add new dids.
|
|
1504
1640
|
|
|
@@ -1542,12 +1678,12 @@ def __bulk_add_file_dids(files, account, dataset_meta=None, *, session: "Session
|
|
|
1542
1678
|
return new_files + available_files
|
|
1543
1679
|
|
|
1544
1680
|
|
|
1545
|
-
def tombstone_from_delay(tombstone_delay):
|
|
1681
|
+
def tombstone_from_delay(tombstone_delay: Optional[Union[str, timedelta]]) -> Optional[datetime]:
|
|
1546
1682
|
# Tolerate None for tombstone_delay
|
|
1547
1683
|
if not tombstone_delay:
|
|
1548
1684
|
return None
|
|
1549
1685
|
|
|
1550
|
-
tombstone_delay = timedelta(seconds=int(tombstone_delay))
|
|
1686
|
+
tombstone_delay = timedelta(seconds=int(tombstone_delay)) # type: ignore
|
|
1551
1687
|
|
|
1552
1688
|
if not tombstone_delay:
|
|
1553
1689
|
return None
|
|
@@ -1559,7 +1695,13 @@ def tombstone_from_delay(tombstone_delay):
|
|
|
1559
1695
|
|
|
1560
1696
|
|
|
1561
1697
|
@transactional_session
|
|
1562
|
-
def __bulk_add_replicas(
|
|
1698
|
+
def __bulk_add_replicas(
|
|
1699
|
+
rse_id: str,
|
|
1700
|
+
files: "Iterable[dict[str, Any]]",
|
|
1701
|
+
account: InternalAccount,
|
|
1702
|
+
*,
|
|
1703
|
+
session: "Session"
|
|
1704
|
+
) -> tuple[int, int]:
|
|
1563
1705
|
"""
|
|
1564
1706
|
Bulk add new dids.
|
|
1565
1707
|
|
|
@@ -1630,8 +1772,15 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
|
|
|
1630
1772
|
|
|
1631
1773
|
|
|
1632
1774
|
@transactional_session
|
|
1633
|
-
def add_replicas(
|
|
1634
|
-
|
|
1775
|
+
def add_replicas(
|
|
1776
|
+
rse_id: str,
|
|
1777
|
+
files: "Iterable[dict[str, Any]]",
|
|
1778
|
+
account: InternalAccount,
|
|
1779
|
+
ignore_availability: bool = True,
|
|
1780
|
+
dataset_meta: Optional["Mapping[str, Any]"] = None,
|
|
1781
|
+
*,
|
|
1782
|
+
session: "Session"
|
|
1783
|
+
) -> None:
|
|
1635
1784
|
"""
|
|
1636
1785
|
Bulk add file replicas.
|
|
1637
1786
|
|
|
@@ -1640,8 +1789,6 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1640
1789
|
:param account: The account owner.
|
|
1641
1790
|
:param ignore_availability: Ignore the RSE blocklisting.
|
|
1642
1791
|
:param session: The database session in use.
|
|
1643
|
-
|
|
1644
|
-
:returns: list of replicas.
|
|
1645
1792
|
"""
|
|
1646
1793
|
|
|
1647
1794
|
def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=None):
|
|
@@ -1685,18 +1832,24 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1685
1832
|
else:
|
|
1686
1833
|
# Check that the pfns match to the expected pfns
|
|
1687
1834
|
lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
|
|
1688
|
-
pfns[scheme] = clean_pfns(pfns[scheme])
|
|
1835
|
+
pfns[scheme] = set(clean_pfns(pfns[scheme]))
|
|
1836
|
+
expected_pfns = set()
|
|
1689
1837
|
|
|
1690
1838
|
for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan'):
|
|
1691
|
-
|
|
1839
|
+
expected_pfns.update(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan',
|
|
1840
|
+
protocol_attr=protocol_attr))
|
|
1841
|
+
pfns[scheme] -= expected_pfns
|
|
1692
1842
|
|
|
1693
1843
|
if len(pfns[scheme]) > 0:
|
|
1694
1844
|
for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='lan'):
|
|
1695
|
-
|
|
1845
|
+
expected_pfns.update(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='lan',
|
|
1846
|
+
protocol_attr=protocol_attr))
|
|
1847
|
+
pfns[scheme] -= expected_pfns
|
|
1696
1848
|
|
|
1697
1849
|
if len(pfns[scheme]) > 0:
|
|
1698
1850
|
# pfns not found in wan or lan
|
|
1699
|
-
|
|
1851
|
+
pfns_scheme = pfns[scheme]
|
|
1852
|
+
raise exception.InvalidPath(f"One of the PFNs provided {pfns_scheme!r} for {lfns!r} does not match the Rucio expected PFNs: {expected_pfns!r}")
|
|
1700
1853
|
|
|
1701
1854
|
nbfiles, bytes_ = __bulk_add_replicas(rse_id=rse_id, files=files, account=account, session=session)
|
|
1702
1855
|
increase(rse_id=rse_id, files=nbfiles, bytes_=bytes_, session=session)
|
|
@@ -1709,16 +1862,16 @@ def add_replica(
|
|
|
1709
1862
|
name: str,
|
|
1710
1863
|
bytes_: int,
|
|
1711
1864
|
account: models.InternalAccount,
|
|
1712
|
-
adler32:
|
|
1713
|
-
md5:
|
|
1714
|
-
dsn:
|
|
1715
|
-
pfn:
|
|
1716
|
-
meta:
|
|
1717
|
-
rules:
|
|
1865
|
+
adler32: Optional[str] = None,
|
|
1866
|
+
md5: Optional[str] = None,
|
|
1867
|
+
dsn: Optional[str] = None,
|
|
1868
|
+
pfn: Optional[str] = None,
|
|
1869
|
+
meta: Optional[dict[str, Any]] = None,
|
|
1870
|
+
rules: Optional[list[dict[str, Any]]] = None,
|
|
1718
1871
|
tombstone: "Optional[datetime]" = None,
|
|
1719
1872
|
*,
|
|
1720
1873
|
session: "Session"
|
|
1721
|
-
) ->
|
|
1874
|
+
) -> list[dict[str, Any]]:
|
|
1722
1875
|
"""
|
|
1723
1876
|
Add File replica.
|
|
1724
1877
|
|
|
@@ -1748,7 +1901,13 @@ def add_replica(
|
|
|
1748
1901
|
|
|
1749
1902
|
@METRICS.time_it
|
|
1750
1903
|
@transactional_session
|
|
1751
|
-
def delete_replicas(
|
|
1904
|
+
def delete_replicas(
|
|
1905
|
+
rse_id: str,
|
|
1906
|
+
files: Optional["Sequence[dict[str, Any]]"],
|
|
1907
|
+
ignore_availability: bool = True,
|
|
1908
|
+
*,
|
|
1909
|
+
session: "Session"
|
|
1910
|
+
) -> None:
|
|
1752
1911
|
"""
|
|
1753
1912
|
Delete file replicas.
|
|
1754
1913
|
|
|
@@ -1847,7 +2006,15 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
|
|
|
1847
2006
|
|
|
1848
2007
|
|
|
1849
2008
|
@transactional_session
|
|
1850
|
-
def __cleanup_after_replica_deletion(
|
|
2009
|
+
def __cleanup_after_replica_deletion(
|
|
2010
|
+
scope_name_temp_table: Any,
|
|
2011
|
+
scope_name_temp_table2: Any,
|
|
2012
|
+
association_temp_table: Any,
|
|
2013
|
+
rse_id: str,
|
|
2014
|
+
files: "Iterable[dict[str, Any]]",
|
|
2015
|
+
*,
|
|
2016
|
+
session: "Session"
|
|
2017
|
+
) -> None:
|
|
1851
2018
|
"""
|
|
1852
2019
|
Perform update of collections/archive associations/dids after the removal of their replicas
|
|
1853
2020
|
:param rse_id: the rse id
|
|
@@ -2336,7 +2503,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2336
2503
|
|
|
2337
2504
|
|
|
2338
2505
|
@transactional_session
|
|
2339
|
-
def get_replica(
|
|
2506
|
+
def get_replica(
|
|
2507
|
+
rse_id: str,
|
|
2508
|
+
scope: InternalScope,
|
|
2509
|
+
name: str,
|
|
2510
|
+
*,
|
|
2511
|
+
session: "Session"
|
|
2512
|
+
) -> dict[str, Any]:
|
|
2340
2513
|
"""
|
|
2341
2514
|
Get File replica.
|
|
2342
2515
|
|
|
@@ -2361,7 +2534,15 @@ def get_replica(rse_id, scope, name, *, session: "Session"):
|
|
|
2361
2534
|
|
|
2362
2535
|
|
|
2363
2536
|
@transactional_session
|
|
2364
|
-
def list_and_mark_unlocked_replicas(
|
|
2537
|
+
def list_and_mark_unlocked_replicas(
|
|
2538
|
+
limit: int,
|
|
2539
|
+
bytes_: Optional[int] = None,
|
|
2540
|
+
rse_id: Optional[str] = None,
|
|
2541
|
+
delay_seconds: int = 600,
|
|
2542
|
+
only_delete_obsolete: bool = False,
|
|
2543
|
+
*,
|
|
2544
|
+
session: "Session"
|
|
2545
|
+
) -> list[dict[str, Any]]:
|
|
2365
2546
|
"""
|
|
2366
2547
|
List RSE File replicas with no locks.
|
|
2367
2548
|
|
|
@@ -2478,7 +2659,7 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2478
2659
|
if len(rows) >= limit or (not only_delete_obsolete and needed_space is not None and total_bytes > needed_space):
|
|
2479
2660
|
break
|
|
2480
2661
|
if state != ReplicaState.UNAVAILABLE:
|
|
2481
|
-
total_bytes += bytes_
|
|
2662
|
+
total_bytes += bytes_ # type: ignore
|
|
2482
2663
|
|
|
2483
2664
|
rows.append({'scope': scope, 'name': name, 'path': path,
|
|
2484
2665
|
'bytes': bytes_, 'tombstone': tombstone,
|
|
@@ -2513,7 +2694,12 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2513
2694
|
|
|
2514
2695
|
|
|
2515
2696
|
@transactional_session
|
|
2516
|
-
def update_replicas_states(
|
|
2697
|
+
def update_replicas_states(
|
|
2698
|
+
replicas: "Iterable[dict[str, Any]]",
|
|
2699
|
+
nowait: bool = False,
|
|
2700
|
+
*,
|
|
2701
|
+
session: "Session"
|
|
2702
|
+
) -> bool:
|
|
2517
2703
|
"""
|
|
2518
2704
|
Update File replica information and state.
|
|
2519
2705
|
|
|
@@ -2614,7 +2800,11 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
|
|
|
2614
2800
|
|
|
2615
2801
|
|
|
2616
2802
|
@transactional_session
|
|
2617
|
-
def touch_replica(
|
|
2803
|
+
def touch_replica(
|
|
2804
|
+
replica: dict[str, Any],
|
|
2805
|
+
*,
|
|
2806
|
+
session: "Session"
|
|
2807
|
+
) -> bool:
|
|
2618
2808
|
"""
|
|
2619
2809
|
Update the accessed_at timestamp of the given file replica/did but don't wait if row is locked.
|
|
2620
2810
|
|
|
@@ -2699,7 +2889,14 @@ def touch_replica(replica, *, session: "Session"):
|
|
|
2699
2889
|
|
|
2700
2890
|
|
|
2701
2891
|
@transactional_session
|
|
2702
|
-
def update_replica_state(
|
|
2892
|
+
def update_replica_state(
|
|
2893
|
+
rse_id: str,
|
|
2894
|
+
scope: InternalScope,
|
|
2895
|
+
name: str,
|
|
2896
|
+
state: BadFilesStatus,
|
|
2897
|
+
*,
|
|
2898
|
+
session: "Session"
|
|
2899
|
+
) -> bool:
|
|
2703
2900
|
"""
|
|
2704
2901
|
Update File replica information and state.
|
|
2705
2902
|
|
|
@@ -2713,7 +2910,14 @@ def update_replica_state(rse_id, scope, name, state, *, session: "Session"):
|
|
|
2713
2910
|
|
|
2714
2911
|
|
|
2715
2912
|
@transactional_session
|
|
2716
|
-
def get_and_lock_file_replicas(
|
|
2913
|
+
def get_and_lock_file_replicas(
|
|
2914
|
+
scope: InternalScope,
|
|
2915
|
+
name: str,
|
|
2916
|
+
nowait: bool = False,
|
|
2917
|
+
restrict_rses: Optional["Sequence[str]"] = None,
|
|
2918
|
+
*,
|
|
2919
|
+
session: "Session"
|
|
2920
|
+
) -> "Sequence[models.RSEFileAssociation]":
|
|
2717
2921
|
"""
|
|
2718
2922
|
Get file replicas for a specific scope:name.
|
|
2719
2923
|
|
|
@@ -2743,7 +2947,13 @@ def get_and_lock_file_replicas(scope, name, nowait=False, restrict_rses=None, *,
|
|
|
2743
2947
|
|
|
2744
2948
|
|
|
2745
2949
|
@transactional_session
|
|
2746
|
-
def get_source_replicas(
|
|
2950
|
+
def get_source_replicas(
|
|
2951
|
+
scope: InternalScope,
|
|
2952
|
+
name: str,
|
|
2953
|
+
source_rses: Optional["Sequence[str]"] = None,
|
|
2954
|
+
*,
|
|
2955
|
+
session: "Session"
|
|
2956
|
+
) -> "Sequence[str]":
|
|
2747
2957
|
"""
|
|
2748
2958
|
Get source replicas for a specific scope:name.
|
|
2749
2959
|
|
|
@@ -2772,9 +2982,16 @@ def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
|
|
|
2772
2982
|
|
|
2773
2983
|
|
|
2774
2984
|
@transactional_session
|
|
2775
|
-
def get_and_lock_file_replicas_for_dataset(
|
|
2776
|
-
|
|
2777
|
-
|
|
2985
|
+
def get_and_lock_file_replicas_for_dataset(
|
|
2986
|
+
scope: InternalScope,
|
|
2987
|
+
name: str,
|
|
2988
|
+
nowait: bool = False,
|
|
2989
|
+
restrict_rses: Optional["Sequence[str]"] = None,
|
|
2990
|
+
total_threads: Optional[int] = None,
|
|
2991
|
+
thread_id: Optional[int] = None,
|
|
2992
|
+
*,
|
|
2993
|
+
session: "Session"
|
|
2994
|
+
) -> tuple[list[dict[str, Any]], dict[tuple[InternalScope, str], Any]]:
|
|
2778
2995
|
"""
|
|
2779
2996
|
Get file replicas for all files of a dataset.
|
|
2780
2997
|
|
|
@@ -2812,7 +3029,7 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
|
|
|
2812
3029
|
if restrict_rses is not None and len(restrict_rses) < 10:
|
|
2813
3030
|
rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
|
|
2814
3031
|
|
|
2815
|
-
if session.bind.dialect.name == 'postgresql':
|
|
3032
|
+
if session.bind.dialect.name == 'postgresql': # type: ignore
|
|
2816
3033
|
if total_threads and total_threads > 1:
|
|
2817
3034
|
base_stmt = filter_thread_work(session=session,
|
|
2818
3035
|
query=base_stmt,
|
|
@@ -2876,9 +3093,15 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
|
|
|
2876
3093
|
|
|
2877
3094
|
|
|
2878
3095
|
@transactional_session
|
|
2879
|
-
def get_source_replicas_for_dataset(
|
|
2880
|
-
|
|
2881
|
-
|
|
3096
|
+
def get_source_replicas_for_dataset(
|
|
3097
|
+
scope: InternalScope,
|
|
3098
|
+
name: str,
|
|
3099
|
+
source_rses: Optional["Sequence[str]"] = None,
|
|
3100
|
+
total_threads: Optional[int] = None,
|
|
3101
|
+
thread_id: Optional[int] = None,
|
|
3102
|
+
*,
|
|
3103
|
+
session: "Session"
|
|
3104
|
+
) -> dict[tuple[InternalScope, str], Any]:
|
|
2882
3105
|
"""
|
|
2883
3106
|
Get file replicas for all files of a dataset.
|
|
2884
3107
|
|
|
@@ -2955,7 +3178,11 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
|
|
|
2955
3178
|
|
|
2956
3179
|
|
|
2957
3180
|
@read_session
|
|
2958
|
-
def get_replica_atime(
|
|
3181
|
+
def get_replica_atime(
|
|
3182
|
+
replica: dict[str, Any],
|
|
3183
|
+
*,
|
|
3184
|
+
session: "Session"
|
|
3185
|
+
) -> Optional[datetime]:
|
|
2959
3186
|
"""
|
|
2960
3187
|
Get the accessed_at timestamp for a replica. Just for testing.
|
|
2961
3188
|
:param replicas: List of dictionaries {scope, name, rse_id, path}
|
|
@@ -2978,7 +3205,11 @@ def get_replica_atime(replica, *, session: "Session"):
|
|
|
2978
3205
|
|
|
2979
3206
|
|
|
2980
3207
|
@transactional_session
|
|
2981
|
-
def touch_collection_replicas(
|
|
3208
|
+
def touch_collection_replicas(
|
|
3209
|
+
collection_replicas: "Iterable[dict[str, Any]]",
|
|
3210
|
+
*,
|
|
3211
|
+
session: "Session"
|
|
3212
|
+
) -> bool:
|
|
2982
3213
|
"""
|
|
2983
3214
|
Update the accessed_at timestamp of the given collection replicas.
|
|
2984
3215
|
|
|
@@ -3010,7 +3241,13 @@ def touch_collection_replicas(collection_replicas, *, session: "Session"):
|
|
|
3010
3241
|
|
|
3011
3242
|
|
|
3012
3243
|
@stream_session
|
|
3013
|
-
def list_dataset_replicas(
|
|
3244
|
+
def list_dataset_replicas(
|
|
3245
|
+
scope: "InternalScope",
|
|
3246
|
+
name: str,
|
|
3247
|
+
deep: bool = False,
|
|
3248
|
+
*,
|
|
3249
|
+
session: "Session"
|
|
3250
|
+
) -> "Iterator[dict[str, Any]]":
|
|
3014
3251
|
"""
|
|
3015
3252
|
:param scope: The scope of the dataset.
|
|
3016
3253
|
:param name: The name of the dataset.
|
|
@@ -3184,7 +3421,11 @@ def list_dataset_replicas(scope, name, deep=False, *, session: "Session"):
|
|
|
3184
3421
|
|
|
3185
3422
|
|
|
3186
3423
|
@stream_session
|
|
3187
|
-
def list_dataset_replicas_bulk(
|
|
3424
|
+
def list_dataset_replicas_bulk(
|
|
3425
|
+
names_by_intscope: dict[str, Any],
|
|
3426
|
+
*,
|
|
3427
|
+
session: "Session"
|
|
3428
|
+
) -> "Iterator[dict[str, Any]]":
|
|
3188
3429
|
"""
|
|
3189
3430
|
:param names_by_intscope: The dictionary of internal scopes pointing at the list of names.
|
|
3190
3431
|
:param session: Database session to use.
|
|
@@ -3228,7 +3469,14 @@ def list_dataset_replicas_bulk(names_by_intscope, *, session: "Session"):
|
|
|
3228
3469
|
|
|
3229
3470
|
|
|
3230
3471
|
@stream_session
|
|
3231
|
-
def list_dataset_replicas_vp(
|
|
3472
|
+
def list_dataset_replicas_vp(
|
|
3473
|
+
scope: InternalScope,
|
|
3474
|
+
name: str,
|
|
3475
|
+
deep: bool = False,
|
|
3476
|
+
*,
|
|
3477
|
+
session: "Session",
|
|
3478
|
+
logger: "LoggerFunction" = logging.log
|
|
3479
|
+
) -> Union[list[str], "Iterator[dict[str, Any]]"]:
|
|
3232
3480
|
"""
|
|
3233
3481
|
List dataset replicas for a DID (scope:name) using the
|
|
3234
3482
|
Virtual Placement service.
|
|
@@ -3284,7 +3532,13 @@ def list_dataset_replicas_vp(scope, name, deep=False, *, session: "Session", log
|
|
|
3284
3532
|
|
|
3285
3533
|
|
|
3286
3534
|
@stream_session
|
|
3287
|
-
def list_datasets_per_rse(
|
|
3535
|
+
def list_datasets_per_rse(
|
|
3536
|
+
rse_id: str,
|
|
3537
|
+
filters: Optional[dict[str, Any]] = None,
|
|
3538
|
+
limit: Optional[int] = None,
|
|
3539
|
+
*,
|
|
3540
|
+
session: "Session"
|
|
3541
|
+
) -> "Iterator[dict[str, Any]]":
|
|
3288
3542
|
"""
|
|
3289
3543
|
List datasets at a RSE.
|
|
3290
3544
|
|
|
@@ -3317,9 +3571,9 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
|
|
|
3317
3571
|
|
|
3318
3572
|
for (k, v) in filters and filters.items() or []:
|
|
3319
3573
|
if k == 'name' or k == 'scope':
|
|
3320
|
-
v_str = v if k != 'scope' else v.internal
|
|
3574
|
+
v_str = v if k != 'scope' else v.internal # type: ignore
|
|
3321
3575
|
if '*' in v_str or '%' in v_str:
|
|
3322
|
-
if session.bind.dialect.name == 'postgresql': # PostgreSQL escapes automatically
|
|
3576
|
+
if session.bind.dialect.name == 'postgresql': # type: ignore | PostgreSQL escapes automatically
|
|
3323
3577
|
stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%')))
|
|
3324
3578
|
else:
|
|
3325
3579
|
stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%'), escape='\\'))
|
|
@@ -3345,7 +3599,7 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
|
|
|
3345
3599
|
@stream_session
|
|
3346
3600
|
def list_replicas_per_rse(
|
|
3347
3601
|
rse_id: str,
|
|
3348
|
-
limit:
|
|
3602
|
+
limit: Optional[int] = None,
|
|
3349
3603
|
*,
|
|
3350
3604
|
session: "Session"
|
|
3351
3605
|
) -> "Iterator[dict[str, Any]]":
|
|
@@ -3364,7 +3618,13 @@ def list_replicas_per_rse(
|
|
|
3364
3618
|
|
|
3365
3619
|
|
|
3366
3620
|
@transactional_session
|
|
3367
|
-
def get_cleaned_updated_collection_replicas(
|
|
3621
|
+
def get_cleaned_updated_collection_replicas(
|
|
3622
|
+
total_workers: int,
|
|
3623
|
+
worker_number: int,
|
|
3624
|
+
limit: Optional[int] = None,
|
|
3625
|
+
*,
|
|
3626
|
+
session: "Session"
|
|
3627
|
+
) -> list[dict[str, Any]]:
|
|
3368
3628
|
"""
|
|
3369
3629
|
Get update request for collection replicas.
|
|
3370
3630
|
:param total_workers: Number of total workers.
|
|
@@ -3401,12 +3661,12 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
|
|
|
3401
3661
|
session.execute(stmt)
|
|
3402
3662
|
|
|
3403
3663
|
# Delete duplicates
|
|
3404
|
-
if session.bind.dialect.name == 'oracle':
|
|
3664
|
+
if session.bind.dialect.name == 'oracle': # type: ignore
|
|
3405
3665
|
schema = ''
|
|
3406
3666
|
if BASE.metadata.schema:
|
|
3407
3667
|
schema = BASE.metadata.schema + '.'
|
|
3408
3668
|
session.execute(text('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))) # NOQA: E501
|
|
3409
|
-
elif session.bind.dialect.name == 'mysql':
|
|
3669
|
+
elif session.bind.dialect.name == 'mysql': # type: ignore
|
|
3410
3670
|
subquery1 = select(
|
|
3411
3671
|
func.max(models.UpdatedCollectionReplica.id).label('max_id')
|
|
3412
3672
|
).group_by(
|
|
@@ -3464,7 +3724,11 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
|
|
|
3464
3724
|
|
|
3465
3725
|
|
|
3466
3726
|
@transactional_session
|
|
3467
|
-
def update_collection_replica(
|
|
3727
|
+
def update_collection_replica(
|
|
3728
|
+
update_request: dict[str, Any],
|
|
3729
|
+
*,
|
|
3730
|
+
session: "Session"
|
|
3731
|
+
) -> None:
|
|
3468
3732
|
"""
|
|
3469
3733
|
Update a collection replica.
|
|
3470
3734
|
:param update_request: update request from the upated_col_rep table.
|
|
@@ -3620,7 +3884,13 @@ def update_collection_replica(update_request, *, session: "Session"):
|
|
|
3620
3884
|
|
|
3621
3885
|
|
|
3622
3886
|
@read_session
|
|
3623
|
-
def get_bad_pfns(
|
|
3887
|
+
def get_bad_pfns(
|
|
3888
|
+
limit: int = 10000,
|
|
3889
|
+
thread: Optional[int] = None,
|
|
3890
|
+
total_threads: Optional[int] = None,
|
|
3891
|
+
*,
|
|
3892
|
+
session: "Session"
|
|
3893
|
+
) -> list[dict[str, Any]]:
|
|
3624
3894
|
"""
|
|
3625
3895
|
Returns a list of bad PFNs
|
|
3626
3896
|
|
|
@@ -3653,7 +3923,15 @@ def get_bad_pfns(limit=10000, thread=None, total_threads=None, *, session: "Sess
|
|
|
3653
3923
|
|
|
3654
3924
|
|
|
3655
3925
|
@transactional_session
|
|
3656
|
-
def bulk_add_bad_replicas(
|
|
3926
|
+
def bulk_add_bad_replicas(
|
|
3927
|
+
replicas: "Iterable[dict[str, Any]]",
|
|
3928
|
+
account: InternalAccount,
|
|
3929
|
+
state: BadFilesStatus = BadFilesStatus.TEMPORARY_UNAVAILABLE,
|
|
3930
|
+
reason: Optional[str] = None,
|
|
3931
|
+
expires_at: Optional[datetime] = None,
|
|
3932
|
+
*,
|
|
3933
|
+
session: "Session"
|
|
3934
|
+
) -> bool:
|
|
3657
3935
|
"""
|
|
3658
3936
|
Bulk add new bad replicas.
|
|
3659
3937
|
|
|
@@ -3711,7 +3989,11 @@ def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAV
|
|
|
3711
3989
|
|
|
3712
3990
|
|
|
3713
3991
|
@transactional_session
|
|
3714
|
-
def bulk_delete_bad_pfns(
|
|
3992
|
+
def bulk_delete_bad_pfns(
|
|
3993
|
+
pfns: "Iterable[str]",
|
|
3994
|
+
*,
|
|
3995
|
+
session: "Session"
|
|
3996
|
+
) -> Literal[True]:
|
|
3715
3997
|
"""
|
|
3716
3998
|
Bulk delete bad PFNs.
|
|
3717
3999
|
|
|
@@ -3738,7 +4020,11 @@ def bulk_delete_bad_pfns(pfns, *, session: "Session"):
|
|
|
3738
4020
|
|
|
3739
4021
|
|
|
3740
4022
|
@transactional_session
|
|
3741
|
-
def bulk_delete_bad_replicas(
|
|
4023
|
+
def bulk_delete_bad_replicas(
|
|
4024
|
+
bad_replicas: "Iterable[dict[str, Any]]",
|
|
4025
|
+
*,
|
|
4026
|
+
session: "Session"
|
|
4027
|
+
) -> Literal[True]:
|
|
3742
4028
|
"""
|
|
3743
4029
|
Bulk delete bad replica.
|
|
3744
4030
|
|
|
@@ -3767,7 +4053,15 @@ def bulk_delete_bad_replicas(bad_replicas, *, session: "Session"):
|
|
|
3767
4053
|
|
|
3768
4054
|
|
|
3769
4055
|
@transactional_session
|
|
3770
|
-
def add_bad_pfns(
|
|
4056
|
+
def add_bad_pfns(
|
|
4057
|
+
pfns: "Iterable[str]",
|
|
4058
|
+
account: InternalAccount,
|
|
4059
|
+
state: BadFilesStatus,
|
|
4060
|
+
reason: Optional[str] = None,
|
|
4061
|
+
expires_at: Optional[datetime] = None,
|
|
4062
|
+
*,
|
|
4063
|
+
session: "Session"
|
|
4064
|
+
) -> Literal[True]:
|
|
3771
4065
|
"""
|
|
3772
4066
|
Add bad PFNs.
|
|
3773
4067
|
|
|
@@ -3811,7 +4105,13 @@ def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, *, session:
|
|
|
3811
4105
|
|
|
3812
4106
|
|
|
3813
4107
|
@read_session
|
|
3814
|
-
def list_expired_temporary_unavailable_replicas(
|
|
4108
|
+
def list_expired_temporary_unavailable_replicas(
|
|
4109
|
+
total_workers: int,
|
|
4110
|
+
worker_number: int,
|
|
4111
|
+
limit: int = 10000,
|
|
4112
|
+
*,
|
|
4113
|
+
session: "Session"
|
|
4114
|
+
) -> "Sequence[Row]":
|
|
3815
4115
|
"""
|
|
3816
4116
|
List the expired temporary unavailable replicas
|
|
3817
4117
|
|
|
@@ -3843,7 +4143,12 @@ def list_expired_temporary_unavailable_replicas(total_workers, worker_number, li
|
|
|
3843
4143
|
|
|
3844
4144
|
|
|
3845
4145
|
@read_session
|
|
3846
|
-
def get_replicas_state(
|
|
4146
|
+
def get_replicas_state(
|
|
4147
|
+
scope: Optional[InternalScope] = None,
|
|
4148
|
+
name: Optional[str] = None,
|
|
4149
|
+
*,
|
|
4150
|
+
session: "Session"
|
|
4151
|
+
) -> dict[ReplicaState, list[str]]:
|
|
3847
4152
|
"""
|
|
3848
4153
|
Method used by the necromancer to get all the replicas of a DIDs
|
|
3849
4154
|
:param scope: The scope of the file.
|
|
@@ -3873,16 +4178,16 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
|
|
|
3873
4178
|
def get_suspicious_files(
|
|
3874
4179
|
rse_expression: str,
|
|
3875
4180
|
available_elsewhere: int,
|
|
3876
|
-
filter_:
|
|
4181
|
+
filter_: Optional[dict[str, Any]] = None,
|
|
3877
4182
|
logger: "LoggerFunction" = logging.log,
|
|
3878
|
-
younger_than:
|
|
4183
|
+
younger_than: Optional[datetime] = None,
|
|
3879
4184
|
nattempts: int = 0,
|
|
3880
4185
|
nattempts_exact: bool = False,
|
|
3881
4186
|
*,
|
|
3882
4187
|
session: "Session",
|
|
3883
|
-
exclude_states:
|
|
4188
|
+
exclude_states: Optional["Iterable[str]"] = None,
|
|
3884
4189
|
is_suspicious: bool = False
|
|
3885
|
-
) ->
|
|
4190
|
+
) -> list[dict[str, Any]]:
|
|
3886
4191
|
"""
|
|
3887
4192
|
Gets a list of replicas from bad_replicas table which are: declared more than <nattempts> times since <younger_than> date,
|
|
3888
4193
|
present on the RSE specified by the <rse_expression> and do not have a state in <exclude_states> list.
|
|
@@ -4025,7 +4330,15 @@ def get_suspicious_files(
|
|
|
4025
4330
|
|
|
4026
4331
|
|
|
4027
4332
|
@read_session
|
|
4028
|
-
def get_suspicious_reason(
|
|
4333
|
+
def get_suspicious_reason(
|
|
4334
|
+
rse_id: str,
|
|
4335
|
+
scope: InternalScope,
|
|
4336
|
+
name: str,
|
|
4337
|
+
nattempts: int = 0,
|
|
4338
|
+
logger: "LoggerFunction" = logging.log,
|
|
4339
|
+
*,
|
|
4340
|
+
session: "Session"
|
|
4341
|
+
) -> list[dict[str, Any]]:
|
|
4029
4342
|
"""
|
|
4030
4343
|
Returns the error message(s) which lead to the replica(s) being declared suspicious.
|
|
4031
4344
|
|
|
@@ -4085,7 +4398,14 @@ def get_suspicious_reason(rse_id, scope, name, nattempts=0, logger=logging.log,
|
|
|
4085
4398
|
|
|
4086
4399
|
|
|
4087
4400
|
@transactional_session
|
|
4088
|
-
def set_tombstone(
|
|
4401
|
+
def set_tombstone(
|
|
4402
|
+
rse_id: str,
|
|
4403
|
+
scope: InternalScope,
|
|
4404
|
+
name: str,
|
|
4405
|
+
tombstone: datetime = OBSOLETE,
|
|
4406
|
+
*,
|
|
4407
|
+
session: "Session"
|
|
4408
|
+
) -> None:
|
|
4089
4409
|
"""
|
|
4090
4410
|
Sets a tombstone on a replica.
|
|
4091
4411
|
|
|
@@ -4127,7 +4447,12 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, *, session: "Session"
|
|
|
4127
4447
|
|
|
4128
4448
|
|
|
4129
4449
|
@read_session
|
|
4130
|
-
def
|
|
4450
|
+
def get_rse_coverage_of_dataset(
|
|
4451
|
+
scope: "InternalScope",
|
|
4452
|
+
name: str,
|
|
4453
|
+
*,
|
|
4454
|
+
session: "Session"
|
|
4455
|
+
) -> dict[str, int]:
|
|
4131
4456
|
"""
|
|
4132
4457
|
Get total bytes present on RSEs
|
|
4133
4458
|
|