rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/{daemons/c3po/collectors → cli}/__init__.py +1 -0
- rucio/cli/account.py +216 -0
- rucio-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/dumper/__init__.py +45 -38
- rucio/common/dumper/consistency.py +75 -30
- rucio/common/dumper/data_models.py +63 -19
- rucio/common/dumper/path_parsing.py +19 -8
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/stomp_utils.py +383 -119
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/core/account_limit.py +14 -12
- rucio/core/authentication.py +2 -2
- rucio/core/config.py +23 -42
- rucio/core/credential.py +14 -15
- rucio/core/did.py +5 -1
- rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
- rucio/core/did_meta_plugins/filter_engine.py +62 -3
- rucio/core/did_meta_plugins/json_meta.py +2 -2
- rucio/core/did_meta_plugins/mongo_meta.py +43 -30
- rucio/core/did_meta_plugins/postgres_meta.py +75 -39
- rucio/core/identity.py +6 -5
- rucio/core/importer.py +4 -3
- rucio/core/lifetime_exception.py +2 -2
- rucio/core/lock.py +8 -7
- rucio/core/message.py +6 -0
- rucio/core/monitor.py +30 -29
- rucio/core/naming_convention.py +2 -2
- rucio/core/nongrid_trace.py +2 -2
- rucio/core/oidc.py +11 -9
- rucio/core/permission/__init__.py +79 -37
- rucio/core/permission/generic.py +1 -7
- rucio/core/permission/generic_multi_vo.py +1 -7
- rucio/core/quarantined_replica.py +4 -3
- rucio/core/replica.py +464 -139
- rucio/core/replica_sorter.py +55 -59
- rucio/core/request.py +34 -32
- rucio/core/rse.py +301 -97
- rucio/core/rse_counter.py +1 -2
- rucio/core/rse_expression_parser.py +7 -7
- rucio/core/rse_selector.py +9 -7
- rucio/core/rule.py +41 -40
- rucio/core/rule_grouping.py +42 -40
- rucio/core/scope.py +5 -4
- rucio/core/subscription.py +26 -28
- rucio/core/topology.py +11 -11
- rucio/core/trace.py +2 -2
- rucio/core/transfer.py +29 -15
- rucio/core/volatile_replica.py +4 -3
- rucio/daemons/atropos/atropos.py +1 -1
- rucio/daemons/auditor/__init__.py +2 -2
- rucio/daemons/auditor/srmdumps.py +6 -6
- rucio/daemons/automatix/automatix.py +32 -21
- rucio/daemons/badreplicas/necromancer.py +2 -2
- rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
- rucio/daemons/bb8/t2_background_rebalance.py +1 -1
- rucio/daemons/cache/consumer.py +26 -90
- rucio/daemons/common.py +15 -25
- rucio/daemons/conveyor/finisher.py +2 -2
- rucio/daemons/conveyor/poller.py +18 -28
- rucio/daemons/conveyor/receiver.py +53 -123
- rucio/daemons/conveyor/stager.py +1 -0
- rucio/daemons/conveyor/submitter.py +3 -3
- rucio/daemons/hermes/hermes.py +129 -369
- rucio/daemons/judge/evaluator.py +2 -2
- rucio/daemons/oauthmanager/oauthmanager.py +3 -3
- rucio/daemons/reaper/dark_reaper.py +7 -3
- rucio/daemons/reaper/reaper.py +12 -16
- rucio/daemons/rsedecommissioner/config.py +1 -1
- rucio/daemons/rsedecommissioner/profiles/generic.py +5 -4
- rucio/daemons/rsedecommissioner/profiles/types.py +7 -6
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +1 -1
- rucio/daemons/storage/consistency/actions.py +8 -6
- rucio/daemons/tracer/kronos.py +117 -142
- rucio/db/sqla/constants.py +5 -0
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +4 -4
- rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
- rucio/db/sqla/models.py +157 -154
- rucio/db/sqla/session.py +58 -27
- rucio/db/sqla/types.py +2 -2
- rucio/db/sqla/util.py +2 -2
- rucio/gateway/account.py +18 -12
- rucio/gateway/account_limit.py +137 -60
- rucio/gateway/authentication.py +18 -12
- rucio/gateway/config.py +30 -20
- rucio/gateway/credential.py +9 -10
- rucio/gateway/did.py +70 -53
- rucio/gateway/dirac.py +6 -4
- rucio/gateway/exporter.py +3 -2
- rucio/gateway/heartbeat.py +6 -4
- rucio/gateway/identity.py +36 -51
- rucio/gateway/importer.py +3 -2
- rucio/gateway/lifetime_exception.py +3 -2
- rucio/gateway/meta_conventions.py +17 -6
- rucio/gateway/permission.py +4 -1
- rucio/gateway/quarantined_replica.py +3 -2
- rucio/gateway/replica.py +31 -22
- rucio/gateway/request.py +27 -18
- rucio/gateway/rse.py +69 -37
- rucio/gateway/rule.py +46 -26
- rucio/gateway/scope.py +3 -2
- rucio/gateway/subscription.py +14 -11
- rucio/gateway/vo.py +12 -8
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/tests/common.py +23 -13
- rucio/tests/common_server.py +26 -9
- rucio/transfertool/bittorrent.py +15 -14
- rucio/transfertool/bittorrent_driver.py +5 -7
- rucio/transfertool/bittorrent_driver_qbittorrent.py +9 -8
- rucio/transfertool/fts3.py +20 -16
- rucio/transfertool/mock.py +2 -3
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- rucio/web/rest/flaskapi/v1/accounts.py +17 -3
- rucio/web/rest/flaskapi/v1/auth.py +5 -5
- rucio/web/rest/flaskapi/v1/credentials.py +3 -2
- rucio/web/rest/flaskapi/v1/dids.py +21 -15
- rucio/web/rest/flaskapi/v1/identities.py +33 -9
- rucio/web/rest/flaskapi/v1/redirect.py +5 -4
- rucio/web/rest/flaskapi/v1/replicas.py +12 -8
- rucio/web/rest/flaskapi/v1/rses.py +15 -4
- rucio/web/rest/flaskapi/v1/traces.py +56 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.template +3 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio_multi_vo.cfg.template +1 -18
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/requirements.server.txt +97 -68
- rucio-37.0.0rc2.data/scripts/rucio +133 -0
- rucio-37.0.0rc2.data/scripts/rucio-admin +97 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-atropos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-auditor +2 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-automatix +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-client +17 -10
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-receiver +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-kronos +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos-temporary-expiration +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-necromancer +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-reaper +6 -6
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-transmogrifier +2 -2
- rucio-37.0.0rc2.dist-info/METADATA +92 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/RECORD +239 -245
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/core/permission/atlas.py +0 -1348
- rucio/core/permission/belleii.py +0 -1077
- rucio/core/permission/escape.py +0 -1078
- rucio/daemons/c3po/algorithms/__init__.py +0 -13
- rucio/daemons/c3po/algorithms/simple.py +0 -134
- rucio/daemons/c3po/algorithms/t2_free_space.py +0 -128
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +0 -130
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +0 -294
- rucio/daemons/c3po/c3po.py +0 -371
- rucio/daemons/c3po/collectors/agis.py +0 -108
- rucio/daemons/c3po/collectors/free_space.py +0 -81
- rucio/daemons/c3po/collectors/jedi_did.py +0 -57
- rucio/daemons/c3po/collectors/mock_did.py +0 -51
- rucio/daemons/c3po/collectors/network_metrics.py +0 -71
- rucio/daemons/c3po/collectors/workload.py +0 -112
- rucio/daemons/c3po/utils/__init__.py +0 -13
- rucio/daemons/c3po/utils/dataset_cache.py +0 -50
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +0 -56
- rucio/daemons/c3po/utils/expiring_list.py +0 -62
- rucio/daemons/c3po/utils/popularity.py +0 -85
- rucio/daemons/c3po/utils/timeseries.py +0 -89
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio-35.7.0.data/scripts/rucio-c3po +0 -85
- rucio-35.7.0.dist-info/METADATA +0 -72
- /rucio/{daemons/c3po → cli/bin_legacy}/__init__.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-account +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-bb8 +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dumper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-follower +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-hermes +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-injector +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-undertaker +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/WHEEL +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/LICENSE +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/top_level.txt +0 -0
rucio/core/rse.py
CHANGED
|
@@ -13,24 +13,24 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import json
|
|
16
|
-
from collections.abc import Iterable, Iterator
|
|
17
16
|
from datetime import datetime
|
|
18
17
|
from io import StringIO
|
|
19
18
|
from re import match
|
|
20
|
-
from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union
|
|
19
|
+
from typing import TYPE_CHECKING, Any, Generic, Literal, Optional, TypeVar, Union, overload
|
|
21
20
|
|
|
22
21
|
import sqlalchemy
|
|
23
|
-
from dogpile.cache.api import
|
|
22
|
+
from dogpile.cache.api import NoValue
|
|
24
23
|
from sqlalchemy.exc import DatabaseError, IntegrityError, OperationalError
|
|
25
24
|
from sqlalchemy.orm import aliased
|
|
26
25
|
from sqlalchemy.orm.exc import FlushError
|
|
27
|
-
from sqlalchemy.sql.expression import and_, delete, desc, false, func, or_, select, true
|
|
26
|
+
from sqlalchemy.sql.expression import Executable, and_, delete, desc, false, func, or_, select, true
|
|
28
27
|
|
|
29
28
|
from rucio.common import exception, types, utils
|
|
30
|
-
from rucio.common.cache import
|
|
29
|
+
from rucio.common.cache import MemcacheRegion
|
|
30
|
+
from rucio.common.checksum import CHECKSUM_KEY, GLOBALLY_SUPPORTED_CHECKSUMS
|
|
31
31
|
from rucio.common.config import get_lfn2pfn_algorithm_default
|
|
32
|
-
from rucio.common.constants import
|
|
33
|
-
from rucio.common.utils import
|
|
32
|
+
from rucio.common.constants import RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS, RSE_ATTRS_BOOL, RSE_ATTRS_STR, SUPPORTED_SIGN_URL_SERVICES_LITERAL, RseAttr
|
|
33
|
+
from rucio.common.utils import Availability
|
|
34
34
|
from rucio.core.rse_counter import add_counter, get_counter
|
|
35
35
|
from rucio.db.sqla import models
|
|
36
36
|
from rucio.db.sqla.constants import ReplicaState, RSEType
|
|
@@ -38,19 +38,33 @@ from rucio.db.sqla.session import read_session, stream_session, transactional_se
|
|
|
38
38
|
from rucio.db.sqla.util import temp_table_mngr
|
|
39
39
|
|
|
40
40
|
if TYPE_CHECKING:
|
|
41
|
+
from collections.abc import Iterable, Iterator
|
|
42
|
+
|
|
41
43
|
from sqlalchemy.orm import Session
|
|
44
|
+
from sqlalchemy.sql._typing import _ColumnsClauseArgument
|
|
45
|
+
from typing_extensions import Self
|
|
42
46
|
|
|
43
47
|
T = TypeVar('T', bound="RseData")
|
|
44
48
|
|
|
45
49
|
RSE_SETTINGS = ["continent", "city", "region_code", "country_name", "time_zone", "ISP", "ASN"]
|
|
46
|
-
REGION =
|
|
50
|
+
REGION = MemcacheRegion(expiration_time=900)
|
|
47
51
|
|
|
48
52
|
|
|
49
53
|
class RseData:
|
|
50
54
|
"""
|
|
51
55
|
Helper data class storing rse data grouped in one place.
|
|
52
56
|
"""
|
|
53
|
-
def __init__(
|
|
57
|
+
def __init__(
|
|
58
|
+
self,
|
|
59
|
+
id_: str,
|
|
60
|
+
name: Optional[str] = None,
|
|
61
|
+
columns: Optional[dict[str, Any]] = None,
|
|
62
|
+
attributes: Optional[dict[str, Any]] = None,
|
|
63
|
+
info: Optional[types.RSESettingsDict] = None,
|
|
64
|
+
usage: Optional[list[dict[str, Any]]] = None,
|
|
65
|
+
limits: Optional[dict[str, Any]] = None,
|
|
66
|
+
transfer_limits: Optional[dict[str, Any]] = None
|
|
67
|
+
):
|
|
54
68
|
self.id = id_
|
|
55
69
|
self._name = name
|
|
56
70
|
self._columns = columns
|
|
@@ -60,78 +74,80 @@ class RseData:
|
|
|
60
74
|
self._limits = limits
|
|
61
75
|
self._transfer_limits = transfer_limits
|
|
62
76
|
|
|
77
|
+
def _get_loaded_attribute(self, attribute_name: str) -> Any:
|
|
78
|
+
attribute = getattr(self, f'_{attribute_name}', None)
|
|
79
|
+
if attribute is None:
|
|
80
|
+
raise ValueError(f'{attribute_name} not loaded for rse {self}')
|
|
81
|
+
return attribute
|
|
82
|
+
|
|
63
83
|
@property
|
|
64
84
|
def name(self) -> str:
|
|
65
|
-
|
|
66
|
-
raise ValueError(f'name not loaded for rse {self}')
|
|
67
|
-
return self._name
|
|
85
|
+
return self._get_loaded_attribute('name')
|
|
68
86
|
|
|
69
87
|
@name.setter
|
|
70
|
-
def name(self, name):
|
|
88
|
+
def name(self, name: str) -> None:
|
|
71
89
|
self._name = name
|
|
72
90
|
|
|
73
91
|
@property
|
|
74
92
|
def columns(self) -> dict[str, Any]:
|
|
75
|
-
|
|
76
|
-
raise ValueError(f'columns not loaded for rse {self}')
|
|
77
|
-
return self._columns
|
|
93
|
+
return self._get_loaded_attribute('columns')
|
|
78
94
|
|
|
79
95
|
@property
|
|
80
96
|
def attributes(self) -> dict[str, Any]:
|
|
81
|
-
|
|
82
|
-
raise ValueError(f'attributes not loaded for rse {self}')
|
|
83
|
-
return self._attributes
|
|
97
|
+
return self._get_loaded_attribute('attributes')
|
|
84
98
|
|
|
85
99
|
@property
|
|
86
100
|
def info(self) -> types.RSESettingsDict:
|
|
87
|
-
|
|
88
|
-
raise ValueError(f'info not loaded for rse {self}')
|
|
89
|
-
return self._info
|
|
101
|
+
return self._get_loaded_attribute('info')
|
|
90
102
|
|
|
91
103
|
@property
|
|
92
104
|
def usage(self) -> list[dict[str, Any]]:
|
|
93
|
-
|
|
94
|
-
raise ValueError(f'usage not loaded for rse {self}')
|
|
95
|
-
return self._usage
|
|
105
|
+
return self._get_loaded_attribute('usage')
|
|
96
106
|
|
|
97
107
|
@property
|
|
98
108
|
def limits(self) -> dict[str, Any]:
|
|
99
|
-
|
|
100
|
-
raise ValueError(f'limits not loaded for rse {self}')
|
|
101
|
-
return self._limits
|
|
109
|
+
return self._get_loaded_attribute('limits')
|
|
102
110
|
|
|
103
111
|
@property
|
|
104
|
-
def transfer_limits(self):
|
|
105
|
-
|
|
106
|
-
raise ValueError(f'transfer_limits not loaded for rse {self}')
|
|
107
|
-
return self._transfer_limits
|
|
112
|
+
def transfer_limits(self) -> dict[str, Any]:
|
|
113
|
+
return self._get_loaded_attribute('transfer_limits')
|
|
108
114
|
|
|
109
|
-
def __hash__(self):
|
|
115
|
+
def __hash__(self) -> int:
|
|
110
116
|
return hash(self.id)
|
|
111
117
|
|
|
112
|
-
def __repr__(self):
|
|
118
|
+
def __repr__(self) -> str:
|
|
113
119
|
if self._name is not None:
|
|
114
120
|
return self._name
|
|
115
121
|
return self.id
|
|
116
122
|
|
|
117
|
-
def __eq__(self, other):
|
|
123
|
+
def __eq__(self, other) -> bool:
|
|
118
124
|
if other is None:
|
|
119
125
|
return False
|
|
120
126
|
return self.id == other.id
|
|
121
127
|
|
|
122
|
-
def is_tape(self):
|
|
128
|
+
def is_tape(self) -> bool:
|
|
123
129
|
if self.info['rse_type'] == RSEType.TAPE or self.info['rse_type'] == 'TAPE' or self.attributes.get(RseAttr.STAGING_REQUIRED, False):
|
|
124
130
|
return True
|
|
125
131
|
return False
|
|
126
132
|
|
|
127
|
-
def is_tape_or_staging_required(self):
|
|
133
|
+
def is_tape_or_staging_required(self) -> bool:
|
|
128
134
|
if self.is_tape() or self.attributes.get(RseAttr.STAGING_REQUIRED, False):
|
|
129
135
|
return True
|
|
130
136
|
return False
|
|
131
137
|
|
|
132
138
|
@read_session
|
|
133
|
-
def ensure_loaded(
|
|
134
|
-
|
|
139
|
+
def ensure_loaded(
|
|
140
|
+
self,
|
|
141
|
+
load_name: bool = False,
|
|
142
|
+
load_columns: bool = False,
|
|
143
|
+
load_attributes: bool = False,
|
|
144
|
+
load_info: bool = False,
|
|
145
|
+
load_usage: bool = False,
|
|
146
|
+
load_limits: bool = False,
|
|
147
|
+
load_transfer_limits: bool = False,
|
|
148
|
+
*,
|
|
149
|
+
session: "Session"
|
|
150
|
+
) -> "Self":
|
|
135
151
|
if self._columns is None and load_columns:
|
|
136
152
|
self._columns = get_rse(rse_id=self.id, session=session)
|
|
137
153
|
self._name = self._columns['rse']
|
|
@@ -163,7 +179,7 @@ class RseData:
|
|
|
163
179
|
include_deleted: bool = False,
|
|
164
180
|
*,
|
|
165
181
|
session: "Session"
|
|
166
|
-
):
|
|
182
|
+
) -> None:
|
|
167
183
|
"""
|
|
168
184
|
Given a dict of RseData objects indexed by rse_id, ensure that the desired fields are initialised
|
|
169
185
|
in all objects from the input.
|
|
@@ -300,22 +316,32 @@ class RseCollection(Generic[T]):
|
|
|
300
316
|
Container which keeps track of information loaded from the database for a group of RSEs.
|
|
301
317
|
"""
|
|
302
318
|
|
|
303
|
-
def __init__(self, rse_ids: Optional[Iterable[str]] = None, rse_data_cls: type[T] = RseData):
|
|
319
|
+
def __init__(self, rse_ids: Optional['Iterable[str]'] = None, rse_data_cls: type[T] = RseData):
|
|
304
320
|
self._rse_data_cls = rse_data_cls
|
|
305
321
|
self.rse_id_to_data_map: dict[str, T] = {}
|
|
306
322
|
if rse_ids is not None:
|
|
307
323
|
for rse_id in rse_ids:
|
|
308
324
|
self.rse_id_to_data_map[rse_id] = self._rse_data_cls(rse_id)
|
|
309
325
|
|
|
310
|
-
def __getitem__(
|
|
326
|
+
def __getitem__(
|
|
327
|
+
self,
|
|
328
|
+
item: str
|
|
329
|
+
) -> "T":
|
|
311
330
|
return self.get_or_create(item)
|
|
312
331
|
|
|
313
|
-
def __setitem__(
|
|
332
|
+
def __setitem__(
|
|
333
|
+
self,
|
|
334
|
+
key: str,
|
|
335
|
+
value: "T"
|
|
336
|
+
) -> None:
|
|
314
337
|
rse_id = key
|
|
315
338
|
rse_data = value
|
|
316
339
|
self.rse_id_to_data_map[rse_id] = rse_data
|
|
317
340
|
|
|
318
|
-
def __contains__(
|
|
341
|
+
def __contains__(
|
|
342
|
+
self,
|
|
343
|
+
item: Any
|
|
344
|
+
) -> bool:
|
|
319
345
|
if isinstance(item, RseData):
|
|
320
346
|
return item.id in self.rse_id_to_data_map
|
|
321
347
|
if isinstance(item, str):
|
|
@@ -334,7 +360,7 @@ class RseCollection(Generic[T]):
|
|
|
334
360
|
@transactional_session
|
|
335
361
|
def ensure_loaded(
|
|
336
362
|
self,
|
|
337
|
-
rse_ids:
|
|
363
|
+
rse_ids: Optional["Iterable[str]"] = None,
|
|
338
364
|
load_name: bool = False,
|
|
339
365
|
load_columns: bool = False,
|
|
340
366
|
load_attributes: bool = False,
|
|
@@ -344,7 +370,7 @@ class RseCollection(Generic[T]):
|
|
|
344
370
|
include_deleted: bool = False,
|
|
345
371
|
*,
|
|
346
372
|
session: "Session",
|
|
347
|
-
):
|
|
373
|
+
) -> None:
|
|
348
374
|
RseData.bulk_load(
|
|
349
375
|
rse_id_to_data={rse_id: self.get_or_create(rse_id) for rse_id in rse_ids} if rse_ids else self.rse_id_to_data_map,
|
|
350
376
|
load_name=load_name,
|
|
@@ -359,7 +385,11 @@ class RseCollection(Generic[T]):
|
|
|
359
385
|
|
|
360
386
|
|
|
361
387
|
@stream_session
|
|
362
|
-
def _group_query_result_by_rse_id(
|
|
388
|
+
def _group_query_result_by_rse_id(
|
|
389
|
+
stmt: "Executable",
|
|
390
|
+
*,
|
|
391
|
+
session: "Session"
|
|
392
|
+
) -> 'Iterator[tuple[str, list[Any]]]':
|
|
363
393
|
"""
|
|
364
394
|
Given a sqlalchemy query statement which fetches rows of two elements: (rse_id, object) ordered by rse_id.
|
|
365
395
|
Will execute the query and return objects grouped by rse_id: (rse_id, [object1, object2])
|
|
@@ -383,9 +413,28 @@ def _group_query_result_by_rse_id(stmt, *, session: "Session") -> Iterator[tuple
|
|
|
383
413
|
|
|
384
414
|
|
|
385
415
|
@transactional_session
|
|
386
|
-
def add_rse(
|
|
387
|
-
|
|
388
|
-
|
|
416
|
+
def add_rse(
|
|
417
|
+
rse: str,
|
|
418
|
+
vo: str = 'def',
|
|
419
|
+
deterministic: bool = True,
|
|
420
|
+
volatile: bool = False,
|
|
421
|
+
city: Optional[str] = None,
|
|
422
|
+
region_code: Optional[str] = None,
|
|
423
|
+
country_name: Optional[str] = None,
|
|
424
|
+
continent: Optional[str] = None,
|
|
425
|
+
time_zone: Optional[str] = None,
|
|
426
|
+
ISP: Optional[str] = None, # noqa: N803
|
|
427
|
+
staging_area: bool = False,
|
|
428
|
+
rse_type: RSEType = RSEType.DISK,
|
|
429
|
+
longitude: Optional[float] = None,
|
|
430
|
+
latitude: Optional[float] = None,
|
|
431
|
+
ASN: Optional[str] = None, # noqa: N803
|
|
432
|
+
availability_read: Optional[bool] = None,
|
|
433
|
+
availability_write: Optional[bool] = None,
|
|
434
|
+
availability_delete: Optional[bool] = None,
|
|
435
|
+
*,
|
|
436
|
+
session: "Session"
|
|
437
|
+
) -> str:
|
|
389
438
|
"""
|
|
390
439
|
Add a rse with the given location name.
|
|
391
440
|
|
|
@@ -446,7 +495,13 @@ def add_rse(rse, vo='def', deterministic=True, volatile=False, city=None, region
|
|
|
446
495
|
|
|
447
496
|
|
|
448
497
|
@read_session
|
|
449
|
-
def rse_exists(
|
|
498
|
+
def rse_exists(
|
|
499
|
+
rse: str,
|
|
500
|
+
vo: str = 'def',
|
|
501
|
+
include_deleted: bool = False,
|
|
502
|
+
*,
|
|
503
|
+
session: "Session"
|
|
504
|
+
) -> bool:
|
|
450
505
|
"""
|
|
451
506
|
Checks to see if RSE exists.
|
|
452
507
|
|
|
@@ -468,7 +523,11 @@ def rse_exists(rse, vo='def', include_deleted=False, *, session: "Session"):
|
|
|
468
523
|
|
|
469
524
|
|
|
470
525
|
@transactional_session
|
|
471
|
-
def del_rse(
|
|
526
|
+
def del_rse(
|
|
527
|
+
rse_id: str,
|
|
528
|
+
*,
|
|
529
|
+
session: "Session"
|
|
530
|
+
) -> None:
|
|
472
531
|
"""
|
|
473
532
|
Disable a rse with the given rse id.
|
|
474
533
|
|
|
@@ -497,7 +556,11 @@ def del_rse(rse_id, *, session: "Session"):
|
|
|
497
556
|
|
|
498
557
|
|
|
499
558
|
@transactional_session
|
|
500
|
-
def restore_rse(
|
|
559
|
+
def restore_rse(
|
|
560
|
+
rse_id: str,
|
|
561
|
+
*,
|
|
562
|
+
session: "Session"
|
|
563
|
+
) -> None:
|
|
501
564
|
"""
|
|
502
565
|
Restore a rse with the given rse id.
|
|
503
566
|
|
|
@@ -523,7 +586,11 @@ def restore_rse(rse_id, *, session: "Session"):
|
|
|
523
586
|
|
|
524
587
|
|
|
525
588
|
@read_session
|
|
526
|
-
def rse_is_empty(
|
|
589
|
+
def rse_is_empty(
|
|
590
|
+
rse_id: str,
|
|
591
|
+
*,
|
|
592
|
+
session: "Session"
|
|
593
|
+
) -> bool:
|
|
527
594
|
"""
|
|
528
595
|
Check if a RSE is empty.
|
|
529
596
|
|
|
@@ -566,7 +633,11 @@ def _format_get_rse(
|
|
|
566
633
|
|
|
567
634
|
|
|
568
635
|
@read_session
|
|
569
|
-
def get_rse(
|
|
636
|
+
def get_rse(
|
|
637
|
+
rse_id: str,
|
|
638
|
+
*,
|
|
639
|
+
session: "Session"
|
|
640
|
+
) -> dict[str, Any]:
|
|
570
641
|
"""
|
|
571
642
|
Get a RSE or raise if it does not exist.
|
|
572
643
|
|
|
@@ -589,7 +660,13 @@ def get_rse(rse_id, *, session: "Session"):
|
|
|
589
660
|
|
|
590
661
|
|
|
591
662
|
@read_session
|
|
592
|
-
def get_rse_id(
|
|
663
|
+
def get_rse_id(
|
|
664
|
+
rse: str,
|
|
665
|
+
vo: str = 'def',
|
|
666
|
+
include_deleted: bool = True,
|
|
667
|
+
*,
|
|
668
|
+
session: "Session"
|
|
669
|
+
) -> str:
|
|
593
670
|
"""
|
|
594
671
|
Get a RSE ID or raise if it does not exist.
|
|
595
672
|
|
|
@@ -608,7 +685,7 @@ def get_rse_id(rse, vo='def', include_deleted=True, *, session: "Session"):
|
|
|
608
685
|
else:
|
|
609
686
|
cache_key = 'rse-id_{}'.format(rse).replace(' ', '.')
|
|
610
687
|
result = REGION.get(cache_key)
|
|
611
|
-
if result
|
|
688
|
+
if not isinstance(result, NoValue):
|
|
612
689
|
return result
|
|
613
690
|
|
|
614
691
|
try:
|
|
@@ -630,11 +707,18 @@ def get_rse_id(rse, vo='def', include_deleted=True, *, session: "Session"):
|
|
|
630
707
|
|
|
631
708
|
|
|
632
709
|
@read_session
|
|
633
|
-
def _get_rse_db_column(
|
|
710
|
+
def _get_rse_db_column(
|
|
711
|
+
rse_id: str,
|
|
712
|
+
column: "_ColumnsClauseArgument",
|
|
713
|
+
cache_prefix: str,
|
|
714
|
+
include_deleted: bool = True,
|
|
715
|
+
*,
|
|
716
|
+
session: "Session"
|
|
717
|
+
) -> Any:
|
|
634
718
|
if include_deleted:
|
|
635
719
|
cache_key = '{}_{}'.format(cache_prefix, rse_id)
|
|
636
720
|
result = REGION.get(cache_key)
|
|
637
|
-
if result
|
|
721
|
+
if not isinstance(result, NoValue):
|
|
638
722
|
return result
|
|
639
723
|
|
|
640
724
|
try:
|
|
@@ -655,7 +739,12 @@ def _get_rse_db_column(rse_id: str, column, cache_prefix: str, include_deleted:
|
|
|
655
739
|
|
|
656
740
|
|
|
657
741
|
@read_session
|
|
658
|
-
def get_rse_name(
|
|
742
|
+
def get_rse_name(
|
|
743
|
+
rse_id: str,
|
|
744
|
+
include_deleted: bool = True,
|
|
745
|
+
*,
|
|
746
|
+
session: "Session"
|
|
747
|
+
) -> str:
|
|
659
748
|
"""
|
|
660
749
|
Get a RSE name or raise if it does not exist.
|
|
661
750
|
|
|
@@ -677,7 +766,12 @@ def get_rse_name(rse_id: str, include_deleted: bool = True, *, session: "Session
|
|
|
677
766
|
|
|
678
767
|
|
|
679
768
|
@read_session
|
|
680
|
-
def get_rse_vo(
|
|
769
|
+
def get_rse_vo(
|
|
770
|
+
rse_id: str,
|
|
771
|
+
include_deleted: bool = True,
|
|
772
|
+
*,
|
|
773
|
+
session: "Session"
|
|
774
|
+
) -> str:
|
|
681
775
|
"""
|
|
682
776
|
Get the VO for a given RSE id.
|
|
683
777
|
|
|
@@ -750,7 +844,13 @@ def list_rses(filters: Optional[dict[str, Any]] = None, *, session: "Session") -
|
|
|
750
844
|
|
|
751
845
|
|
|
752
846
|
@transactional_session
|
|
753
|
-
def add_rse_attribute(
|
|
847
|
+
def add_rse_attribute(
|
|
848
|
+
rse_id: str,
|
|
849
|
+
key: str,
|
|
850
|
+
value: Union[bool, str],
|
|
851
|
+
*,
|
|
852
|
+
session: "Session"
|
|
853
|
+
) -> bool:
|
|
754
854
|
""" Adds a RSE attribute.
|
|
755
855
|
|
|
756
856
|
:param rse_id: the rse id.
|
|
@@ -772,7 +872,12 @@ def add_rse_attribute(rse_id, key, value, *, session: "Session"):
|
|
|
772
872
|
|
|
773
873
|
|
|
774
874
|
@transactional_session
|
|
775
|
-
def del_rse_attribute(
|
|
875
|
+
def del_rse_attribute(
|
|
876
|
+
rse_id: str,
|
|
877
|
+
key: str,
|
|
878
|
+
*,
|
|
879
|
+
session: "Session"
|
|
880
|
+
) -> bool:
|
|
776
881
|
"""
|
|
777
882
|
Delete a RSE attribute.
|
|
778
883
|
|
|
@@ -797,7 +902,12 @@ def del_rse_attribute(rse_id, key, *, session: "Session"):
|
|
|
797
902
|
|
|
798
903
|
|
|
799
904
|
@read_session
|
|
800
|
-
def list_rse_attributes(
|
|
905
|
+
def list_rse_attributes(
|
|
906
|
+
rse_id: str,
|
|
907
|
+
use_cache: bool = False,
|
|
908
|
+
*,
|
|
909
|
+
session: "Session"
|
|
910
|
+
) -> dict[str, Union[str, bool]]:
|
|
801
911
|
"""
|
|
802
912
|
List RSE attributes for a RSE.
|
|
803
913
|
|
|
@@ -811,7 +921,7 @@ def list_rse_attributes(rse_id: str, use_cache: bool = False, *, session: "Sessi
|
|
|
811
921
|
if use_cache:
|
|
812
922
|
value = REGION.get(cache_key)
|
|
813
923
|
|
|
814
|
-
if value
|
|
924
|
+
if not isinstance(value, NoValue):
|
|
815
925
|
return value
|
|
816
926
|
|
|
817
927
|
rse_attrs = {}
|
|
@@ -832,11 +942,11 @@ def list_rse_attributes(rse_id: str, use_cache: bool = False, *, session: "Sessi
|
|
|
832
942
|
|
|
833
943
|
@stream_session
|
|
834
944
|
def _fetch_many_rses_attributes(
|
|
835
|
-
rse_id_temp_table,
|
|
836
|
-
keys: Optional[Iterable[str]] = None,
|
|
945
|
+
rse_id_temp_table: Any,
|
|
946
|
+
keys: Optional['Iterable[str]'] = None,
|
|
837
947
|
*,
|
|
838
948
|
session: "Session"
|
|
839
|
-
) -> Iterator[tuple[str, dict[str, Any]]]:
|
|
949
|
+
) -> 'Iterator[tuple[str, dict[str, Any]]]':
|
|
840
950
|
"""
|
|
841
951
|
Given a temporary table pre-filled with RSE IDs, fetch the attributes of these RSEs.
|
|
842
952
|
It's possible to only fetch a subset of attributes by setting the `keys` parameter.
|
|
@@ -863,7 +973,12 @@ def _fetch_many_rses_attributes(
|
|
|
863
973
|
|
|
864
974
|
|
|
865
975
|
@read_session
|
|
866
|
-
def has_rse_attribute(
|
|
976
|
+
def has_rse_attribute(
|
|
977
|
+
rse_id: str,
|
|
978
|
+
key: str,
|
|
979
|
+
*,
|
|
980
|
+
session: "Session"
|
|
981
|
+
) -> bool:
|
|
867
982
|
"""
|
|
868
983
|
Indicates whether the named key is present for the RSE.
|
|
869
984
|
|
|
@@ -885,7 +1000,11 @@ def has_rse_attribute(rse_id, key, *, session: "Session"):
|
|
|
885
1000
|
|
|
886
1001
|
|
|
887
1002
|
@read_session
|
|
888
|
-
def get_rses_with_attribute(
|
|
1003
|
+
def get_rses_with_attribute(
|
|
1004
|
+
key: str,
|
|
1005
|
+
*,
|
|
1006
|
+
session: "Session"
|
|
1007
|
+
) -> list[dict[str, Any]]:
|
|
889
1008
|
"""
|
|
890
1009
|
Return all RSEs with a certain attribute.
|
|
891
1010
|
|
|
@@ -913,7 +1032,13 @@ def get_rses_with_attribute(key, *, session: "Session"):
|
|
|
913
1032
|
|
|
914
1033
|
|
|
915
1034
|
@read_session
|
|
916
|
-
def get_rses_with_attribute_value(
|
|
1035
|
+
def get_rses_with_attribute_value(
|
|
1036
|
+
key: str,
|
|
1037
|
+
value: Union[bool, str],
|
|
1038
|
+
vo: str = 'def',
|
|
1039
|
+
*,
|
|
1040
|
+
session: "Session"
|
|
1041
|
+
) -> list[dict[str, str]]:
|
|
917
1042
|
"""
|
|
918
1043
|
Return all RSEs with a certain attribute.
|
|
919
1044
|
|
|
@@ -929,7 +1054,7 @@ def get_rses_with_attribute_value(key, value, vo='def', *, session: "Session"):
|
|
|
929
1054
|
cache_key = 'av-%s-%s' % (key, value)
|
|
930
1055
|
|
|
931
1056
|
result = REGION.get(cache_key)
|
|
932
|
-
if result
|
|
1057
|
+
if isinstance(result, NoValue):
|
|
933
1058
|
|
|
934
1059
|
rse_list = []
|
|
935
1060
|
|
|
@@ -958,6 +1083,36 @@ def get_rses_with_attribute_value(key, value, vo='def', *, session: "Session"):
|
|
|
958
1083
|
return result
|
|
959
1084
|
|
|
960
1085
|
|
|
1086
|
+
@overload
|
|
1087
|
+
def get_rse_attribute(rse_id: str, key: Literal['sign_url'], use_cache: bool = True, *, session: "Session") -> Optional[SUPPORTED_SIGN_URL_SERVICES_LITERAL]:
|
|
1088
|
+
...
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
@overload
|
|
1092
|
+
def get_rse_attribute(rse_id: str, key: Literal['sign_url'], use_cache: bool = True) -> Optional[SUPPORTED_SIGN_URL_SERVICES_LITERAL]:
|
|
1093
|
+
...
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
@overload
|
|
1097
|
+
def get_rse_attribute(rse_id: str, key: 'RSE_ATTRS_STR', use_cache: bool = True) -> Optional[str]:
|
|
1098
|
+
...
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
@overload
|
|
1102
|
+
def get_rse_attribute(rse_id: str, key: 'RSE_ATTRS_STR', use_cache: bool = True, *, session: "Session") -> Optional[str]:
|
|
1103
|
+
...
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
@overload
|
|
1107
|
+
def get_rse_attribute(rse_id: str, key: 'RSE_ATTRS_BOOL', use_cache: bool = True) -> Optional[bool]:
|
|
1108
|
+
...
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
@overload
|
|
1112
|
+
def get_rse_attribute(rse_id: str, key: 'RSE_ATTRS_BOOL', use_cache: bool = True, *, session: "Session") -> Optional[bool]:
|
|
1113
|
+
...
|
|
1114
|
+
|
|
1115
|
+
|
|
961
1116
|
@read_session
|
|
962
1117
|
def get_rse_attribute(rse_id: str, key: str, use_cache: bool = True, *, session: "Session") -> Optional[Union[str, bool]]:
|
|
963
1118
|
"""
|
|
@@ -975,7 +1130,7 @@ def get_rse_attribute(rse_id: str, key: str, use_cache: bool = True, *, session:
|
|
|
975
1130
|
if use_cache:
|
|
976
1131
|
value = REGION.get(cache_key)
|
|
977
1132
|
|
|
978
|
-
if value
|
|
1133
|
+
if not isinstance(value, NoValue):
|
|
979
1134
|
return value
|
|
980
1135
|
|
|
981
1136
|
stmt = select(
|
|
@@ -1023,7 +1178,15 @@ def parse_checksum_support_attribute(checksum_attribute: str) -> list[str]:
|
|
|
1023
1178
|
|
|
1024
1179
|
|
|
1025
1180
|
@transactional_session
|
|
1026
|
-
def set_rse_usage(
|
|
1181
|
+
def set_rse_usage(
|
|
1182
|
+
rse_id: str,
|
|
1183
|
+
source: str,
|
|
1184
|
+
used: int,
|
|
1185
|
+
free: int,
|
|
1186
|
+
files: Optional[int] = None,
|
|
1187
|
+
*,
|
|
1188
|
+
session: "Session"
|
|
1189
|
+
) -> bool:
|
|
1027
1190
|
"""
|
|
1028
1191
|
Set RSE usage information.
|
|
1029
1192
|
|
|
@@ -1048,7 +1211,13 @@ def set_rse_usage(rse_id, source, used, free, files=None, *, session: "Session")
|
|
|
1048
1211
|
|
|
1049
1212
|
|
|
1050
1213
|
@read_session
|
|
1051
|
-
def get_rse_usage(
|
|
1214
|
+
def get_rse_usage(
|
|
1215
|
+
rse_id: str,
|
|
1216
|
+
source: Optional[str] = None,
|
|
1217
|
+
per_account: bool = False,
|
|
1218
|
+
*,
|
|
1219
|
+
session: "Session"
|
|
1220
|
+
) -> list[dict[str, Any]]:
|
|
1052
1221
|
"""
|
|
1053
1222
|
get rse usage information.
|
|
1054
1223
|
|
|
@@ -1077,7 +1246,7 @@ def get_rse_usage(rse_id, source=None, per_account=False, *, session: "Session")
|
|
|
1077
1246
|
|
|
1078
1247
|
def _format_get_rse_usage(
|
|
1079
1248
|
rse_id: str,
|
|
1080
|
-
db_usages: Iterable[models.RSEUsage],
|
|
1249
|
+
db_usages: 'Iterable[models.RSEUsage]',
|
|
1081
1250
|
per_account: bool,
|
|
1082
1251
|
*,
|
|
1083
1252
|
session: "Session"
|
|
@@ -1174,14 +1343,19 @@ def delete_rse_limits(rse_id: str, name: "Optional[str]" = None, *, session: 'Se
|
|
|
1174
1343
|
raise exception.RucioException(error.args)
|
|
1175
1344
|
|
|
1176
1345
|
|
|
1177
|
-
def _sanitize_rse_transfer_limit_dict(limit_dict):
|
|
1346
|
+
def _sanitize_rse_transfer_limit_dict(limit_dict: dict[str, Any]) -> dict[str, Any]:
|
|
1178
1347
|
if limit_dict['activity'] == 'all_activities':
|
|
1179
1348
|
limit_dict['activity'] = None
|
|
1180
1349
|
return limit_dict
|
|
1181
1350
|
|
|
1182
1351
|
|
|
1183
1352
|
@read_session
|
|
1184
|
-
def get_rse_transfer_limits(
|
|
1353
|
+
def get_rse_transfer_limits(
|
|
1354
|
+
rse_id: str,
|
|
1355
|
+
activity: Optional[str] = None,
|
|
1356
|
+
*,
|
|
1357
|
+
session: "Session"
|
|
1358
|
+
) -> dict[str, Any]:
|
|
1185
1359
|
"""
|
|
1186
1360
|
Get RSE transfer limits.
|
|
1187
1361
|
|
|
@@ -1216,7 +1390,12 @@ def get_rse_transfer_limits(rse_id, activity=None, *, session: "Session"):
|
|
|
1216
1390
|
|
|
1217
1391
|
|
|
1218
1392
|
@stream_session
|
|
1219
|
-
def list_rse_usage_history(
|
|
1393
|
+
def list_rse_usage_history(
|
|
1394
|
+
rse_id: str,
|
|
1395
|
+
source: Optional[str] = None,
|
|
1396
|
+
*,
|
|
1397
|
+
session: "Session"
|
|
1398
|
+
) -> 'Iterator[dict[str, Any]]':
|
|
1220
1399
|
"""
|
|
1221
1400
|
List RSE usage history information.
|
|
1222
1401
|
|
|
@@ -1289,7 +1468,7 @@ def add_protocol(
|
|
|
1289
1468
|
if domain not in utils.rse_supported_protocol_domains():
|
|
1290
1469
|
raise exception.RSEProtocolDomainNotSupported(f"The protocol domain '{domain}' is not defined in the schema.")
|
|
1291
1470
|
for op in parameter['domains'][domain]:
|
|
1292
|
-
if op not in
|
|
1471
|
+
if op not in RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS:
|
|
1293
1472
|
raise exception.RSEOperationNotSupported(f"Operation '{op}' not defined in schema.")
|
|
1294
1473
|
op_name = op if op.startswith('third_party_copy') else f'{op}_{domain}'.lower()
|
|
1295
1474
|
priority = parameter['domains'][domain][op]
|
|
@@ -1334,7 +1513,12 @@ def add_protocol(
|
|
|
1334
1513
|
|
|
1335
1514
|
|
|
1336
1515
|
@read_session
|
|
1337
|
-
def get_rse_protocols(
|
|
1516
|
+
def get_rse_protocols(
|
|
1517
|
+
rse_id: str,
|
|
1518
|
+
schemes: Optional[list[str]] = None,
|
|
1519
|
+
*,
|
|
1520
|
+
session: "Session"
|
|
1521
|
+
) -> types.RSESettingsDict:
|
|
1338
1522
|
"""
|
|
1339
1523
|
Returns protocol information. Parameter combinations are: (operation OR default) XOR scheme.
|
|
1340
1524
|
|
|
@@ -1353,7 +1537,7 @@ def get_rse_protocols(rse_id, schemes=None, *, session: "Session") -> types.RSES
|
|
|
1353
1537
|
|
|
1354
1538
|
terms = [models.RSEProtocol.rse_id == rse_id]
|
|
1355
1539
|
if schemes:
|
|
1356
|
-
if not
|
|
1540
|
+
if not isinstance(schemes, list):
|
|
1357
1541
|
schemes = [schemes]
|
|
1358
1542
|
terms.extend([models.RSEProtocol.scheme.in_(schemes)])
|
|
1359
1543
|
|
|
@@ -1369,7 +1553,7 @@ def get_rse_protocols(rse_id, schemes=None, *, session: "Session") -> types.RSES
|
|
|
1369
1553
|
|
|
1370
1554
|
def _format_get_rse_protocols(
|
|
1371
1555
|
rse: "models.RSE | dict[str, Any]",
|
|
1372
|
-
db_protocols: Iterable[models.RSEProtocol],
|
|
1556
|
+
db_protocols: 'Iterable[models.RSEProtocol]',
|
|
1373
1557
|
rse_attributes: Optional[dict[str, Any]] = None,
|
|
1374
1558
|
*,
|
|
1375
1559
|
session: "Session"
|
|
@@ -1413,7 +1597,7 @@ def _format_get_rse_protocols(
|
|
|
1413
1597
|
'verify_checksum': verify_checksum if verify_checksum is not None else True,
|
|
1414
1598
|
'volatile': _rse['volatile']}
|
|
1415
1599
|
|
|
1416
|
-
for op in
|
|
1600
|
+
for op in RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS:
|
|
1417
1601
|
info['%s_protocol' % op] = 1 # 1 indicates the default protocol
|
|
1418
1602
|
|
|
1419
1603
|
for row in db_protocols:
|
|
@@ -1445,7 +1629,7 @@ def _format_get_rse_protocols(
|
|
|
1445
1629
|
|
|
1446
1630
|
|
|
1447
1631
|
@read_session
|
|
1448
|
-
def get_rse_info(rse_id, *, session: "Session") -> types.RSESettingsDict:
|
|
1632
|
+
def get_rse_info(rse_id: str, *, session: "Session") -> types.RSESettingsDict:
|
|
1449
1633
|
"""
|
|
1450
1634
|
For historical reasons, related to usage of rsemanager, "rse_info" is equivalent to
|
|
1451
1635
|
a cached call to get_rse_protocols without any schemes set.
|
|
@@ -1456,7 +1640,7 @@ def get_rse_info(rse_id, *, session: "Session") -> types.RSESettingsDict:
|
|
|
1456
1640
|
"""
|
|
1457
1641
|
key = 'rse_info_%s' % rse_id
|
|
1458
1642
|
result = REGION.get(key)
|
|
1459
|
-
if result
|
|
1643
|
+
if isinstance(result, NoValue):
|
|
1460
1644
|
result = get_rse_protocols(rse_id=rse_id, session=session)
|
|
1461
1645
|
REGION.set(key, result)
|
|
1462
1646
|
return result
|
|
@@ -1500,7 +1684,7 @@ def update_protocols(
|
|
|
1500
1684
|
if domain not in utils.rse_supported_protocol_domains():
|
|
1501
1685
|
raise exception.RSEProtocolDomainNotSupported(f"The protocol domain '{domain}' is not defined in the schema.")
|
|
1502
1686
|
for op in data['domains'][domain]:
|
|
1503
|
-
if op not in
|
|
1687
|
+
if op not in RSE_ALL_SUPPORTED_PROTOCOL_OPERATIONS:
|
|
1504
1688
|
raise exception.RSEOperationNotSupported(f"Operation '{op}' not defined in schema.")
|
|
1505
1689
|
op_name = op if op.startswith('third_party_copy') else f'{op}_{domain}'.lower()
|
|
1506
1690
|
priority = data['domains'][domain][op]
|
|
@@ -1678,7 +1862,11 @@ def update_rse(rse_id: str, parameters: dict[str, Any], *, session: "Session"):
|
|
|
1678
1862
|
|
|
1679
1863
|
|
|
1680
1864
|
@read_session
|
|
1681
|
-
def export_rse(
|
|
1865
|
+
def export_rse(
|
|
1866
|
+
rse_id: str,
|
|
1867
|
+
*,
|
|
1868
|
+
session: "Session"
|
|
1869
|
+
) -> dict[str, Any]:
|
|
1682
1870
|
"""
|
|
1683
1871
|
Get the internal representation of an RSE.
|
|
1684
1872
|
|
|
@@ -1724,7 +1912,12 @@ def export_rse(rse_id, *, session: "Session"):
|
|
|
1724
1912
|
|
|
1725
1913
|
|
|
1726
1914
|
@transactional_session
|
|
1727
|
-
def add_qos_policy(
|
|
1915
|
+
def add_qos_policy(
|
|
1916
|
+
rse_id: str,
|
|
1917
|
+
qos_policy: str,
|
|
1918
|
+
*,
|
|
1919
|
+
session: "Session"
|
|
1920
|
+
) -> bool:
|
|
1728
1921
|
"""
|
|
1729
1922
|
Add a QoS policy from an RSE.
|
|
1730
1923
|
|
|
@@ -1756,7 +1949,12 @@ def add_qos_policy(rse_id, qos_policy, *, session: "Session"):
|
|
|
1756
1949
|
|
|
1757
1950
|
|
|
1758
1951
|
@transactional_session
|
|
1759
|
-
def delete_qos_policy(
|
|
1952
|
+
def delete_qos_policy(
|
|
1953
|
+
rse_id: str,
|
|
1954
|
+
qos_policy: str,
|
|
1955
|
+
*,
|
|
1956
|
+
session: "Session"
|
|
1957
|
+
) -> bool:
|
|
1760
1958
|
"""
|
|
1761
1959
|
Delete a QoS policy from an RSE.
|
|
1762
1960
|
|
|
@@ -1782,7 +1980,11 @@ def delete_qos_policy(rse_id, qos_policy, *, session: "Session"):
|
|
|
1782
1980
|
|
|
1783
1981
|
|
|
1784
1982
|
@read_session
|
|
1785
|
-
def list_qos_policies(
|
|
1983
|
+
def list_qos_policies(
|
|
1984
|
+
rse_id: str,
|
|
1985
|
+
*,
|
|
1986
|
+
session: "Session"
|
|
1987
|
+
) -> list[str]:
|
|
1786
1988
|
"""
|
|
1787
1989
|
List all QoS policies of an RSE.
|
|
1788
1990
|
|
|
@@ -1808,13 +2010,15 @@ def list_qos_policies(rse_id, *, session: "Session"):
|
|
|
1808
2010
|
|
|
1809
2011
|
|
|
1810
2012
|
@transactional_session
|
|
1811
|
-
def fill_rse_expired(
|
|
2013
|
+
def fill_rse_expired(
|
|
2014
|
+
rse_id: str,
|
|
2015
|
+
*,
|
|
2016
|
+
session: "Session"
|
|
2017
|
+
) -> None:
|
|
1812
2018
|
"""
|
|
1813
2019
|
Fill the rse_usage for source expired
|
|
1814
2020
|
|
|
1815
2021
|
:param rse_id: The RSE id.
|
|
1816
|
-
|
|
1817
|
-
:returns: True if successful, except otherwise.
|
|
1818
2022
|
"""
|
|
1819
2023
|
stmt = select(
|
|
1820
2024
|
func.sum(models.RSEFileAssociation.bytes).label("bytes"),
|
|
@@ -1851,8 +2055,8 @@ def determine_audience_for_rse(rse_id: str) -> str:
|
|
|
1851
2055
|
|
|
1852
2056
|
def determine_scope_for_rse(
|
|
1853
2057
|
rse_id: str,
|
|
1854
|
-
scopes: Iterable[str],
|
|
1855
|
-
extra_scopes: Optional[Iterable[str]] = None,
|
|
2058
|
+
scopes: 'Iterable[str]',
|
|
2059
|
+
extra_scopes: Optional['Iterable[str]'] = None,
|
|
1856
2060
|
) -> str:
|
|
1857
2061
|
"""Construct the Scope claim for an RSE."""
|
|
1858
2062
|
if extra_scopes is None:
|
|
@@ -1868,7 +2072,7 @@ def determine_scope_for_rse(
|
|
|
1868
2072
|
# a base which should be removed from the prefix (in order for '/' to
|
|
1869
2073
|
# mean the entire resource associated with that issuer).
|
|
1870
2074
|
prefix = protocol['prefix']
|
|
1871
|
-
if base_path := get_rse_attribute(rse_id, RseAttr.OIDC_BASE_PATH):
|
|
2075
|
+
if base_path := get_rse_attribute(rse_id, RseAttr.OIDC_BASE_PATH): # type: ignore (session parameter missing)
|
|
1872
2076
|
prefix = prefix.removeprefix(base_path)
|
|
1873
2077
|
filtered_prefixes.add(prefix)
|
|
1874
2078
|
all_scopes = [f'{s}:{p}' for s in scopes for p in filtered_prefixes] + list(extra_scopes)
|