rucio 35.7.0__py3-none-any.whl → 37.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/alembicrevision.py +1 -1
- rucio/{daemons/c3po/collectors → cli}/__init__.py +1 -0
- rucio/cli/account.py +216 -0
- rucio-35.7.0.data/scripts/rucio → rucio/cli/bin_legacy/rucio.py +769 -486
- rucio-35.7.0.data/scripts/rucio-admin → rucio/cli/bin_legacy/rucio_admin.py +476 -423
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/accountclient.py +0 -1
- rucio/client/baseclient.py +33 -24
- rucio/client/client.py +45 -1
- rucio/client/didclient.py +5 -3
- rucio/client/downloadclient.py +6 -8
- rucio/client/replicaclient.py +0 -2
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +4 -4
- rucio/client/uploadclient.py +26 -12
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +66 -29
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +22 -35
- rucio/common/constants.py +61 -3
- rucio/common/didtype.py +72 -24
- rucio/common/dumper/__init__.py +45 -38
- rucio/common/dumper/consistency.py +75 -30
- rucio/common/dumper/data_models.py +63 -19
- rucio/common/dumper/path_parsing.py +19 -8
- rucio/common/exception.py +65 -8
- rucio/common/extra.py +5 -10
- rucio/common/logging.py +13 -13
- rucio/common/pcache.py +8 -7
- rucio/common/plugins.py +59 -27
- rucio/common/policy.py +12 -3
- rucio/common/schema/__init__.py +84 -34
- rucio/common/schema/generic.py +0 -17
- rucio/common/schema/generic_multi_vo.py +0 -17
- rucio/common/stomp_utils.py +383 -119
- rucio/common/test_rucio_server.py +12 -6
- rucio/common/types.py +132 -52
- rucio/common/utils.py +93 -643
- rucio/core/account_limit.py +14 -12
- rucio/core/authentication.py +2 -2
- rucio/core/config.py +23 -42
- rucio/core/credential.py +14 -15
- rucio/core/did.py +5 -1
- rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
- rucio/core/did_meta_plugins/filter_engine.py +62 -3
- rucio/core/did_meta_plugins/json_meta.py +2 -2
- rucio/core/did_meta_plugins/mongo_meta.py +43 -30
- rucio/core/did_meta_plugins/postgres_meta.py +75 -39
- rucio/core/identity.py +6 -5
- rucio/core/importer.py +4 -3
- rucio/core/lifetime_exception.py +2 -2
- rucio/core/lock.py +8 -7
- rucio/core/message.py +6 -0
- rucio/core/monitor.py +30 -29
- rucio/core/naming_convention.py +2 -2
- rucio/core/nongrid_trace.py +2 -2
- rucio/core/oidc.py +11 -9
- rucio/core/permission/__init__.py +79 -37
- rucio/core/permission/generic.py +1 -7
- rucio/core/permission/generic_multi_vo.py +1 -7
- rucio/core/quarantined_replica.py +4 -3
- rucio/core/replica.py +464 -139
- rucio/core/replica_sorter.py +55 -59
- rucio/core/request.py +34 -32
- rucio/core/rse.py +301 -97
- rucio/core/rse_counter.py +1 -2
- rucio/core/rse_expression_parser.py +7 -7
- rucio/core/rse_selector.py +9 -7
- rucio/core/rule.py +41 -40
- rucio/core/rule_grouping.py +42 -40
- rucio/core/scope.py +5 -4
- rucio/core/subscription.py +26 -28
- rucio/core/topology.py +11 -11
- rucio/core/trace.py +2 -2
- rucio/core/transfer.py +29 -15
- rucio/core/volatile_replica.py +4 -3
- rucio/daemons/atropos/atropos.py +1 -1
- rucio/daemons/auditor/__init__.py +2 -2
- rucio/daemons/auditor/srmdumps.py +6 -6
- rucio/daemons/automatix/automatix.py +32 -21
- rucio/daemons/badreplicas/necromancer.py +2 -2
- rucio/daemons/bb8/nuclei_background_rebalance.py +1 -1
- rucio/daemons/bb8/t2_background_rebalance.py +1 -1
- rucio/daemons/cache/consumer.py +26 -90
- rucio/daemons/common.py +15 -25
- rucio/daemons/conveyor/finisher.py +2 -2
- rucio/daemons/conveyor/poller.py +18 -28
- rucio/daemons/conveyor/receiver.py +53 -123
- rucio/daemons/conveyor/stager.py +1 -0
- rucio/daemons/conveyor/submitter.py +3 -3
- rucio/daemons/hermes/hermes.py +129 -369
- rucio/daemons/judge/evaluator.py +2 -2
- rucio/daemons/oauthmanager/oauthmanager.py +3 -3
- rucio/daemons/reaper/dark_reaper.py +7 -3
- rucio/daemons/reaper/reaper.py +12 -16
- rucio/daemons/rsedecommissioner/config.py +1 -1
- rucio/daemons/rsedecommissioner/profiles/generic.py +5 -4
- rucio/daemons/rsedecommissioner/profiles/types.py +7 -6
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +1 -1
- rucio/daemons/storage/consistency/actions.py +8 -6
- rucio/daemons/tracer/kronos.py +117 -142
- rucio/db/sqla/constants.py +5 -0
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +4 -4
- rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
- rucio/db/sqla/models.py +157 -154
- rucio/db/sqla/session.py +58 -27
- rucio/db/sqla/types.py +2 -2
- rucio/db/sqla/util.py +2 -2
- rucio/gateway/account.py +18 -12
- rucio/gateway/account_limit.py +137 -60
- rucio/gateway/authentication.py +18 -12
- rucio/gateway/config.py +30 -20
- rucio/gateway/credential.py +9 -10
- rucio/gateway/did.py +70 -53
- rucio/gateway/dirac.py +6 -4
- rucio/gateway/exporter.py +3 -2
- rucio/gateway/heartbeat.py +6 -4
- rucio/gateway/identity.py +36 -51
- rucio/gateway/importer.py +3 -2
- rucio/gateway/lifetime_exception.py +3 -2
- rucio/gateway/meta_conventions.py +17 -6
- rucio/gateway/permission.py +4 -1
- rucio/gateway/quarantined_replica.py +3 -2
- rucio/gateway/replica.py +31 -22
- rucio/gateway/request.py +27 -18
- rucio/gateway/rse.py +69 -37
- rucio/gateway/rule.py +46 -26
- rucio/gateway/scope.py +3 -2
- rucio/gateway/subscription.py +14 -11
- rucio/gateway/vo.py +12 -8
- rucio/rse/__init__.py +3 -3
- rucio/rse/protocols/bittorrent.py +11 -1
- rucio/rse/protocols/cache.py +0 -11
- rucio/rse/protocols/dummy.py +0 -11
- rucio/rse/protocols/gfal.py +14 -9
- rucio/rse/protocols/globus.py +1 -1
- rucio/rse/protocols/http_cache.py +1 -1
- rucio/rse/protocols/posix.py +2 -2
- rucio/rse/protocols/protocol.py +84 -317
- rucio/rse/protocols/rclone.py +2 -1
- rucio/rse/protocols/rfio.py +10 -1
- rucio/rse/protocols/ssh.py +2 -1
- rucio/rse/protocols/storm.py +2 -13
- rucio/rse/protocols/webdav.py +74 -30
- rucio/rse/protocols/xrootd.py +2 -1
- rucio/rse/rsemanager.py +170 -53
- rucio/rse/translation.py +260 -0
- rucio/tests/common.py +23 -13
- rucio/tests/common_server.py +26 -9
- rucio/transfertool/bittorrent.py +15 -14
- rucio/transfertool/bittorrent_driver.py +5 -7
- rucio/transfertool/bittorrent_driver_qbittorrent.py +9 -8
- rucio/transfertool/fts3.py +20 -16
- rucio/transfertool/mock.py +2 -3
- rucio/vcsversion.py +4 -4
- rucio/version.py +7 -0
- rucio/web/rest/flaskapi/v1/accounts.py +17 -3
- rucio/web/rest/flaskapi/v1/auth.py +5 -5
- rucio/web/rest/flaskapi/v1/credentials.py +3 -2
- rucio/web/rest/flaskapi/v1/dids.py +21 -15
- rucio/web/rest/flaskapi/v1/identities.py +33 -9
- rucio/web/rest/flaskapi/v1/redirect.py +5 -4
- rucio/web/rest/flaskapi/v1/replicas.py +12 -8
- rucio/web/rest/flaskapi/v1/rses.py +15 -4
- rucio/web/rest/flaskapi/v1/traces.py +56 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/alembic_offline.ini.template +1 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.atlas.client.template +3 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio.cfg.template +3 -19
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rucio_multi_vo.cfg.template +1 -18
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/requirements.server.txt +97 -68
- rucio-37.0.0rc2.data/scripts/rucio +133 -0
- rucio-37.0.0rc2.data/scripts/rucio-admin +97 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-atropos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-auditor +2 -1
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-automatix +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-client +17 -10
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-receiver +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-kronos +1 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-minos-temporary-expiration +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-necromancer +2 -2
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-reaper +6 -6
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-transmogrifier +2 -2
- rucio-37.0.0rc2.dist-info/METADATA +92 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/RECORD +239 -245
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/common/schema/atlas.py +0 -413
- rucio/common/schema/belleii.py +0 -408
- rucio/common/schema/domatpc.py +0 -401
- rucio/common/schema/escape.py +0 -426
- rucio/common/schema/icecube.py +0 -406
- rucio/core/permission/atlas.py +0 -1348
- rucio/core/permission/belleii.py +0 -1077
- rucio/core/permission/escape.py +0 -1078
- rucio/daemons/c3po/algorithms/__init__.py +0 -13
- rucio/daemons/c3po/algorithms/simple.py +0 -134
- rucio/daemons/c3po/algorithms/t2_free_space.py +0 -128
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +0 -130
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +0 -294
- rucio/daemons/c3po/c3po.py +0 -371
- rucio/daemons/c3po/collectors/agis.py +0 -108
- rucio/daemons/c3po/collectors/free_space.py +0 -81
- rucio/daemons/c3po/collectors/jedi_did.py +0 -57
- rucio/daemons/c3po/collectors/mock_did.py +0 -51
- rucio/daemons/c3po/collectors/network_metrics.py +0 -71
- rucio/daemons/c3po/collectors/workload.py +0 -112
- rucio/daemons/c3po/utils/__init__.py +0 -13
- rucio/daemons/c3po/utils/dataset_cache.py +0 -50
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +0 -56
- rucio/daemons/c3po/utils/expiring_list.py +0 -62
- rucio/daemons/c3po/utils/popularity.py +0 -85
- rucio/daemons/c3po/utils/timeseries.py +0 -89
- rucio/rse/protocols/gsiftp.py +0 -92
- rucio-35.7.0.data/scripts/rucio-c3po +0 -85
- rucio-35.7.0.dist-info/METADATA +0 -72
- /rucio/{daemons/c3po → cli/bin_legacy}/__init__.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/ldap.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/bootstrap.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/merge_rucio_configs.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/data/rucio/tools/reset_database.py +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-account +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-collection-replica +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-abacus-rse +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-bb8 +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-cache-consumer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-finisher +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-poller +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-preparer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-stager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-submitter +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-conveyor-throttler +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dark-reaper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-dumper +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-follower +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-hermes +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-cleaner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-evaluator +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-injector +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-judge-repairer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-oauth-manager +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-replica-recoverer +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-rse-decommissioner +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-storage-consistency-actions +0 -0
- {rucio-35.7.0.data → rucio-37.0.0rc2.data}/scripts/rucio-undertaker +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/WHEEL +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/licenses/LICENSE +0 -0
- {rucio-35.7.0.dist-info → rucio-37.0.0rc2.dist-info}/top_level.txt +0 -0
rucio/rse/translation.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
import hashlib
|
|
15
|
+
import importlib
|
|
16
|
+
import logging
|
|
17
|
+
from configparser import NoOptionError, NoSectionError
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
19
|
+
|
|
20
|
+
from rucio.common import config
|
|
21
|
+
from rucio.common.constants import RseAttr
|
|
22
|
+
from rucio.common.exception import ConfigNotFound
|
|
23
|
+
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from collections.abc import Callable, Mapping
|
|
27
|
+
|
|
28
|
+
from rucio.common.types import RSESettingsDict
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RSEDeterministicScopeTranslation(PolicyPackageAlgorithms):
|
|
32
|
+
"""
|
|
33
|
+
Translates a pfn dictionary into a scope and name
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
_algorithm_type = "pfn2lfn"
|
|
37
|
+
|
|
38
|
+
def __init__(self, vo: str = 'def'):
|
|
39
|
+
super().__init__()
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
algorithm_name = config.config_get('policy', self._algorithm_type)
|
|
45
|
+
except (ConfigNotFound, NoOptionError, NoSectionError, RuntimeError):
|
|
46
|
+
logger.debug("PFN2LFN: no algorithm specified in the config.")
|
|
47
|
+
if super()._supports(self._algorithm_type, vo):
|
|
48
|
+
algorithm_name = vo
|
|
49
|
+
else:
|
|
50
|
+
algorithm_name = "def"
|
|
51
|
+
logger.debug("PFN2LFN: Falling back to %s algorithm.", 'default' if algorithm_name == 'def' else algorithm_name)
|
|
52
|
+
|
|
53
|
+
self.parser = self.get_parser(algorithm_name)
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def _module_init_(cls) -> None:
|
|
57
|
+
"""
|
|
58
|
+
Registers the included scope extraction algorithms
|
|
59
|
+
"""
|
|
60
|
+
cls.register(cls._default, "def")
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def get_parser(cls, algorithm_name: str) -> 'Callable[..., Any]':
|
|
64
|
+
return super()._get_one_algorithm(cls._algorithm_type, algorithm_name)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def register(
|
|
68
|
+
cls,
|
|
69
|
+
pfn2lfn_callable: 'Callable',
|
|
70
|
+
name: Optional[str] = None
|
|
71
|
+
) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Provided a callable function, register it as one of the valid PFN2LFN algorithms.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
:param pfn2lfn_callable: Callable function to use.
|
|
77
|
+
:param name: Algorithm name used for registration.
|
|
78
|
+
"""
|
|
79
|
+
if name is None:
|
|
80
|
+
name = pfn2lfn_callable.__name__
|
|
81
|
+
algorithm_dict = {name: pfn2lfn_callable}
|
|
82
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
83
|
+
|
|
84
|
+
@staticmethod
|
|
85
|
+
def _default(parsed_pfn: 'Mapping[str, str]') -> tuple[str, str]:
|
|
86
|
+
""" Translate pfn to name/scope pair
|
|
87
|
+
|
|
88
|
+
:param parsed_pfn: dictionary representing pfn containing:
|
|
89
|
+
- path: str,
|
|
90
|
+
- name: str
|
|
91
|
+
:return: tuple containing name, scope
|
|
92
|
+
"""
|
|
93
|
+
path = parsed_pfn['path']
|
|
94
|
+
scope = path.lstrip('/').split('/')[0]
|
|
95
|
+
name = parsed_pfn['name']
|
|
96
|
+
return name, scope
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
RSEDeterministicScopeTranslation._module_init_() # pylint: disable=protected-access
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
103
|
+
"""
|
|
104
|
+
Execute the logic for translating a LFN to a path.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
_DEFAULT_LFN2PFN = "hash"
|
|
108
|
+
_algorithm_type = "lfn2pfn"
|
|
109
|
+
|
|
110
|
+
def __init__(
|
|
111
|
+
self,
|
|
112
|
+
rse: Optional[str] = None,
|
|
113
|
+
rse_attributes: Optional["RSESettingsDict"] = None,
|
|
114
|
+
protocol_attributes: Optional[dict[str, Any]] = None
|
|
115
|
+
):
|
|
116
|
+
"""
|
|
117
|
+
Initialize a translator object from the RSE, its attributes, and the protocol-specific
|
|
118
|
+
attributes.
|
|
119
|
+
|
|
120
|
+
:param rse: Name of RSE for this translation.
|
|
121
|
+
:param rse_attributes: A dictionary of RSE-specific attributes for use in the translation.
|
|
122
|
+
:param protocol_attributes: A dictionary of RSE/protocol-specific attributes.
|
|
123
|
+
"""
|
|
124
|
+
super().__init__()
|
|
125
|
+
self.rse = rse
|
|
126
|
+
self.rse_attributes = rse_attributes if rse_attributes else {}
|
|
127
|
+
self.protocol_attributes = protocol_attributes if protocol_attributes else {}
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def supports(
|
|
131
|
+
cls,
|
|
132
|
+
name: str
|
|
133
|
+
) -> bool:
|
|
134
|
+
"""
|
|
135
|
+
Check to see if a specific algorithm is supported.
|
|
136
|
+
|
|
137
|
+
:param name: Name of the deterministic algorithm.
|
|
138
|
+
:returns: True if `name` is an algorithm supported by the translator class, False otherwise
|
|
139
|
+
"""
|
|
140
|
+
return super()._supports(cls._algorithm_type, name)
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def register(
|
|
144
|
+
cls,
|
|
145
|
+
lfn2pfn_callable: 'Callable',
|
|
146
|
+
name: Optional[str] = None
|
|
147
|
+
) -> None:
|
|
148
|
+
"""
|
|
149
|
+
Provided a callable function, register it as one of the valid LFN2PFN algorithms.
|
|
150
|
+
|
|
151
|
+
The callable will receive five arguments:
|
|
152
|
+
- scope: Scope of the LFN.
|
|
153
|
+
- name: LFN's path name
|
|
154
|
+
- rse: RSE name the translation is being done for.
|
|
155
|
+
- rse_attributes: Attributes of the RSE.
|
|
156
|
+
- protocol_attributes: Attributes of the RSE's protocol
|
|
157
|
+
The return value should be the last part of the PFN - it will be appended to the
|
|
158
|
+
rest of the URL.
|
|
159
|
+
|
|
160
|
+
:param lfn2pfn_callable: Callable function to use for generating paths.
|
|
161
|
+
:param name: Algorithm name used for registration. If None, then `lfn2pfn_callable.__name__` is used.
|
|
162
|
+
"""
|
|
163
|
+
if name is None:
|
|
164
|
+
name = lfn2pfn_callable.__name__
|
|
165
|
+
algorithm_dict = {name: lfn2pfn_callable}
|
|
166
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
def __hash(
|
|
170
|
+
scope: str,
|
|
171
|
+
name: str,
|
|
172
|
+
rse: str,
|
|
173
|
+
rse_attrs: dict[str, Any],
|
|
174
|
+
protocol_attrs: dict[str, Any]
|
|
175
|
+
) -> str:
|
|
176
|
+
"""
|
|
177
|
+
Given a LFN, turn it into a sub-directory structure using a hash function.
|
|
178
|
+
|
|
179
|
+
This takes the MD5 of the LFN and uses the first four characters as a subdirectory
|
|
180
|
+
name.
|
|
181
|
+
|
|
182
|
+
:param scope: Scope of the LFN.
|
|
183
|
+
:param name: File name of the LFN.
|
|
184
|
+
:param rse: RSE for PFN (ignored)
|
|
185
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
186
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
187
|
+
:returns: Path for use in the PFN generation.
|
|
188
|
+
"""
|
|
189
|
+
del rse
|
|
190
|
+
del rse_attrs
|
|
191
|
+
del protocol_attrs
|
|
192
|
+
hstr = hashlib.md5(('%s:%s' % (scope, name)).encode('utf-8')).hexdigest()
|
|
193
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
194
|
+
scope = scope.replace('.', '/')
|
|
195
|
+
return '%s/%s/%s/%s' % (scope, hstr[0:2], hstr[2:4], name)
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
def __identity(
|
|
199
|
+
scope: str,
|
|
200
|
+
name: str,
|
|
201
|
+
rse: str,
|
|
202
|
+
rse_attrs: dict[str, Any],
|
|
203
|
+
protocol_attrs: dict[str, Any]
|
|
204
|
+
) -> str:
|
|
205
|
+
"""
|
|
206
|
+
Given a LFN, convert it directly to a path using the mapping:
|
|
207
|
+
|
|
208
|
+
scope:path -> scope/path
|
|
209
|
+
|
|
210
|
+
:param scope: Scope of the LFN.
|
|
211
|
+
:param name: File name of the LFN.
|
|
212
|
+
:param rse: RSE for PFN (ignored)
|
|
213
|
+
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
214
|
+
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
215
|
+
:returns: Path for use in the PFN generation.
|
|
216
|
+
"""
|
|
217
|
+
del rse
|
|
218
|
+
del rse_attrs
|
|
219
|
+
del protocol_attrs
|
|
220
|
+
if scope.startswith('user') or scope.startswith('group'):
|
|
221
|
+
scope = scope.replace('.', '/')
|
|
222
|
+
return '%s/%s' % (scope, name)
|
|
223
|
+
|
|
224
|
+
@classmethod
|
|
225
|
+
def _module_init_(cls) -> None:
|
|
226
|
+
"""
|
|
227
|
+
Initialize the class object on first module load.
|
|
228
|
+
"""
|
|
229
|
+
cls.register(cls.__hash, "hash")
|
|
230
|
+
cls.register(cls.__identity, "identity")
|
|
231
|
+
policy_module = None
|
|
232
|
+
try:
|
|
233
|
+
policy_module = config.config_get('policy', 'lfn2pfn_module')
|
|
234
|
+
except (ConfigNotFound, NoOptionError, NoSectionError):
|
|
235
|
+
pass
|
|
236
|
+
if policy_module:
|
|
237
|
+
importlib.import_module(policy_module)
|
|
238
|
+
|
|
239
|
+
cls._DEFAULT_LFN2PFN = config.get_lfn2pfn_algorithm_default()
|
|
240
|
+
|
|
241
|
+
def path(
|
|
242
|
+
self,
|
|
243
|
+
scope: str,
|
|
244
|
+
name: str
|
|
245
|
+
) -> str:
|
|
246
|
+
""" Transforms the logical file name into a PFN's path.
|
|
247
|
+
|
|
248
|
+
:param lfn: filename
|
|
249
|
+
:param scope: scope
|
|
250
|
+
|
|
251
|
+
:returns: RSE specific URI of the physical file
|
|
252
|
+
"""
|
|
253
|
+
algorithm = self.rse_attributes.get(RseAttr.LFN2PFN_ALGORITHM, 'default')
|
|
254
|
+
if algorithm == 'default':
|
|
255
|
+
algorithm = RSEDeterministicTranslation._DEFAULT_LFN2PFN
|
|
256
|
+
algorithm_callable = super()._get_one_algorithm(RSEDeterministicTranslation._algorithm_type, algorithm)
|
|
257
|
+
return algorithm_callable(scope, name, self.rse, self.rse_attributes, self.protocol_attributes)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
RSEDeterministicTranslation._module_init_() # pylint: disable=protected-access
|
rucio/tests/common.py
CHANGED
|
@@ -18,12 +18,11 @@ import json
|
|
|
18
18
|
import os
|
|
19
19
|
import tempfile
|
|
20
20
|
from collections import namedtuple
|
|
21
|
-
from collections.abc import Callable, Iterable
|
|
22
21
|
from functools import wraps
|
|
23
22
|
from os import rename
|
|
24
23
|
from random import choice, choices
|
|
25
24
|
from string import ascii_letters, ascii_uppercase, digits
|
|
26
|
-
from typing import Any, Optional
|
|
25
|
+
from typing import IO, TYPE_CHECKING, Any, Literal, Optional
|
|
27
26
|
|
|
28
27
|
import pytest
|
|
29
28
|
import requests
|
|
@@ -32,6 +31,12 @@ from rucio.common.config import config_get, config_get_bool, get_config_dirs
|
|
|
32
31
|
from rucio.common.utils import execute
|
|
33
32
|
from rucio.common.utils import generate_uuid as uuid
|
|
34
33
|
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
36
|
+
from types import ModuleType
|
|
37
|
+
|
|
38
|
+
from werkzeug.test import TestResponse
|
|
39
|
+
|
|
35
40
|
skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
|
|
36
41
|
reason='fails if no rse-accounts.cfg found')
|
|
37
42
|
skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and os.environ['RDBMS'] == 'sqlite',
|
|
@@ -40,6 +45,8 @@ skip_multivo = pytest.mark.skipif('SUITE' in os.environ and os.environ['SUITE']
|
|
|
40
45
|
reason="does not work for multiVO")
|
|
41
46
|
skip_non_belleii = pytest.mark.skipif(not ('POLICY' in os.environ and os.environ['POLICY'] == 'belleii'),
|
|
42
47
|
reason="specific belleii tests")
|
|
48
|
+
skip_outside_gh_actions = pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") != "true",
|
|
49
|
+
reason="Skipping tests outside GitHub Actions")
|
|
43
50
|
|
|
44
51
|
|
|
45
52
|
def is_influxdb_available() -> bool:
|
|
@@ -167,10 +174,13 @@ def make_temp_file(dir_: str, data: str) -> str:
|
|
|
167
174
|
|
|
168
175
|
|
|
169
176
|
@contextlib.contextmanager
|
|
170
|
-
def mock_open(module, file_like_object):
|
|
177
|
+
def mock_open(module: "ModuleType", file_like_object: IO):
|
|
171
178
|
call_info = {}
|
|
172
179
|
|
|
173
|
-
def mocked_open(
|
|
180
|
+
def mocked_open(
|
|
181
|
+
filename: str,
|
|
182
|
+
mode: str = 'r'
|
|
183
|
+
) -> contextlib.closing[IO]:
|
|
174
184
|
call_info['filename'] = filename
|
|
175
185
|
call_info['mode'] = mode
|
|
176
186
|
file_like_object.close = lambda: None
|
|
@@ -184,7 +194,7 @@ def mock_open(module, file_like_object):
|
|
|
184
194
|
delattr(module, 'open')
|
|
185
195
|
|
|
186
196
|
|
|
187
|
-
def print_response(rest_response):
|
|
197
|
+
def print_response(rest_response: "TestResponse") -> None:
|
|
188
198
|
print('Status:', rest_response.status)
|
|
189
199
|
print()
|
|
190
200
|
nohdrs = True
|
|
@@ -202,31 +212,31 @@ def print_response(rest_response):
|
|
|
202
212
|
print(text if text else '<no content>')
|
|
203
213
|
|
|
204
214
|
|
|
205
|
-
def headers(*iterables: Iterable):
|
|
215
|
+
def headers(*iterables: "Iterable") -> list[itertools.chain]:
|
|
206
216
|
return list(itertools.chain(*iterables))
|
|
207
217
|
|
|
208
218
|
|
|
209
|
-
def loginhdr(account: str, username: str, password: str):
|
|
219
|
+
def loginhdr(account: str, username: str, password: str) -> "Iterator[tuple[Literal['X-Rucio-Account', 'X-Rucio-Username', 'X-Rucio-Password'], str]]":
|
|
210
220
|
yield 'X-Rucio-Account', str(account)
|
|
211
221
|
yield 'X-Rucio-Username', str(username)
|
|
212
222
|
yield 'X-Rucio-Password', str(password)
|
|
213
223
|
|
|
214
224
|
|
|
215
|
-
def auth(token):
|
|
225
|
+
def auth(token) -> "Iterator[tuple[Literal['X-Rucio-Auth-Token'], str]]":
|
|
216
226
|
yield 'X-Rucio-Auth-Token', str(token)
|
|
217
227
|
|
|
218
228
|
|
|
219
|
-
def vohdr(vo: str):
|
|
229
|
+
def vohdr(vo: str) -> "Iterator[tuple[Literal['X-Rucio-VO'], str]]":
|
|
220
230
|
if vo:
|
|
221
231
|
yield 'X-Rucio-VO', str(vo)
|
|
222
232
|
|
|
223
233
|
|
|
224
|
-
def hdrdict(dictionary: dict):
|
|
234
|
+
def hdrdict(dictionary: dict[str, Any]) -> "Iterator[tuple[str, str]]":
|
|
225
235
|
for key in dictionary:
|
|
226
236
|
yield str(key), str(dictionary[key])
|
|
227
237
|
|
|
228
238
|
|
|
229
|
-
def accept(mimetype):
|
|
239
|
+
def accept(mimetype: "Mime") -> "Iterator[tuple[Literal['Accept'], Mime]]":
|
|
230
240
|
yield 'Accept', mimetype
|
|
231
241
|
|
|
232
242
|
|
|
@@ -244,9 +254,9 @@ def load_test_conf_file(file_name: str) -> dict[str, Any]:
|
|
|
244
254
|
return json.load(f)
|
|
245
255
|
|
|
246
256
|
|
|
247
|
-
def remove_config(func: Callable) -> Callable:
|
|
257
|
+
def remove_config(func: "Callable") -> "Callable":
|
|
248
258
|
@wraps(func)
|
|
249
|
-
def wrapper(*args, **kwargs):
|
|
259
|
+
def wrapper(*args, **kwargs) -> None:
|
|
250
260
|
for configfile in get_config_dirs():
|
|
251
261
|
# Rename the config to <config>.tmp
|
|
252
262
|
try:
|
rucio/tests/common_server.py
CHANGED
|
@@ -12,9 +12,10 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import copy
|
|
15
|
+
from typing import TYPE_CHECKING, Optional
|
|
15
16
|
|
|
16
17
|
from sqlalchemy import and_, delete, exists, select
|
|
17
|
-
from sqlalchemy.orm import aliased
|
|
18
|
+
from sqlalchemy.orm import Session, aliased
|
|
18
19
|
|
|
19
20
|
from rucio.core import config as core_config
|
|
20
21
|
from rucio.core.vo import map_vo
|
|
@@ -23,15 +24,26 @@ from rucio.db.sqla.session import get_session, transactional_session
|
|
|
23
24
|
|
|
24
25
|
from .common import get_long_vo
|
|
25
26
|
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from collections.abc import Iterator
|
|
29
|
+
|
|
30
|
+
from sqlalchemy.sql.elements import ColumnElement
|
|
31
|
+
from sqlalchemy.sql.schema import ForeignKeyConstraint
|
|
32
|
+
from sqlalchemy.sql.selectable import FromClause
|
|
33
|
+
|
|
26
34
|
# Functions containing server-only includes that can't be included in client tests
|
|
27
35
|
# For each table, get the foreign key constraints from all other tables towards this table.
|
|
28
|
-
INBOUND_FOREIGN_KEYS = {}
|
|
36
|
+
INBOUND_FOREIGN_KEYS: "dict[FromClause, set[ForeignKeyConstraint]]" = {}
|
|
29
37
|
for __table in models.BASE.metadata.tables.values():
|
|
30
38
|
for __fk in __table.foreign_key_constraints:
|
|
31
39
|
INBOUND_FOREIGN_KEYS.setdefault(__fk.referred_table, set()).add(__fk)
|
|
32
40
|
|
|
33
41
|
|
|
34
|
-
def _dependency_paths(
|
|
42
|
+
def _dependency_paths(
|
|
43
|
+
stack: list["ForeignKeyConstraint"],
|
|
44
|
+
nb_times_in_stack: dict["FromClause", int],
|
|
45
|
+
cur_table: "FromClause"
|
|
46
|
+
) -> "Iterator":
|
|
35
47
|
"""
|
|
36
48
|
Generates lists of foreign keys: paths starting at cur_table and
|
|
37
49
|
navigating the table graph via foreign key constraints.
|
|
@@ -61,7 +73,12 @@ def _dependency_paths(stack, nb_times_in_stack, cur_table):
|
|
|
61
73
|
|
|
62
74
|
|
|
63
75
|
@transactional_session
|
|
64
|
-
def cleanup_db_deps(
|
|
76
|
+
def cleanup_db_deps(
|
|
77
|
+
model: models.BASE,
|
|
78
|
+
select_rows_stmt: "ColumnElement",
|
|
79
|
+
*,
|
|
80
|
+
session: Optional["Session"] = None
|
|
81
|
+
) -> None:
|
|
65
82
|
"""
|
|
66
83
|
Removes rows which have foreign key constraints pointing to rows
|
|
67
84
|
selected by `select_rows_stmt` in `model`. The deletion is transitive.
|
|
@@ -81,10 +98,10 @@ def cleanup_db_deps(model, select_rows_stmt, *, session=None):
|
|
|
81
98
|
else:
|
|
82
99
|
seen_tables.add(current_table)
|
|
83
100
|
|
|
84
|
-
filters.append(and_(current_table.columns.get(e.parent.name) == referred_table.columns.get(e.column.name) for e in fk.elements))
|
|
101
|
+
filters.append(and_(current_table.columns.get(e.parent.name) == referred_table.columns.get(e.column.name) for e in fk.elements)) # type: ignore
|
|
85
102
|
referred_table = current_table
|
|
86
103
|
|
|
87
|
-
if session.bind.dialect.name == 'mysql':
|
|
104
|
+
if session.bind.dialect.name == 'mysql': # type: ignore (bind could be None)
|
|
88
105
|
stmt = delete(
|
|
89
106
|
current_table
|
|
90
107
|
).where(
|
|
@@ -111,10 +128,10 @@ def cleanup_db_deps(model, select_rows_stmt, *, session=None):
|
|
|
111
128
|
synchronize_session=False
|
|
112
129
|
)
|
|
113
130
|
|
|
114
|
-
session.execute(stmt)
|
|
131
|
+
session.execute(stmt) # type: ignore (Session could be None)
|
|
115
132
|
|
|
116
133
|
|
|
117
|
-
def reset_config_table():
|
|
134
|
+
def reset_config_table() -> None:
|
|
118
135
|
""" Clear the config table and install any default entries needed for the tests.
|
|
119
136
|
"""
|
|
120
137
|
db_session = get_session()
|
|
@@ -124,7 +141,7 @@ def reset_config_table():
|
|
|
124
141
|
core_config.set("vo-map", "testvo2", "ts2")
|
|
125
142
|
|
|
126
143
|
|
|
127
|
-
def get_vo():
|
|
144
|
+
def get_vo() -> str:
|
|
128
145
|
""" Gets the current short/mapped VO name for testing.
|
|
129
146
|
Maps the vo name to the short name, if configured.
|
|
130
147
|
:returns: VO name string.
|
rucio/transfertool/bittorrent.py
CHANGED
|
@@ -14,26 +14,27 @@
|
|
|
14
14
|
|
|
15
15
|
import base64
|
|
16
16
|
import logging
|
|
17
|
-
from collections.abc import Mapping, Sequence
|
|
18
17
|
from os import path
|
|
19
18
|
from typing import TYPE_CHECKING, Any, Optional
|
|
20
19
|
|
|
21
|
-
from rucio.common import
|
|
20
|
+
from rucio.common.bittorrent import construct_torrent
|
|
22
21
|
from rucio.common.config import config_get
|
|
23
22
|
from rucio.common.constants import RseAttr
|
|
24
23
|
from rucio.common.extra import import_extras
|
|
25
|
-
from rucio.common.utils import construct_torrent
|
|
26
24
|
from rucio.core.did_meta_plugins import get_metadata
|
|
27
25
|
from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder
|
|
28
26
|
|
|
29
|
-
from .bittorrent_driver import BittorrentDriver
|
|
30
|
-
|
|
31
27
|
if TYPE_CHECKING:
|
|
28
|
+
from collections.abc import Mapping, Sequence
|
|
29
|
+
|
|
30
|
+
from rucio.common import types
|
|
32
31
|
from rucio.core.request import DirectTransfer
|
|
33
32
|
from rucio.core.rse import RseData
|
|
34
33
|
|
|
34
|
+
from .bittorrent_driver import BittorrentDriver
|
|
35
|
+
|
|
35
36
|
DRIVER_NAME_RSE_ATTRIBUTE = 'bittorrent_driver'
|
|
36
|
-
DRIVER_CLASSES_BY_NAME: dict[str, type[BittorrentDriver]] = {}
|
|
37
|
+
DRIVER_CLASSES_BY_NAME: dict[str, type['BittorrentDriver']] = {}
|
|
37
38
|
|
|
38
39
|
EXTRA_MODULES = import_extras(['qbittorrentapi'])
|
|
39
40
|
|
|
@@ -51,7 +52,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
51
52
|
|
|
52
53
|
required_rse_attrs = (DRIVER_NAME_RSE_ATTRIBUTE, )
|
|
53
54
|
|
|
54
|
-
def __init__(self, external_host: str, logger: types.LoggerFunction = logging.log) -> None:
|
|
55
|
+
def __init__(self, external_host: str, logger: 'types.LoggerFunction' = logging.log) -> None:
|
|
55
56
|
super().__init__(external_host=external_host, logger=logger)
|
|
56
57
|
|
|
57
58
|
self._drivers_by_rse_id = {}
|
|
@@ -60,7 +61,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
60
61
|
self.tracker = config_get('transfers', 'bittorrent_tracker_addr', raise_exception=False, default=None)
|
|
61
62
|
|
|
62
63
|
@classmethod
|
|
63
|
-
def _pick_management_api_driver_cls(cls: "type[BittorrentTransfertool]", rse: "RseData") -> Optional[type[BittorrentDriver]]:
|
|
64
|
+
def _pick_management_api_driver_cls(cls: "type[BittorrentTransfertool]", rse: "RseData") -> Optional[type['BittorrentDriver']]:
|
|
64
65
|
driver_cls = DRIVER_CLASSES_BY_NAME.get(rse.attributes.get(DRIVER_NAME_RSE_ATTRIBUTE, ''))
|
|
65
66
|
if driver_cls is None:
|
|
66
67
|
return None
|
|
@@ -68,7 +69,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
68
69
|
return None
|
|
69
70
|
return driver_cls
|
|
70
71
|
|
|
71
|
-
def _driver_for_rse(self, rse: "RseData") -> Optional[BittorrentDriver]:
|
|
72
|
+
def _driver_for_rse(self, rse: "RseData") -> Optional['BittorrentDriver']:
|
|
72
73
|
driver = self._drivers_by_rse_id.get(rse.id)
|
|
73
74
|
if driver:
|
|
74
75
|
return driver
|
|
@@ -93,7 +94,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
93
94
|
def submission_builder_for_path(
|
|
94
95
|
cls: "type[BittorrentTransfertool]",
|
|
95
96
|
transfer_path: "list[DirectTransfer]",
|
|
96
|
-
logger: types.LoggerFunction = logging.log
|
|
97
|
+
logger: 'types.LoggerFunction' = logging.log
|
|
97
98
|
) -> "tuple[list[DirectTransfer], Optional[TransferToolBuilder]]":
|
|
98
99
|
hop = transfer_path[0]
|
|
99
100
|
if hop.rws.byte_count == 0:
|
|
@@ -121,7 +122,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
121
122
|
return [{'transfers': transfer_path, 'job_params': {}} for transfer_path in transfer_paths]
|
|
122
123
|
|
|
123
124
|
@staticmethod
|
|
124
|
-
def _connect_directly(torrent_id: str, peers_drivers: Sequence[BittorrentDriver]) -> None:
|
|
125
|
+
def _connect_directly(torrent_id: str, peers_drivers: 'Sequence[BittorrentDriver]') -> None:
|
|
125
126
|
peer_addr = []
|
|
126
127
|
for i, driver in enumerate(peers_drivers):
|
|
127
128
|
peer_addr.append(driver.listen_addr())
|
|
@@ -178,7 +179,7 @@ class BittorrentTransfertool(Transfertool):
|
|
|
178
179
|
self._connect_directly(torrent_id, [dst_driver] + list(src_drivers.values()))
|
|
179
180
|
return torrent_id
|
|
180
181
|
|
|
181
|
-
def bulk_query(self, requests_by_eid, timeout: Optional[int] = None) -> Mapping[str, Mapping[str, TransferStatusReport]]:
|
|
182
|
+
def bulk_query(self, requests_by_eid, timeout: Optional[int] = None) -> 'Mapping[str, Mapping[str, TransferStatusReport]]':
|
|
182
183
|
response = {}
|
|
183
184
|
for transfer_id, requests in requests_by_eid.items():
|
|
184
185
|
for request_id, request in requests.items():
|
|
@@ -189,10 +190,10 @@ class BittorrentTransfertool(Transfertool):
|
|
|
189
190
|
response.setdefault(transfer_id, {})[request_id] = driver.get_status(request_id=request_id, torrent_id=transfer_id)
|
|
190
191
|
return response
|
|
191
192
|
|
|
192
|
-
def query(self, transfer_ids: Sequence[str], details: bool = False, timeout: Optional[int] = None) -> None:
|
|
193
|
+
def query(self, transfer_ids: 'Sequence[str]', details: bool = False, timeout: Optional[int] = None) -> None:
|
|
193
194
|
pass
|
|
194
195
|
|
|
195
|
-
def cancel(self, transfer_ids: Sequence[str], timeout: Optional[int] = None) -> None:
|
|
196
|
+
def cancel(self, transfer_ids: 'Sequence[str]', timeout: Optional[int] = None) -> None:
|
|
196
197
|
pass
|
|
197
198
|
|
|
198
199
|
def update_priority(self, transfer_id: str, priority: int, timeout: Optional[int] = None) -> None:
|
|
@@ -14,14 +14,12 @@
|
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
16
|
from abc import ABCMeta, abstractmethod
|
|
17
|
-
from
|
|
18
|
-
from typing import TYPE_CHECKING
|
|
19
|
-
|
|
20
|
-
from rucio.common import types
|
|
17
|
+
from typing import TYPE_CHECKING, Optional
|
|
21
18
|
|
|
22
19
|
if TYPE_CHECKING:
|
|
23
|
-
from
|
|
20
|
+
from collections.abc import Sequence
|
|
24
21
|
|
|
22
|
+
from rucio.common import types
|
|
25
23
|
from rucio.core.rse import RseData
|
|
26
24
|
from rucio.transfertool.transfertool import TransferStatusReport
|
|
27
25
|
|
|
@@ -32,7 +30,7 @@ class BittorrentDriver(metaclass=ABCMeta):
|
|
|
32
30
|
|
|
33
31
|
@classmethod
|
|
34
32
|
@abstractmethod
|
|
35
|
-
def make_driver(cls: "type[BittorrentDriver]", rse: "RseData", logger: types.LoggerFunction = logging.log) -> "Optional[BittorrentDriver]":
|
|
33
|
+
def make_driver(cls: "type[BittorrentDriver]", rse: "RseData", logger: 'types.LoggerFunction' = logging.log) -> "Optional[BittorrentDriver]":
|
|
36
34
|
pass
|
|
37
35
|
|
|
38
36
|
@abstractmethod
|
|
@@ -44,7 +42,7 @@ class BittorrentDriver(metaclass=ABCMeta):
|
|
|
44
42
|
pass
|
|
45
43
|
|
|
46
44
|
@abstractmethod
|
|
47
|
-
def add_peers(self, torrent_id: str, peers: Sequence[tuple[str, int]]) -> None:
|
|
45
|
+
def add_peers(self, torrent_id: str, peers: 'Sequence[tuple[str, int]]') -> None:
|
|
48
46
|
pass
|
|
49
47
|
|
|
50
48
|
@abstractmethod
|
|
@@ -13,13 +13,11 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
|
-
from collections.abc import Sequence
|
|
17
16
|
from typing import TYPE_CHECKING, Optional, cast
|
|
18
17
|
from urllib.parse import urlparse
|
|
19
18
|
|
|
20
19
|
import qbittorrentapi
|
|
21
20
|
|
|
22
|
-
from rucio.common import types
|
|
23
21
|
from rucio.common.config import get_rse_credentials
|
|
24
22
|
from rucio.common.constants import RseAttr
|
|
25
23
|
from rucio.common.utils import resolve_ip
|
|
@@ -30,8 +28,11 @@ from rucio.transfertool.transfertool import TransferStatusReport
|
|
|
30
28
|
from .bittorrent_driver import BittorrentDriver
|
|
31
29
|
|
|
32
30
|
if TYPE_CHECKING:
|
|
31
|
+
from collections.abc import Sequence
|
|
32
|
+
|
|
33
33
|
from sqlalchemy.orm import Session
|
|
34
34
|
|
|
35
|
+
from rucio.common import types
|
|
35
36
|
from rucio.core.rse import RseData
|
|
36
37
|
|
|
37
38
|
|
|
@@ -55,10 +56,10 @@ class QBittorrentTransferStatusReport(TransferStatusReport):
|
|
|
55
56
|
if new_state in [RequestState.FAILED, RequestState.DONE]:
|
|
56
57
|
self.external_id = external_id
|
|
57
58
|
|
|
58
|
-
def initialize(self, session: "Session", logger: types.LoggerFunction = logging.log) -> None:
|
|
59
|
+
def initialize(self, session: "Session", logger: 'types.LoggerFunction' = logging.log) -> None:
|
|
59
60
|
pass
|
|
60
61
|
|
|
61
|
-
def get_monitor_msg_fields(self, session: "Session", logger: types.LoggerFunction = logging.log) -> dict[str, str]:
|
|
62
|
+
def get_monitor_msg_fields(self, session: "Session", logger: 'types.LoggerFunction' = logging.log) -> dict[str, str]:
|
|
62
63
|
return {'protocol': 'qbittorrent'}
|
|
63
64
|
|
|
64
65
|
|
|
@@ -68,7 +69,7 @@ class QBittorrentDriver(BittorrentDriver):
|
|
|
68
69
|
required_rse_attrs = (RseAttr.QBITTORRENT_MANAGEMENT_ADDRESS, )
|
|
69
70
|
|
|
70
71
|
@classmethod
|
|
71
|
-
def make_driver(cls: "type[QBittorrentDriver]", rse: "RseData", logger: types.LoggerFunction = logging.log) -> "Optional[BittorrentDriver]":
|
|
72
|
+
def make_driver(cls: "type[QBittorrentDriver]", rse: "RseData", logger: 'types.LoggerFunction' = logging.log) -> "Optional[BittorrentDriver]":
|
|
72
73
|
|
|
73
74
|
address = rse.attributes.get(RseAttr.QBITTORRENT_MANAGEMENT_ADDRESS)
|
|
74
75
|
if not address:
|
|
@@ -96,7 +97,7 @@ class QBittorrentDriver(BittorrentDriver):
|
|
|
96
97
|
logger=logger,
|
|
97
98
|
)
|
|
98
99
|
|
|
99
|
-
def __init__(self, address: str, username: str, password: str, token: Optional[str] = None, logger: types.LoggerFunction = logging.log) -> None:
|
|
100
|
+
def __init__(self, address: str, username: str, password: str, token: Optional[str] = None, logger: 'types.LoggerFunction' = logging.log) -> None:
|
|
100
101
|
extra_headers = None
|
|
101
102
|
if token:
|
|
102
103
|
extra_headers = {'Authorization': 'Bearer ' + token}
|
|
@@ -112,7 +113,7 @@ class QBittorrentDriver(BittorrentDriver):
|
|
|
112
113
|
|
|
113
114
|
def listen_addr(self) -> tuple[str, int]:
|
|
114
115
|
preferences = self.client.app_preferences()
|
|
115
|
-
port = cast(int, preferences['listen_port'])
|
|
116
|
+
port = cast('int', preferences['listen_port'])
|
|
116
117
|
ip = resolve_ip(urlparse(self.client.host).hostname or self.client.host)
|
|
117
118
|
return ip, port
|
|
118
119
|
|
|
@@ -125,7 +126,7 @@ class QBittorrentDriver(BittorrentDriver):
|
|
|
125
126
|
is_sequential_download=True,
|
|
126
127
|
)
|
|
127
128
|
|
|
128
|
-
def add_peers(self, torrent_id: str, peers: Sequence[tuple[str, int]]) -> None:
|
|
129
|
+
def add_peers(self, torrent_id: str, peers: 'Sequence[tuple[str, int]]') -> None:
|
|
129
130
|
self.client.torrents_add_peers(torrent_hashes=[torrent_id], peers=[f'{ip}:{port}' for ip, port in peers])
|
|
130
131
|
|
|
131
132
|
def get_status(self, request_id: str, torrent_id: str) -> TransferStatusReport:
|