rucio-clients 34.3.0__tar.gz → 34.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio-clients might be problematic. Click here for more details.
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/PKG-INFO +1 -1
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/baseclient.py +2 -2
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/downloadclient.py +56 -7
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/replicaclient.py +13 -1
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/pcache.py +3 -3
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/types.py +14 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/utils.py +3 -3
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/posix.py +1 -1
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/protocol.py +0 -22
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/srm.py +2 -2
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/storm.py +1 -1
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/webdav.py +10 -6
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/vcsversion.py +3 -3
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio_clients.egg-info/SOURCES.txt +0 -3
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/pyproject.toml +31 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_replica.py +1 -1
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_replica_recoverer.py +73 -0
- rucio_clients-34.3.0/lib/rucio/common/schema/cms.py +0 -477
- rucio_clients-34.3.0/lib/rucio/common/schema/lsst.py +0 -422
- rucio_clients-34.3.0/tests/test_schema_cms.py +0 -229
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/AUTHORS.rst +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/ChangeLog +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/LICENSE +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/MANIFEST.in +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/README.rst +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/bin/rucio +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/bin/rucio-admin +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/etc/rse-accounts.cfg.template +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/etc/rucio.cfg.template +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/alembicrevision.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/accountclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/accountlimitclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/client.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/configclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/credentialclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/didclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/diracclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/exportclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/fileclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/importclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/lifetimeclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/lockclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/metaconventionsclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/pingclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/requestclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/rseclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/ruleclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/scopeclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/subscriptionclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/touchclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/client/uploadclient.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/cache.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/config.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/constants.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/constraints.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/didtype.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/exception.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/extra.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/logging.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/plugins.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/policy.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/atlas.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/belleii.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/domatpc.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/escape.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/generic.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/generic_multi_vo.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/schema/icecube.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/stomp_utils.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/stopwatch.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/common/test_rucio_server.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/__init__.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/bittorrent.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/cache.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/dummy.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/gfal.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/globus.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/gsiftp.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/http_cache.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/mock.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/ngarc.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/rclone.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/rfio.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/ssh.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/protocols/xrootd.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/rse/rsemanager.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/lib/rucio/version.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/pylintrc +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/requirements.txt +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/setup.cfg +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/setup.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/setuputil.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_abacus_account.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_abacus_collection_replica.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_abacus_rse.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_account.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_account_limits.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_api_external_representation.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_archive.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_auditor.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_auditor_hdfs.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_auditor_srmdumps.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_authentication.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_automatix.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_bad_replica.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_bb8.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_belleii.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_bin_rucio.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_boolean.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_clients.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_common_types.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_config.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_conveyor.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_conveyor_submitter.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_counter.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_credential.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_curl.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_daemons.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_dataset_replicas.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_db.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_did.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_did_meta_plugins.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_didtype.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_download.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_dumper.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_dumper_consistency.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_dumper_data_model.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_dumper_path_parsing.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_filter_engine.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_heartbeat.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_hermes.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_identity.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_impl_upload_download.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_import_export.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_judge_cleaner.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_judge_evaluator.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_judge_injector.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_judge_repairer.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_lifetime.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_message.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_meta_conventions.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_meta_did.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_module_import.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_monitor.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_multi_vo.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_naming_convention.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_oauthmanager.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_oidc.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_permission.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_pfns.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_ping.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_preparer.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_qos.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_quarantined_replica.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_reaper.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_redirect.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_replica_sorting.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_request.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_root_proxy.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_expression_parser.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_lfn2path.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_gfal2.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_gfal2_impl.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_posix.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_rclone.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_rsync.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_srm.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_ssh.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_webdav.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_protocol_xrootd.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rse_selector.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rucio_server.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_rule.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_scope.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_subscription.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_throttler.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_tpc.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_trace.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_transfer.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_transfer_plugins.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_undertaker.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_upload.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tests/test_utils.py +0 -0
- {rucio_clients-34.3.0 → rucio_clients-34.4.0}/tools/merge_rucio_configs.py +0 -0
|
@@ -77,8 +77,8 @@ class BaseClient:
|
|
|
77
77
|
|
|
78
78
|
AUTH_RETRIES, REQUEST_RETRIES = 2, 3
|
|
79
79
|
TOKEN_PATH_PREFIX = get_tmp_dir() + '/.rucio_'
|
|
80
|
-
TOKEN_PREFIX = 'auth_token_'
|
|
81
|
-
TOKEN_EXP_PREFIX = 'auth_token_exp_'
|
|
80
|
+
TOKEN_PREFIX = 'auth_token_' # noqa: S105
|
|
81
|
+
TOKEN_EXP_PREFIX = 'auth_token_exp_' # noqa: S105
|
|
82
82
|
|
|
83
83
|
def __init__(self,
|
|
84
84
|
rucio_host: Optional[str] = None,
|
|
@@ -25,6 +25,7 @@ import subprocess
|
|
|
25
25
|
import time
|
|
26
26
|
from queue import Empty, Queue, deque
|
|
27
27
|
from threading import Thread
|
|
28
|
+
from typing import Any, Optional
|
|
28
29
|
|
|
29
30
|
from rucio import version
|
|
30
31
|
from rucio.client.client import Client
|
|
@@ -174,7 +175,14 @@ class DownloadClient:
|
|
|
174
175
|
self.extraction_tools.append(BaseExtractionTool('tar', '--version', extract_args, logger=self.logger))
|
|
175
176
|
self.extract_scope_convention = config_get('common', 'extract_scope', False, None)
|
|
176
177
|
|
|
177
|
-
def download_pfns(
|
|
178
|
+
def download_pfns(
|
|
179
|
+
self,
|
|
180
|
+
items: list[dict[str, Any]],
|
|
181
|
+
num_threads: int = 2,
|
|
182
|
+
trace_custom_fields: Optional[dict[str, Any]] = None,
|
|
183
|
+
traces_copy_out: Optional[list[dict[str, Any]]] = None,
|
|
184
|
+
deactivate_file_download_exceptions: bool = False
|
|
185
|
+
) -> list[dict[str, Any]]:
|
|
178
186
|
"""
|
|
179
187
|
Download items with a given PFN. This function can only download files, no datasets.
|
|
180
188
|
|
|
@@ -202,6 +210,7 @@ class DownloadClient:
|
|
|
202
210
|
:raises NotAllFilesDownloaded: if not all files could be downloaded
|
|
203
211
|
:raises RucioException: if something unexpected went wrong during the download
|
|
204
212
|
"""
|
|
213
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
205
214
|
logger = self.logger
|
|
206
215
|
trace_custom_fields['uuid'] = generate_uuid()
|
|
207
216
|
|
|
@@ -250,8 +259,15 @@ class DownloadClient:
|
|
|
250
259
|
|
|
251
260
|
return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
|
|
252
261
|
|
|
253
|
-
def download_dids(
|
|
254
|
-
|
|
262
|
+
def download_dids(
|
|
263
|
+
self,
|
|
264
|
+
items: list[dict[str, Any]],
|
|
265
|
+
num_threads: int = 2,
|
|
266
|
+
trace_custom_fields: Optional[dict[str, Any]] = None,
|
|
267
|
+
traces_copy_out: Optional[list[dict[str, Any]]] = None,
|
|
268
|
+
deactivate_file_download_exceptions: bool = False,
|
|
269
|
+
sort: Optional[str] = None
|
|
270
|
+
) -> list[dict[str, Any]]:
|
|
255
271
|
"""
|
|
256
272
|
Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
|
|
257
273
|
|
|
@@ -286,6 +302,7 @@ class DownloadClient:
|
|
|
286
302
|
:raises NotAllFilesDownloaded: if not all files could be downloaded
|
|
287
303
|
:raises RucioException: if something unexpected went wrong during the download
|
|
288
304
|
"""
|
|
305
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
289
306
|
logger = self.logger
|
|
290
307
|
trace_custom_fields['uuid'] = generate_uuid()
|
|
291
308
|
|
|
@@ -304,7 +321,15 @@ class DownloadClient:
|
|
|
304
321
|
|
|
305
322
|
return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
|
|
306
323
|
|
|
307
|
-
def download_from_metalink_file(
|
|
324
|
+
def download_from_metalink_file(
|
|
325
|
+
self,
|
|
326
|
+
item: dict[str, Any],
|
|
327
|
+
metalink_file_path: str,
|
|
328
|
+
num_threads: int = 2,
|
|
329
|
+
trace_custom_fields: Optional[dict[str, Any]] = None,
|
|
330
|
+
traces_copy_out: Optional[list[dict[str, Any]]] = None,
|
|
331
|
+
deactivate_file_download_exceptions: bool = False
|
|
332
|
+
) -> list[dict[str, Any]]:
|
|
308
333
|
"""
|
|
309
334
|
Download items using a given metalink file.
|
|
310
335
|
|
|
@@ -327,6 +352,7 @@ class DownloadClient:
|
|
|
327
352
|
:raises NotAllFilesDownloaded: if not all files could be downloaded
|
|
328
353
|
:raises RucioException: if something unexpected went wrong during the download
|
|
329
354
|
"""
|
|
355
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
330
356
|
logger = self.logger
|
|
331
357
|
|
|
332
358
|
logger(logging.INFO, 'Getting sources from metalink file')
|
|
@@ -351,7 +377,13 @@ class DownloadClient:
|
|
|
351
377
|
|
|
352
378
|
return self._check_output(output_items, deactivate_file_download_exceptions=deactivate_file_download_exceptions)
|
|
353
379
|
|
|
354
|
-
def _download_multithreaded(
|
|
380
|
+
def _download_multithreaded(
|
|
381
|
+
self,
|
|
382
|
+
input_items: list[dict[str, Any]],
|
|
383
|
+
num_threads: int,
|
|
384
|
+
trace_custom_fields: Optional[dict[str, Any]] = None,
|
|
385
|
+
traces_copy_out: Optional[list[dict[str, Any]]] = None
|
|
386
|
+
) -> list[dict[str, Any]]:
|
|
355
387
|
"""
|
|
356
388
|
Starts an appropriate number of threads to download items from the input list.
|
|
357
389
|
(This function is meant to be used as class internal only)
|
|
@@ -363,6 +395,7 @@ class DownloadClient:
|
|
|
363
395
|
|
|
364
396
|
:returns: list with output items as dictionaries
|
|
365
397
|
"""
|
|
398
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
366
399
|
logger = self.logger
|
|
367
400
|
|
|
368
401
|
num_files = len(input_items)
|
|
@@ -730,7 +763,14 @@ class DownloadClient:
|
|
|
730
763
|
|
|
731
764
|
return item
|
|
732
765
|
|
|
733
|
-
def download_aria2c(
|
|
766
|
+
def download_aria2c(
|
|
767
|
+
self,
|
|
768
|
+
items: list[dict[str, Any]],
|
|
769
|
+
trace_custom_fields: Optional[dict[str, Any]] = None,
|
|
770
|
+
filters: Optional[dict[str, Any]] = None,
|
|
771
|
+
deactivate_file_download_exceptions: bool = False,
|
|
772
|
+
sort: Optional[str] = None
|
|
773
|
+
) -> list[dict[str, Any]]:
|
|
734
774
|
"""
|
|
735
775
|
Uses aria2c to download the items with given DIDs. This function can also download datasets and wildcarded DIDs.
|
|
736
776
|
It only can download files that are available via https/davs.
|
|
@@ -760,6 +800,8 @@ class DownloadClient:
|
|
|
760
800
|
:raises NotAllFilesDownloaded: if not all files could be downloaded
|
|
761
801
|
:raises RucioException: if something went wrong during the download (e.g. aria2c could not be started)
|
|
762
802
|
"""
|
|
803
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
804
|
+
filters = filters or {}
|
|
763
805
|
logger = self.logger
|
|
764
806
|
trace_custom_fields['uuid'] = generate_uuid()
|
|
765
807
|
|
|
@@ -860,7 +902,13 @@ class DownloadClient:
|
|
|
860
902
|
raise RucioException('Failed to initialise rpc proxy!', error)
|
|
861
903
|
return (rpcproc, aria_rpc)
|
|
862
904
|
|
|
863
|
-
def _download_items_aria2c(
|
|
905
|
+
def _download_items_aria2c(
|
|
906
|
+
self,
|
|
907
|
+
items: list[dict[str, Any]],
|
|
908
|
+
aria_rpc: Any,
|
|
909
|
+
rpc_auth: str,
|
|
910
|
+
trace_custom_fields: Optional[dict[str, Any]] = None
|
|
911
|
+
) -> list[dict[str, Any]]:
|
|
864
912
|
"""
|
|
865
913
|
Uses aria2c to download the given items. Aria2c needs to be started
|
|
866
914
|
as RPC background process first and a RPC proxy is needed.
|
|
@@ -873,6 +921,7 @@ class DownloadClient:
|
|
|
873
921
|
|
|
874
922
|
:returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
|
|
875
923
|
"""
|
|
924
|
+
trace_custom_fields = trace_custom_fields or {}
|
|
876
925
|
logger = self.logger
|
|
877
926
|
|
|
878
927
|
gid_to_item = {} # maps an aria2c download id (gid) to the download item
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
from datetime import datetime
|
|
16
16
|
from json import dumps, loads
|
|
17
|
+
from typing import Any, Optional
|
|
17
18
|
from urllib.parse import quote_plus
|
|
18
19
|
|
|
19
20
|
from requests.status_codes import codes
|
|
@@ -233,7 +234,17 @@ class ReplicaClient(BaseClient):
|
|
|
233
234
|
exc_cls, exc_msg = self._get_exception(headers=r.headers, status_code=r.status_code, data=r.content)
|
|
234
235
|
raise exc_cls(exc_msg)
|
|
235
236
|
|
|
236
|
-
def add_replica(
|
|
237
|
+
def add_replica(
|
|
238
|
+
self,
|
|
239
|
+
rse: str,
|
|
240
|
+
scope: str,
|
|
241
|
+
name: str,
|
|
242
|
+
bytes_: int,
|
|
243
|
+
adler32: str,
|
|
244
|
+
pfn: Optional[str] = None,
|
|
245
|
+
md5: Optional[str] = None,
|
|
246
|
+
meta: Optional[dict[str, Any]] = None
|
|
247
|
+
) -> bool:
|
|
237
248
|
"""
|
|
238
249
|
Add file replicas to a RSE.
|
|
239
250
|
|
|
@@ -249,6 +260,7 @@ class ReplicaClient(BaseClient):
|
|
|
249
260
|
:return: True if files were created successfully.
|
|
250
261
|
|
|
251
262
|
"""
|
|
263
|
+
meta = meta or {}
|
|
252
264
|
dict_ = {'scope': scope, 'name': name, 'bytes': bytes_, 'meta': meta, 'adler32': adler32}
|
|
253
265
|
if md5:
|
|
254
266
|
dict_['md5'] = md5
|
|
@@ -529,7 +529,7 @@ class Pcache:
|
|
|
529
529
|
|
|
530
530
|
# Execute original command, no further action
|
|
531
531
|
if (not (self.dst.startswith(self.scratch_dir) and self.accept(self.src) and (not self.reject(self.src)))):
|
|
532
|
-
os.execvp(self.copy_util, self.args)
|
|
532
|
+
os.execvp(self.copy_util, self.args) # noqa: S606
|
|
533
533
|
os._exit(1)
|
|
534
534
|
|
|
535
535
|
# XXXX todo: fast-path - try to acquire lock
|
|
@@ -682,7 +682,7 @@ class Pcache:
|
|
|
682
682
|
self.fail(103)
|
|
683
683
|
|
|
684
684
|
def get_disk_usage(self):
|
|
685
|
-
p = os.popen("df -P %s | tail -1" % self.pcache_dir, 'r')
|
|
685
|
+
p = os.popen("df -P %s | tail -1" % self.pcache_dir, 'r') # noqa: S605
|
|
686
686
|
data = p.read()
|
|
687
687
|
status = p.close()
|
|
688
688
|
if status:
|
|
@@ -772,7 +772,7 @@ class Pcache:
|
|
|
772
772
|
d = self.pcache_dir + d
|
|
773
773
|
try:
|
|
774
774
|
os.rename(d, d + ts)
|
|
775
|
-
os.system("rm -rf %s &" % (d + ts))
|
|
775
|
+
os.system("rm -rf %s &" % (d + ts)) # noqa: S605
|
|
776
776
|
except OSError as e:
|
|
777
777
|
if e.errno != errno.ENOENT:
|
|
778
778
|
self.log(ERROR, "%s: %s", d, e)
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
from collections.abc import Callable
|
|
16
|
+
from datetime import datetime
|
|
16
17
|
from typing import TYPE_CHECKING, Any, Literal, Optional, TypedDict, Union
|
|
17
18
|
|
|
18
19
|
if TYPE_CHECKING:
|
|
@@ -191,3 +192,16 @@ class HopDict(TypedDict):
|
|
|
191
192
|
dest_rse_id: str
|
|
192
193
|
dest_scheme: "SUPPORTED_PROTOCOLS_LITERAL"
|
|
193
194
|
dest_scheme_priority: int
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class TokenDict(TypedDict):
|
|
198
|
+
token: str
|
|
199
|
+
expires_at: datetime
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class TokenValidationDict(TypedDict):
|
|
203
|
+
account: Optional[InternalAccount]
|
|
204
|
+
identity: Optional[str]
|
|
205
|
+
lifetime: datetime
|
|
206
|
+
audience: Optional[str]
|
|
207
|
+
authz_scope: Optional[str]
|
|
@@ -1245,7 +1245,7 @@ def detect_client_location():
|
|
|
1245
1245
|
pass
|
|
1246
1246
|
|
|
1247
1247
|
if not ip:
|
|
1248
|
-
ip = '0.0.0.0'
|
|
1248
|
+
ip = '0.0.0.0' # noqa: S104
|
|
1249
1249
|
|
|
1250
1250
|
site = os.environ.get('SITE_NAME',
|
|
1251
1251
|
os.environ.get('ATLAS_SITE_NAME',
|
|
@@ -1559,7 +1559,7 @@ def parse_replicas_from_file(path):
|
|
|
1559
1559
|
"""
|
|
1560
1560
|
with open(path) as fp:
|
|
1561
1561
|
try:
|
|
1562
|
-
root = ElementTree.parse(fp).getroot()
|
|
1562
|
+
root = ElementTree.parse(fp).getroot() # noqa: S314
|
|
1563
1563
|
return parse_replicas_metalink(root)
|
|
1564
1564
|
except ElementTree.ParseError as xml_err:
|
|
1565
1565
|
try:
|
|
@@ -1579,7 +1579,7 @@ def parse_replicas_from_string(string):
|
|
|
1579
1579
|
:returns: a list with a dictionary for each file
|
|
1580
1580
|
"""
|
|
1581
1581
|
try:
|
|
1582
|
-
root = ElementTree.fromstring(string)
|
|
1582
|
+
root = ElementTree.fromstring(string) # noqa: S314
|
|
1583
1583
|
return parse_replicas_metalink(root)
|
|
1584
1584
|
except ElementTree.ParseError as xml_err:
|
|
1585
1585
|
try:
|
|
@@ -74,7 +74,7 @@ class Default(protocol.RSEProtocol):
|
|
|
74
74
|
try: # To check if the error happened local or remote
|
|
75
75
|
with open(dest, 'wb'):
|
|
76
76
|
pass
|
|
77
|
-
call(['rm', '-rf', dest])
|
|
77
|
+
call(['rm', '-rf', dest]) # noqa: S607
|
|
78
78
|
except OSError as e:
|
|
79
79
|
if e.errno == 2:
|
|
80
80
|
raise exception.DestinationNotAccessible(e)
|
|
@@ -199,27 +199,6 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
|
199
199
|
|
|
200
200
|
return '%s/%s/%s/%s' % (scope[0:7], scope[4:len(scope)], name.split('-')[0] + "-" + name.split('-')[1], name)
|
|
201
201
|
|
|
202
|
-
@staticmethod
|
|
203
|
-
def __lsst(scope, name, rse, rse_attrs, protocol_attrs):
|
|
204
|
-
"""
|
|
205
|
-
LFN2PFN algorithm for Rubin-LSST in the ESCAPE project
|
|
206
|
-
|
|
207
|
-
Replace convention delimiter '__' by '/'
|
|
208
|
-
The Escape instance does use the 'generic' Rucio schema.
|
|
209
|
-
|
|
210
|
-
:param scope: Scope of the LFN (ignored)
|
|
211
|
-
:param name: File name of the LFN.
|
|
212
|
-
:param rse: RSE for PFN (ignored)
|
|
213
|
-
:param rse_attrs: RSE attributes for PFN (ignored)
|
|
214
|
-
:param protocol_attrs: RSE protocol attributes for PFN (ignored)
|
|
215
|
-
:returns: Path for use in the PFN generation.
|
|
216
|
-
"""
|
|
217
|
-
del scope
|
|
218
|
-
del rse
|
|
219
|
-
del rse_attrs
|
|
220
|
-
del protocol_attrs
|
|
221
|
-
return name.replace('__', '/')
|
|
222
|
-
|
|
223
202
|
@classmethod
|
|
224
203
|
def _module_init_(cls):
|
|
225
204
|
"""
|
|
@@ -230,7 +209,6 @@ class RSEDeterministicTranslation(PolicyPackageAlgorithms):
|
|
|
230
209
|
cls.register(cls.__ligo, "ligo")
|
|
231
210
|
cls.register(cls.__belleii, "belleii")
|
|
232
211
|
cls.register(cls.__xenon, "xenon")
|
|
233
|
-
cls.register(cls.__lsst, "lsst")
|
|
234
212
|
policy_module = None
|
|
235
213
|
try:
|
|
236
214
|
policy_module = config.config_get('policy', 'lfn2pfn_module')
|
|
@@ -173,11 +173,11 @@ class Default(protocol.RSEProtocol):
|
|
|
173
173
|
:raises RSEAccessDenied: Cannot connect.
|
|
174
174
|
"""
|
|
175
175
|
|
|
176
|
-
status, lcglscommand = getstatusoutput('which lcg-ls')
|
|
176
|
+
status, lcglscommand = getstatusoutput('which lcg-ls') # noqa: S605, S607
|
|
177
177
|
if status:
|
|
178
178
|
raise exception.RSEAccessDenied('Cannot find lcg tools')
|
|
179
179
|
endpoint_basepath = self.path2pfn(self.attributes['prefix'])
|
|
180
|
-
status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath))
|
|
180
|
+
status, result = getstatusoutput('%s -vv $LCGVO -b --srm-timeout 60 -D srmv2 -l %s' % (lcglscommand, endpoint_basepath)) # noqa: S605
|
|
181
181
|
if status:
|
|
182
182
|
if result == '':
|
|
183
183
|
raise exception.RSEAccessDenied('Endpoint not reachable. lcg-ls failed with status code %s but no further details.' % (str(status)))
|
|
@@ -155,7 +155,7 @@ class Default(protocol.RSEProtocol):
|
|
|
155
155
|
if rcode != 207:
|
|
156
156
|
rcode, etag_meta = davix_etag(pfn, 300)
|
|
157
157
|
|
|
158
|
-
p_output = minidom.parseString(etag_meta)
|
|
158
|
+
p_output = minidom.parseString(etag_meta) # noqa: S318
|
|
159
159
|
# we need to strip off the quotation marks and the <timestamp> from the etag
|
|
160
160
|
# but since we can have multiple underscores, we have to rely on the uniqueness
|
|
161
161
|
# of the full LFN to make the split
|
|
@@ -16,7 +16,7 @@ import os
|
|
|
16
16
|
import sys
|
|
17
17
|
import xml.etree.ElementTree as ET
|
|
18
18
|
from dataclasses import dataclass
|
|
19
|
-
from typing import Optional
|
|
19
|
+
from typing import Any, Optional
|
|
20
20
|
from urllib.parse import urlparse
|
|
21
21
|
|
|
22
22
|
import requests
|
|
@@ -133,7 +133,7 @@ class _PropfindResponse:
|
|
|
133
133
|
"""
|
|
134
134
|
|
|
135
135
|
try:
|
|
136
|
-
xml = ET.fromstring(document)
|
|
136
|
+
xml = ET.fromstring(document) # noqa: S314
|
|
137
137
|
except ET.ParseError as ex:
|
|
138
138
|
raise ValueError("Couldn't parse XML document") from ex
|
|
139
139
|
|
|
@@ -151,7 +151,7 @@ class Default(protocol.RSEProtocol):
|
|
|
151
151
|
|
|
152
152
|
""" Implementing access to RSEs using the webDAV protocol."""
|
|
153
153
|
|
|
154
|
-
def connect(self, credentials=
|
|
154
|
+
def connect(self, credentials: Optional[dict[str, Any]] = None) -> None:
|
|
155
155
|
""" Establishes the actual connection to the referred RSE.
|
|
156
156
|
|
|
157
157
|
:param credentials: Provides information to establish a connection
|
|
@@ -160,6 +160,7 @@ class Default(protocol.RSEProtocol):
|
|
|
160
160
|
|
|
161
161
|
:raises RSEAccessDenied
|
|
162
162
|
"""
|
|
163
|
+
credentials = credentials or {}
|
|
163
164
|
try:
|
|
164
165
|
parse_url = urlparse(self.path2pfn(''))
|
|
165
166
|
self.server = f'{parse_url.scheme}://{parse_url.netloc}'
|
|
@@ -184,12 +185,15 @@ class Default(protocol.RSEProtocol):
|
|
|
184
185
|
# Trying to get the proxy from the default location
|
|
185
186
|
proxy_path = '/tmp/x509up_u%s' % os.geteuid()
|
|
186
187
|
if os.path.isfile(proxy_path):
|
|
187
|
-
|
|
188
|
+
self.cert = (proxy_path, proxy_path)
|
|
188
189
|
elif self.auth_token:
|
|
190
|
+
# If no proxy is found, we set the cert to None and use the auth_token
|
|
191
|
+
self.cert = None
|
|
189
192
|
pass
|
|
190
193
|
else:
|
|
191
194
|
raise exception.RSEAccessDenied('X509_USER_PROXY is not set')
|
|
192
|
-
|
|
195
|
+
else:
|
|
196
|
+
self.cert = (x509, x509)
|
|
193
197
|
|
|
194
198
|
try:
|
|
195
199
|
self.timeout = credentials['timeout']
|
|
@@ -533,7 +537,7 @@ class Default(protocol.RSEProtocol):
|
|
|
533
537
|
headers = {'Depth': '0'}
|
|
534
538
|
|
|
535
539
|
try:
|
|
536
|
-
root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text)
|
|
540
|
+
root = ET.fromstring(self.session.request('PROPFIND', endpoint_basepath, verify=False, headers=headers, cert=self.session.cert).text) # noqa: S314
|
|
537
541
|
usedsize = root[0][1][0].find('{DAV:}quota-used-bytes').text
|
|
538
542
|
try:
|
|
539
543
|
unusedsize = root[0][1][0].find('{DAV:}quota-available-bytes').text
|
|
@@ -4,8 +4,8 @@ This file is automatically generated; Do not edit it. :)
|
|
|
4
4
|
'''
|
|
5
5
|
VERSION_INFO = {
|
|
6
6
|
'final': True,
|
|
7
|
-
'version': '34.
|
|
7
|
+
'version': '34.4.0',
|
|
8
8
|
'branch_nick': 'release-34',
|
|
9
|
-
'revision_id': '
|
|
10
|
-
'revno':
|
|
9
|
+
'revision_id': 'e9aff7ed5dc1ba4c1085faa212d61a875d1adbd1',
|
|
10
|
+
'revno': 12859
|
|
11
11
|
}
|
|
@@ -63,13 +63,11 @@ lib/rucio/common/utils.py
|
|
|
63
63
|
lib/rucio/common/schema/__init__.py
|
|
64
64
|
lib/rucio/common/schema/atlas.py
|
|
65
65
|
lib/rucio/common/schema/belleii.py
|
|
66
|
-
lib/rucio/common/schema/cms.py
|
|
67
66
|
lib/rucio/common/schema/domatpc.py
|
|
68
67
|
lib/rucio/common/schema/escape.py
|
|
69
68
|
lib/rucio/common/schema/generic.py
|
|
70
69
|
lib/rucio/common/schema/generic_multi_vo.py
|
|
71
70
|
lib/rucio/common/schema/icecube.py
|
|
72
|
-
lib/rucio/common/schema/lsst.py
|
|
73
71
|
lib/rucio/rse/__init__.py
|
|
74
72
|
lib/rucio/rse/rsemanager.py
|
|
75
73
|
lib/rucio/rse/protocols/__init__.py
|
|
@@ -175,7 +173,6 @@ tests/test_rse_protocol_xrootd.py
|
|
|
175
173
|
tests/test_rse_selector.py
|
|
176
174
|
tests/test_rucio_server.py
|
|
177
175
|
tests/test_rule.py
|
|
178
|
-
tests/test_schema_cms.py
|
|
179
176
|
tests/test_scope.py
|
|
180
177
|
tests/test_subscription.py
|
|
181
178
|
tests/test_throttler.py
|
|
@@ -42,6 +42,7 @@ exclude = [
|
|
|
42
42
|
[tool.ruff.lint]
|
|
43
43
|
select = [
|
|
44
44
|
"I", # isort
|
|
45
|
+
"S", # bandit
|
|
45
46
|
"UP", # pyupgrade
|
|
46
47
|
"TID", # flake8-tidy-imports
|
|
47
48
|
]
|
|
@@ -55,6 +56,18 @@ ignore = [
|
|
|
55
56
|
"UP030", # Use implicit references for positional format fields
|
|
56
57
|
"UP031", # Use format specifiers instead of percent format
|
|
57
58
|
"UP032", # Use f-string instead of `format` call
|
|
59
|
+
"S101", # Pending https://github.com/rucio/rucio/issues/6680
|
|
60
|
+
"S105", # Pending https://github.com/rucio/rucio/issues/6696
|
|
61
|
+
"S108", # Pending https://github.com/rucio/rucio/issues/6655
|
|
62
|
+
"S110", # Pending https://github.com/rucio/rucio/issues/6657
|
|
63
|
+
"S112", # Pending https://github.com/rucio/rucio/issues/6657
|
|
64
|
+
"S113", # Pending https://github.com/rucio/rucio/issues/6654
|
|
65
|
+
"S310", # Pending https://github.com/astral-sh/ruff/issues/7918
|
|
66
|
+
"S324", # Pending https://github.com/rucio/rucio/issues/6665
|
|
67
|
+
"S501", # Pending https://github.com/rucio/rucio/issues/6656
|
|
68
|
+
"S602", # Pending https://github.com/astral-sh/ruff/issues/4045
|
|
69
|
+
"S603", # Pending https://github.com/astral-sh/ruff/issues/4045
|
|
70
|
+
"S608", # Pending https://github.com/rucio/rucio/issues/6669
|
|
58
71
|
"SIM210",
|
|
59
72
|
]
|
|
60
73
|
|
|
@@ -99,3 +112,21 @@ known-first-party = ["rucio"]
|
|
|
99
112
|
"typing.Tuple".msg = "Use built-in type `tuple` instead."
|
|
100
113
|
"typing.Type".msg = "Use built-in `type` instead."
|
|
101
114
|
"typing.ValuesView".msg = "Use `collections.abc.ValuesView` instead."
|
|
115
|
+
|
|
116
|
+
[tool.ruff.lint.per-file-ignores]
|
|
117
|
+
'tests/*.py' = [
|
|
118
|
+
'S101', # Usage of assert
|
|
119
|
+
'S105', # Hardcoded password string
|
|
120
|
+
'S106', # Hardcoded password function argument
|
|
121
|
+
'S108', # Hardcoded temporary file
|
|
122
|
+
'S110', # try-except-pass
|
|
123
|
+
'S113', # Probable use of requests call without timeout
|
|
124
|
+
'S306', # Use of insecure and deprecated function
|
|
125
|
+
'S311', # Non-cryptographic random usage
|
|
126
|
+
'S324', # Probable use of insecure hash function
|
|
127
|
+
'S605', # Starting a process with a shell
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
'lib/rucio/db/sqla/migrate_repo/versions/*.py' = [
|
|
131
|
+
'S608' # Hardcoded SQL expression
|
|
132
|
+
]
|
|
@@ -1209,7 +1209,7 @@ def test_client_list_replicas_streaming_error(content_type, vo, did_client, repl
|
|
|
1209
1209
|
assert metalink
|
|
1210
1210
|
print(metalink)
|
|
1211
1211
|
with pytest.raises(ElementTree.ParseError):
|
|
1212
|
-
ElementTree.fromstring(metalink)
|
|
1212
|
+
ElementTree.fromstring(metalink) # noqa: S314
|
|
1213
1213
|
|
|
1214
1214
|
elif content_type == Mime.JSON_STREAM:
|
|
1215
1215
|
# for the json stream mimetype the API method just returns all mocked replicas on error
|
|
@@ -520,3 +520,76 @@ class TestReplicaRecoverer:
|
|
|
520
520
|
assert (self.tmp_file11.name, self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
|
|
521
521
|
assert (self.tmp_file12.name, self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
|
|
522
522
|
assert (self.tmp_file13.name, self.rse4recovery_id, BadFilesStatus.BAD) not in bad_checklist
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
@pytest.mark.parametrize(
|
|
526
|
+
"file_config_mock", [{"overrides": [
|
|
527
|
+
("replicarecoverer", "rule_rse_expression", "type=SCRATCHDISK"),
|
|
528
|
+
('replicarecoverer', 'use_file_metadata', 'False'),
|
|
529
|
+
('replicarecoverer', 'did_name_expression', '^.*')]}],
|
|
530
|
+
indirect=True)
|
|
531
|
+
def test_vo_agnostic_rules(file_config_mock, replica_client, rse_factory, scope_factory, file_factory, vo):
|
|
532
|
+
"""
|
|
533
|
+
Runs only a single time, make sure it makes a new rule with this config.
|
|
534
|
+
|
|
535
|
+
Examining files without a datatype, but because the "use_file_metadata" is turned off,
|
|
536
|
+
files without a datatype can be acted on.
|
|
537
|
+
"""
|
|
538
|
+
rse4suspicious, rse4suspicious_id = rse_factory.make_posix_rse(deterministic=True, vo=vo)
|
|
539
|
+
rse4recovery, rse4recovery_id = rse_factory.make_posix_rse(deterministic=True, vo=vo)
|
|
540
|
+
|
|
541
|
+
add_rse_attribute(rse4suspicious_id, "enable_suspicious_file_recovery", True)
|
|
542
|
+
add_rse_attribute(rse4recovery_id, "enable_suspicious_file_recovery", True)
|
|
543
|
+
|
|
544
|
+
# Create new scopes
|
|
545
|
+
_, [scope_declarebad] = scope_factory(vos=[vo])
|
|
546
|
+
|
|
547
|
+
tmp_file_delare_bad = file_factory.file_generator()
|
|
548
|
+
|
|
549
|
+
replicas_without_types = [{'scope': scope_declarebad, 'name': tmp_file_delare_bad.name, 'type': DIDType.FILE}]
|
|
550
|
+
for rse in [rse4suspicious, rse4recovery]:
|
|
551
|
+
cmd = f'rucio -v upload --rse {rse} --scope {scope_declarebad} {tmp_file_delare_bad}'
|
|
552
|
+
exitcode, _, _ = execute(cmd)
|
|
553
|
+
assert exitcode == 0
|
|
554
|
+
|
|
555
|
+
remove(tmp_file_delare_bad)
|
|
556
|
+
rse_expression_parser.REGION.invalidate()
|
|
557
|
+
|
|
558
|
+
update_replica_state(rse4recovery_id, scope_declarebad, tmp_file_delare_bad.name, ReplicaState.UNAVAILABLE)
|
|
559
|
+
replical_declare_bad = list(list_replicas(dids=replicas_without_types))
|
|
560
|
+
|
|
561
|
+
for replica in replical_declare_bad:
|
|
562
|
+
suspicious_pfns = replica['rses'][rse4suspicious_id]
|
|
563
|
+
for _ in range(3):
|
|
564
|
+
# The reason must contain the word "checksum", so that the replica can be declared bad.
|
|
565
|
+
replica_client.declare_suspicious_file_replicas([suspicious_pfns[0], ], 'checksum')
|
|
566
|
+
sleep(1)
|
|
567
|
+
|
|
568
|
+
recovery_policy = [{"action": "declare bad", "datatype": [str(tmp_file_delare_bad.name)], "scope": [str(scope_declarebad)]}]
|
|
569
|
+
recovery_policy_json = "/opt/rucio/etc/test_replica_recoverer_vo_agonistism.json"
|
|
570
|
+
json.dump(recovery_policy, open(recovery_policy_json, mode="w"))
|
|
571
|
+
|
|
572
|
+
try:
|
|
573
|
+
run(
|
|
574
|
+
once=True,
|
|
575
|
+
younger_than=1,
|
|
576
|
+
nattempts=2,
|
|
577
|
+
limit_suspicious_files_on_rse=2,
|
|
578
|
+
json_file_name=recovery_policy_json,
|
|
579
|
+
sleep_time=0,
|
|
580
|
+
active_mode=True
|
|
581
|
+
)
|
|
582
|
+
except KeyboardInterrupt:
|
|
583
|
+
stop()
|
|
584
|
+
|
|
585
|
+
# dids without types now have a different behavior
|
|
586
|
+
|
|
587
|
+
for replica in replicas_without_types:
|
|
588
|
+
# Files that are now acted on now the datatype can be found.
|
|
589
|
+
if replica['name'] == tmp_file_delare_bad.name:
|
|
590
|
+
# tmp_file11 should no longer be ignored.
|
|
591
|
+
assert not replica.get('states')
|
|
592
|
+
|
|
593
|
+
bad_replicas_list = list_bad_replicas_status(rse_id=rse4suspicious_id, vo=vo)
|
|
594
|
+
bad_checklist = [(badf['name'], badf['rse_id'], badf['state']) for badf in bad_replicas_list]
|
|
595
|
+
assert (tmp_file_delare_bad.name, rse4suspicious_id, BadFilesStatus.BAD) in bad_checklist
|