rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -14,7 +13,7 @@
|
|
|
14
13
|
# limitations under the License.
|
|
15
14
|
|
|
16
15
|
"""
|
|
17
|
-
OAuth Manager is a daemon which is
|
|
16
|
+
OAuth Manager is a daemon which is responsible for:
|
|
18
17
|
- deletion of expired access tokens (in case there is a valid refresh token,
|
|
19
18
|
expired access tokens will be kept until refresh_token expires as well.)
|
|
20
19
|
- deletion of expired OAuth session parameters
|
|
@@ -42,8 +41,8 @@ from rucio.common.stopwatch import Stopwatch
|
|
|
42
41
|
from rucio.core.authentication import delete_expired_tokens
|
|
43
42
|
from rucio.core.monitor import MetricManager
|
|
44
43
|
from rucio.core.oidc import delete_expired_oauthrequests, refresh_jwt_tokens
|
|
45
|
-
from rucio.daemons.common import HeartbeatHandler
|
|
46
|
-
from rucio.
|
|
44
|
+
from rucio.daemons.common import HeartbeatHandler, run_daemon
|
|
45
|
+
from rucio.db.sqla.constants import ORACLE_CONNECTION_LOST_CONTACT_REGEX
|
|
47
46
|
|
|
48
47
|
if TYPE_CHECKING:
|
|
49
48
|
from types import FrameType
|
|
@@ -107,7 +106,7 @@ def run_once(heartbeat_handler: HeartbeatHandler, max_rows: int, sleep_time: int
|
|
|
107
106
|
if match('.*QueuePool.*', str(err.args[0])):
|
|
108
107
|
logger(logging.WARNING, traceback.format_exc())
|
|
109
108
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
110
|
-
elif match(
|
|
109
|
+
elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
|
|
111
110
|
logger(logging.WARNING, traceback.format_exc())
|
|
112
111
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
113
112
|
else:
|
|
@@ -115,7 +114,7 @@ def run_once(heartbeat_handler: HeartbeatHandler, max_rows: int, sleep_time: int
|
|
|
115
114
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
116
115
|
|
|
117
116
|
try:
|
|
118
|
-
# waiting 1 sec as DBs does not store
|
|
117
|
+
# waiting 1 sec as DBs does not store millisecond and tokens
|
|
119
118
|
# eligible for deletion after refresh might not get deleted otherwise
|
|
120
119
|
graceful_stop.wait(1)
|
|
121
120
|
|
|
@@ -134,7 +133,7 @@ def run_once(heartbeat_handler: HeartbeatHandler, max_rows: int, sleep_time: int
|
|
|
134
133
|
if match('.*QueuePool.*', str(err.args[0])):
|
|
135
134
|
logger(logging.WARNING, traceback.format_exc())
|
|
136
135
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
137
|
-
elif match(
|
|
136
|
+
elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
|
|
138
137
|
logger(logging.WARNING, traceback.format_exc())
|
|
139
138
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
140
139
|
else:
|
|
@@ -157,7 +156,7 @@ def run_once(heartbeat_handler: HeartbeatHandler, max_rows: int, sleep_time: int
|
|
|
157
156
|
if match('.*QueuePool.*', str(err.args[0])):
|
|
158
157
|
logger(logging.WARNING, traceback.format_exc())
|
|
159
158
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
160
|
-
elif match(
|
|
159
|
+
elif match(ORACLE_CONNECTION_LOST_CONTACT_REGEX, str(err.args[0])):
|
|
161
160
|
logger(logging.WARNING, traceback.format_exc())
|
|
162
161
|
METRICS.counter('exceptions.{exception}').labels(exception=err.__class__.__name__).inc()
|
|
163
162
|
else:
|
rucio/daemons/reaper/__init__.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -30,15 +29,11 @@ import rucio.core.rse as rse_core
|
|
|
30
29
|
import rucio.db.sqla.util
|
|
31
30
|
from rucio.common import exception
|
|
32
31
|
from rucio.common.config import config_get_bool
|
|
33
|
-
from rucio.common.exception import
|
|
34
|
-
RSEAccessDenied, ResourceTemporaryUnavailable,
|
|
35
|
-
RSENotFound, VONotFound)
|
|
32
|
+
from rucio.common.exception import ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, ServiceUnavailable, SourceNotFound, VONotFound
|
|
36
33
|
from rucio.common.logging import setup_logging
|
|
37
34
|
from rucio.core.message import add_message
|
|
38
35
|
from rucio.core.monitor import MetricManager
|
|
39
|
-
from rucio.core.quarantined_replica import
|
|
40
|
-
delete_quarantined_replicas,
|
|
41
|
-
list_rses_with_quarantined_replicas)
|
|
36
|
+
from rucio.core.quarantined_replica import delete_quarantined_replicas, list_quarantined_replicas, list_rses_with_quarantined_replicas
|
|
42
37
|
from rucio.core.rse_expression_parser import parse_expression
|
|
43
38
|
from rucio.core.vo import list_vos
|
|
44
39
|
from rucio.daemons.common import run_daemon
|
|
@@ -193,8 +188,18 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
|
|
|
193
188
|
GRACEFUL_STOP.set()
|
|
194
189
|
|
|
195
190
|
|
|
196
|
-
def run(
|
|
197
|
-
|
|
191
|
+
def run(
|
|
192
|
+
total_workers: int = 1,
|
|
193
|
+
chunk_size: int = 100,
|
|
194
|
+
once: bool = False,
|
|
195
|
+
rses: "Optional[list[str]]" = None,
|
|
196
|
+
scheme: "Optional[str]" = None,
|
|
197
|
+
exclude_rses: "Optional[str]" = None,
|
|
198
|
+
include_rses: "Optional[str]" = None,
|
|
199
|
+
vos: "Optional[list[str]]" = None,
|
|
200
|
+
delay_seconds: int = 0,
|
|
201
|
+
sleep_time: int = 300
|
|
202
|
+
) -> None:
|
|
198
203
|
"""
|
|
199
204
|
Starts up the reaper threads.
|
|
200
205
|
|
|
@@ -208,6 +213,7 @@ def run(total_workers=1, chunk_size=100, once=False, rses=[], scheme=None,
|
|
|
208
213
|
:param vos: VOs on which to look for RSEs. Only used in multi-VO mode.
|
|
209
214
|
If None, we either use all VOs if run from "def", or the current VO otherwise.
|
|
210
215
|
"""
|
|
216
|
+
rses = rses or []
|
|
211
217
|
setup_logging(process_name=DAEMON_NAME)
|
|
212
218
|
|
|
213
219
|
if rucio.db.sqla.util.is_old_db():
|
rucio/daemons/reaper/reaper.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -17,6 +16,8 @@
|
|
|
17
16
|
Reaper is a daemon to manage file deletion.
|
|
18
17
|
'''
|
|
19
18
|
|
|
19
|
+
import concurrent.futures.thread # noqa (https://github.com/rucio/rucio/issues/6548)
|
|
20
|
+
|
|
20
21
|
import functools
|
|
21
22
|
import logging
|
|
22
23
|
import random
|
|
@@ -25,30 +26,27 @@ import time
|
|
|
25
26
|
import traceback
|
|
26
27
|
from configparser import NoOptionError, NoSectionError
|
|
27
28
|
from datetime import datetime, timedelta
|
|
28
|
-
from
|
|
29
|
+
from math import log2
|
|
30
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
29
31
|
|
|
30
32
|
from dogpile.cache.api import NoValue
|
|
31
|
-
from math import log2
|
|
32
33
|
from sqlalchemy.exc import DatabaseError, IntegrityError
|
|
33
34
|
|
|
34
35
|
import rucio.db.sqla.util
|
|
35
36
|
from rucio.common.cache import make_region_memcached
|
|
36
|
-
from rucio.common.config import
|
|
37
|
-
from rucio.common.
|
|
38
|
-
|
|
39
|
-
RSEAccessDenied, ResourceTemporaryUnavailable, SourceNotFound,
|
|
40
|
-
VONotFound, RSEProtocolNotSupported)
|
|
37
|
+
from rucio.common.config import config_get_bool, config_get_int
|
|
38
|
+
from rucio.common.constants import RseAttr
|
|
39
|
+
from rucio.common.exception import DatabaseException, ReplicaNotFound, ReplicaUnAvailable, ResourceTemporaryUnavailable, RSEAccessDenied, RSENotFound, RSEProtocolNotSupported, ServiceUnavailable, SourceNotFound, VONotFound
|
|
41
40
|
from rucio.common.logging import setup_logging
|
|
42
41
|
from rucio.common.stopwatch import Stopwatch
|
|
43
|
-
from rucio.common.types import InternalAccount
|
|
44
42
|
from rucio.common.utils import chunks
|
|
45
43
|
from rucio.core.credential import get_signed_url
|
|
46
44
|
from rucio.core.heartbeat import list_payload_counts
|
|
47
45
|
from rucio.core.message import add_message
|
|
48
46
|
from rucio.core.monitor import MetricManager
|
|
49
|
-
from rucio.core.oidc import
|
|
50
|
-
from rucio.core.replica import
|
|
51
|
-
from rucio.core.rse import
|
|
47
|
+
from rucio.core.oidc import request_token
|
|
48
|
+
from rucio.core.replica import delete_replicas, list_and_mark_unlocked_replicas
|
|
49
|
+
from rucio.core.rse import RseData, determine_audience_for_rse, determine_scope_for_rse, list_rses
|
|
52
50
|
from rucio.core.rse_expression_parser import parse_expression
|
|
53
51
|
from rucio.core.rule import get_evaluation_backlog
|
|
54
52
|
from rucio.core.vo import list_vos
|
|
@@ -56,10 +54,10 @@ from rucio.daemons.common import run_daemon
|
|
|
56
54
|
from rucio.rse import rsemanager as rsemgr
|
|
57
55
|
|
|
58
56
|
if TYPE_CHECKING:
|
|
59
|
-
from collections.abc import
|
|
57
|
+
from collections.abc import Iterable, Sequence
|
|
60
58
|
from types import FrameType
|
|
61
|
-
from typing import Any, Optional
|
|
62
59
|
|
|
60
|
+
from rucio.common.types import LoggerFunction
|
|
63
61
|
from rucio.daemons.common import HeartbeatHandler
|
|
64
62
|
|
|
65
63
|
GRACEFUL_STOP = threading.Event()
|
|
@@ -70,7 +68,12 @@ DAEMON_NAME = 'reaper'
|
|
|
70
68
|
EXCLUDED_RSE_GAUGE = METRICS.gauge('excluded_rses.{rse}', documentation='Temporarly excluded RSEs')
|
|
71
69
|
|
|
72
70
|
|
|
73
|
-
def get_rses_to_process(
|
|
71
|
+
def get_rses_to_process(
|
|
72
|
+
rses: Optional["Iterable[str]"],
|
|
73
|
+
include_rses: Optional[str],
|
|
74
|
+
exclude_rses: Optional[str],
|
|
75
|
+
vos: Optional["Sequence[str]"]
|
|
76
|
+
) -> Optional[list[dict[str, Any]]]:
|
|
74
77
|
"""
|
|
75
78
|
Return the list of RSEs to process based on rses, include_rses and exclude_rses
|
|
76
79
|
|
|
@@ -105,7 +108,7 @@ def get_rses_to_process(rses, include_rses, exclude_rses, vos):
|
|
|
105
108
|
if not isinstance(result, NoValue):
|
|
106
109
|
return result
|
|
107
110
|
|
|
108
|
-
all_rses = []
|
|
111
|
+
all_rses: list[dict[str, Any]] = []
|
|
109
112
|
for vo in vos:
|
|
110
113
|
all_rses.extend(list_rses(filters={'vo': vo}))
|
|
111
114
|
|
|
@@ -115,21 +118,21 @@ def get_rses_to_process(rses, include_rses, exclude_rses, vos):
|
|
|
115
118
|
msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
|
|
116
119
|
', '.join([repr(rse) for rse in invalid]))
|
|
117
120
|
raise RSENotFound(msg)
|
|
118
|
-
|
|
121
|
+
rses_to_process = [rse for rse in all_rses if rse['rse'] in rses]
|
|
119
122
|
else:
|
|
120
|
-
|
|
123
|
+
rses_to_process = all_rses
|
|
121
124
|
|
|
122
125
|
if include_rses:
|
|
123
126
|
included_rses = parse_expression(include_rses)
|
|
124
|
-
|
|
127
|
+
rses_to_process = [rse for rse in rses_to_process if rse in included_rses]
|
|
125
128
|
|
|
126
129
|
if exclude_rses:
|
|
127
130
|
excluded_rses = parse_expression(exclude_rses)
|
|
128
|
-
|
|
131
|
+
rses_to_process = [rse for rse in rses_to_process if rse not in excluded_rses]
|
|
129
132
|
|
|
130
|
-
REGION.set(cache_key,
|
|
131
|
-
logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in
|
|
132
|
-
return
|
|
133
|
+
REGION.set(cache_key, rses_to_process)
|
|
134
|
+
logging.log(logging.INFO, 'Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses_to_process]))
|
|
135
|
+
return rses_to_process
|
|
133
136
|
|
|
134
137
|
|
|
135
138
|
def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info, is_staging, auto_exclude_threshold, logger=logging.log):
|
|
@@ -200,7 +203,7 @@ def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info,
|
|
|
200
203
|
add_message('deletion-failed', deletion_dict)
|
|
201
204
|
noaccess_attempts += 1
|
|
202
205
|
if noaccess_attempts >= auto_exclude_threshold:
|
|
203
|
-
logger(logging.INFO, 'Too many (%d) NOACCESS attempts for %s. RSE will be
|
|
206
|
+
logger(logging.INFO, 'Too many (%d) NOACCESS attempts for %s. RSE will be temporarily excluded.', noaccess_attempts, rse_name)
|
|
204
207
|
REGION.set('temporary_exclude_%s' % rse_id, True)
|
|
205
208
|
METRICS.gauge('excluded_rses.{rse}').labels(rse=rse_name).set(1)
|
|
206
209
|
|
|
@@ -232,7 +235,7 @@ def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info,
|
|
|
232
235
|
if replica['scope'].vo != 'def':
|
|
233
236
|
payload['vo'] = replica['scope'].vo
|
|
234
237
|
add_message('deletion-failed', payload)
|
|
235
|
-
logger(logging.INFO, 'Cannot connect to %s. RSE will be
|
|
238
|
+
logger(logging.INFO, 'Cannot connect to %s. RSE will be temporarily excluded.', rse_name)
|
|
236
239
|
REGION.set('temporary_exclude_%s' % rse_id, True)
|
|
237
240
|
EXCLUDED_RSE_GAUGE.labels(rse=rse_name).set(1)
|
|
238
241
|
finally:
|
|
@@ -240,7 +243,7 @@ def delete_from_storage(heartbeat_handler, hb_payload, replicas, prot, rse_info,
|
|
|
240
243
|
return deleted_files
|
|
241
244
|
|
|
242
245
|
|
|
243
|
-
def _rse_deletion_hostname(rse: RseData, scheme:
|
|
246
|
+
def _rse_deletion_hostname(rse: RseData, scheme: Optional[str]) -> Optional[str]:
|
|
244
247
|
"""
|
|
245
248
|
Retrieves the hostname of the default deletion protocol
|
|
246
249
|
"""
|
|
@@ -277,7 +280,7 @@ def get_max_deletion_threads_by_hostname(hostname: str) -> int:
|
|
|
277
280
|
return result
|
|
278
281
|
|
|
279
282
|
|
|
280
|
-
def __try_reserve_worker_slot(heartbeat_handler: "HeartbeatHandler", rse: RseData, hostname: str, logger: "
|
|
283
|
+
def __try_reserve_worker_slot(heartbeat_handler: "HeartbeatHandler", rse: RseData, hostname: str, logger: "LoggerFunction") -> Optional[str]:
|
|
281
284
|
"""
|
|
282
285
|
The maximum number of concurrent workers is limited per hostname and per RSE due to storage performance reasons.
|
|
283
286
|
This function tries to reserve a slot to run the deletion worker for the given RSE and hostname.
|
|
@@ -290,7 +293,7 @@ def __try_reserve_worker_slot(heartbeat_handler: "HeartbeatHandler", rse: RseDat
|
|
|
290
293
|
"""
|
|
291
294
|
|
|
292
295
|
rse_hostname_key = '%s,%s' % (rse.id, hostname)
|
|
293
|
-
payload_cnt = list_payload_counts(heartbeat_handler.executable, older_than=heartbeat_handler.older_than)
|
|
296
|
+
payload_cnt = list_payload_counts(heartbeat_handler.executable, older_than=heartbeat_handler.older_than) # type: ignore (argument missing: session)
|
|
294
297
|
tot_threads_for_hostname = 0
|
|
295
298
|
tot_threads_for_rse = 0
|
|
296
299
|
for key in payload_cnt:
|
|
@@ -309,7 +312,7 @@ def __try_reserve_worker_slot(heartbeat_handler: "HeartbeatHandler", rse: RseDat
|
|
|
309
312
|
return rse_hostname_key
|
|
310
313
|
|
|
311
314
|
|
|
312
|
-
def __check_rse_usage_cached(rse: RseData, greedy: bool = False, logger: "
|
|
315
|
+
def __check_rse_usage_cached(rse: RseData, greedy: bool = False, logger: "LoggerFunction" = logging.log) -> tuple[int, bool]:
|
|
313
316
|
"""
|
|
314
317
|
Wrapper around __check_rse_usage which manages the cache entry.
|
|
315
318
|
"""
|
|
@@ -321,7 +324,7 @@ def __check_rse_usage_cached(rse: RseData, greedy: bool = False, logger: "Callab
|
|
|
321
324
|
return result
|
|
322
325
|
|
|
323
326
|
|
|
324
|
-
def __check_rse_usage(rse: RseData, greedy: bool = False, logger: "
|
|
327
|
+
def __check_rse_usage(rse: RseData, greedy: bool = False, logger: "LoggerFunction" = logging.log) -> tuple[int, bool]:
|
|
325
328
|
"""
|
|
326
329
|
Internal method to check RSE usage and limits.
|
|
327
330
|
|
|
@@ -347,12 +350,12 @@ def __check_rse_usage(rse: RseData, greedy: bool = False, logger: "Callable[...,
|
|
|
347
350
|
|
|
348
351
|
# Check from which sources to get used and total spaces (default storage)
|
|
349
352
|
# If specified sources do not exist, only delete obsolete
|
|
350
|
-
source_for_total_space = rse.attributes.get(
|
|
353
|
+
source_for_total_space = rse.attributes.get(RseAttr.SOURCE_FOR_TOTAL_SPACE, 'storage')
|
|
351
354
|
if source_for_total_space not in available_sources['total']:
|
|
352
355
|
logger(logging.WARNING, 'RSE: %s, \'%s\' requested for source_for_total_space but cannot be found. Will only delete obsolete',
|
|
353
356
|
rse.name, source_for_total_space)
|
|
354
357
|
return 0, True
|
|
355
|
-
source_for_used_space = rse.attributes.get(
|
|
358
|
+
source_for_used_space = rse.attributes.get(RseAttr.SOURCE_FOR_USED_SPACE, 'storage')
|
|
356
359
|
if source_for_used_space not in available_sources['used']:
|
|
357
360
|
logger(logging.WARNING, 'RSE: %s, \'%s\' requested for source_for_used_space but cannot be found. Will only delete obsolete',
|
|
358
361
|
rse.name, source_for_used_space)
|
|
@@ -376,8 +379,20 @@ def __check_rse_usage(rse: RseData, greedy: bool = False, logger: "Callable[...,
|
|
|
376
379
|
return 0, True
|
|
377
380
|
|
|
378
381
|
|
|
379
|
-
def reaper(
|
|
380
|
-
|
|
382
|
+
def reaper(
|
|
383
|
+
rses: "Sequence[str]",
|
|
384
|
+
include_rses: Optional[str],
|
|
385
|
+
exclude_rses: Optional[str],
|
|
386
|
+
vos: Optional["Sequence[str]"] = None,
|
|
387
|
+
chunk_size: int = 100,
|
|
388
|
+
once: bool = False,
|
|
389
|
+
greedy: bool = False,
|
|
390
|
+
scheme: Optional[str] = None,
|
|
391
|
+
delay_seconds: int = 0,
|
|
392
|
+
sleep_time: int = 60,
|
|
393
|
+
auto_exclude_threshold: int = 100,
|
|
394
|
+
auto_exclude_timeout: int = 600
|
|
395
|
+
) -> None:
|
|
381
396
|
"""
|
|
382
397
|
Main loop to select and delete files.
|
|
383
398
|
|
|
@@ -395,11 +410,6 @@ def reaper(rses, include_rses, exclude_rses, vos=None, chunk_size=100, once=Fals
|
|
|
395
410
|
:param auto_exclude_threshold: Number of service unavailable exceptions after which the RSE gets temporarily excluded.
|
|
396
411
|
:param auto_exclude_timeout: Timeout for temporarily excluded RSEs.
|
|
397
412
|
"""
|
|
398
|
-
|
|
399
|
-
oidc_account = config_get('reaper', 'oidc_account', False, 'root')
|
|
400
|
-
oidc_scope = config_get('reaper', 'oidc_scope', False, 'delete')
|
|
401
|
-
oidc_audience = config_get('reaper', 'oidc_audience', False, 'rse')
|
|
402
|
-
|
|
403
413
|
run_daemon(
|
|
404
414
|
once=once,
|
|
405
415
|
graceful_stop=GRACEFUL_STOP,
|
|
@@ -418,16 +428,24 @@ def reaper(rses, include_rses, exclude_rses, vos=None, chunk_size=100, once=Fals
|
|
|
418
428
|
delay_seconds=delay_seconds,
|
|
419
429
|
auto_exclude_threshold=auto_exclude_threshold,
|
|
420
430
|
auto_exclude_timeout=auto_exclude_timeout,
|
|
421
|
-
oidc_account=oidc_account,
|
|
422
|
-
oidc_scope=oidc_scope,
|
|
423
|
-
oidc_audience=oidc_audience,
|
|
424
431
|
)
|
|
425
432
|
)
|
|
426
433
|
|
|
427
434
|
|
|
428
|
-
def run_once(
|
|
429
|
-
|
|
430
|
-
|
|
435
|
+
def run_once(
|
|
436
|
+
rses: "Sequence[str]",
|
|
437
|
+
include_rses: Optional[str],
|
|
438
|
+
exclude_rses: Optional[str],
|
|
439
|
+
vos: Optional["Sequence[str]"],
|
|
440
|
+
chunk_size: int,
|
|
441
|
+
greedy: bool,
|
|
442
|
+
scheme: Optional[str],
|
|
443
|
+
delay_seconds: int,
|
|
444
|
+
auto_exclude_threshold: int,
|
|
445
|
+
auto_exclude_timeout: int,
|
|
446
|
+
heartbeat_handler: "HeartbeatHandler",
|
|
447
|
+
**_kwargs
|
|
448
|
+
) -> bool:
|
|
431
449
|
|
|
432
450
|
must_sleep = True
|
|
433
451
|
|
|
@@ -455,10 +473,11 @@ def run_once(rses, include_rses, exclude_rses, vos, chunk_size, greedy, scheme,
|
|
|
455
473
|
return must_sleep
|
|
456
474
|
|
|
457
475
|
rses_to_process = get_rses_to_process(rses, include_rses, exclude_rses, vos)
|
|
458
|
-
rses_to_process = [RseData(id_=rse['id'], name=rse['rse'], columns=rse) for rse in rses_to_process]
|
|
459
476
|
if not rses_to_process:
|
|
460
477
|
logger(logging.ERROR, 'Reaper: No RSEs found. Will sleep for 30 seconds')
|
|
461
478
|
return must_sleep
|
|
479
|
+
else:
|
|
480
|
+
rses_to_process = [RseData(id_=rse['id'], name=rse['rse'], columns=rse) for rse in rses_to_process]
|
|
462
481
|
|
|
463
482
|
# On big deletion campaigns, we desire to re-iterate fast on RSEs which have a lot of data to delete.
|
|
464
483
|
# The called function will return the RSEs which have more work remaining.
|
|
@@ -476,9 +495,6 @@ def run_once(rses, include_rses, exclude_rses, vos, chunk_size, greedy, scheme,
|
|
|
476
495
|
auto_exclude_threshold=auto_exclude_threshold,
|
|
477
496
|
auto_exclude_timeout=auto_exclude_timeout,
|
|
478
497
|
heartbeat_handler=heartbeat_handler,
|
|
479
|
-
oidc_account=oidc_account,
|
|
480
|
-
oidc_scope=oidc_scope,
|
|
481
|
-
oidc_audience=oidc_audience,
|
|
482
498
|
)
|
|
483
499
|
if rses_to_process and iteration < max_fast_reiterations:
|
|
484
500
|
logger(logging.INFO, "Will perform fast-reiteration %d/%d with rses: %s", iteration + 1, max_fast_reiterations, [str(rse) for rse in rses_to_process])
|
|
@@ -492,9 +508,17 @@ def run_once(rses, include_rses, exclude_rses, vos, chunk_size, greedy, scheme,
|
|
|
492
508
|
return must_sleep
|
|
493
509
|
|
|
494
510
|
|
|
495
|
-
def _run_once(
|
|
496
|
-
|
|
497
|
-
|
|
511
|
+
def _run_once(
|
|
512
|
+
rses_to_process: "Iterable[RseData]",
|
|
513
|
+
chunk_size: int,
|
|
514
|
+
greedy: bool,
|
|
515
|
+
scheme: Optional[str],
|
|
516
|
+
delay_seconds: int,
|
|
517
|
+
auto_exclude_threshold: int,
|
|
518
|
+
auto_exclude_timeout: int,
|
|
519
|
+
heartbeat_handler: "HeartbeatHandler",
|
|
520
|
+
**_kwargs
|
|
521
|
+
) -> list[RseData]:
|
|
498
522
|
|
|
499
523
|
dict_rses = {}
|
|
500
524
|
_, total_workers, logger = heartbeat_handler.live()
|
|
@@ -505,7 +529,7 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
505
529
|
logger(logging.DEBUG, 'RSE %s is blocklisted for delete', rse.name)
|
|
506
530
|
continue
|
|
507
531
|
rse.ensure_loaded(load_attributes=True)
|
|
508
|
-
enable_greedy = rse.attributes.get(
|
|
532
|
+
enable_greedy = rse.attributes.get(RseAttr.GREEDYDELETION, False) or greedy
|
|
509
533
|
needed_free_space, only_delete_obsolete = __check_rse_usage_cached(rse, greedy=enable_greedy, logger=logger)
|
|
510
534
|
if needed_free_space:
|
|
511
535
|
dict_rses[rse] = [needed_free_space, only_delete_obsolete, enable_greedy]
|
|
@@ -537,7 +561,7 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
537
561
|
|
|
538
562
|
result = REGION.get('temporary_exclude_%s' % rse.id, expiration_time=auto_exclude_timeout)
|
|
539
563
|
if not isinstance(result, NoValue):
|
|
540
|
-
logger(logging.WARNING, 'Too many failed attempts for %s in last cycle. RSE is
|
|
564
|
+
logger(logging.WARNING, 'Too many failed attempts for %s in last cycle. RSE is temporarily excluded.', rse.name)
|
|
541
565
|
EXCLUDED_RSE_GAUGE.labels(rse=rse.name).set(1)
|
|
542
566
|
continue
|
|
543
567
|
EXCLUDED_RSE_GAUGE.labels(rse=rse.name).set(0)
|
|
@@ -572,7 +596,7 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
572
596
|
rse_id=rse.id,
|
|
573
597
|
delay_seconds=delay_seconds,
|
|
574
598
|
only_delete_obsolete=only_delete_obsolete,
|
|
575
|
-
session=None)
|
|
599
|
+
session=None) # type: ignore (argument missing: session)
|
|
576
600
|
logger(logging.DEBUG, 'list_and_mark_unlocked_replicas on %s for %s bytes in %s seconds: %s replicas', rse.name, needed_free_space, time.time() - del_start_time, len(replicas))
|
|
577
601
|
if (len(replicas) == 0 and enable_greedy) or (len(replicas) < chunk_size and not enable_greedy):
|
|
578
602
|
logger(logging.DEBUG, 'Not enough replicas to delete on %s (%s requested vs %s returned). Will skip any new attempts on this RSE until next cycle', rse.name, chunk_size, len(replicas))
|
|
@@ -590,14 +614,18 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
590
614
|
# Physical deletion will take place there
|
|
591
615
|
try:
|
|
592
616
|
rse.ensure_loaded(load_info=True, load_attributes=True)
|
|
593
|
-
|
|
594
|
-
if
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
617
|
+
prot = rsemgr.create_protocol(rse.info, 'delete', scheme=scheme, logger=logger)
|
|
618
|
+
if rse.attributes.get(RseAttr.OIDC_SUPPORT) is True and prot.attributes['scheme'] == 'davs':
|
|
619
|
+
audience = determine_audience_for_rse(rse.id)
|
|
620
|
+
# FIXME: At the time of writing, StoRM requires `storage.read`
|
|
621
|
+
# in order to perform a stat operation.
|
|
622
|
+
scope = determine_scope_for_rse(rse.id, scopes=['storage.modify', 'storage.read'])
|
|
623
|
+
auth_token = request_token(audience, scope)
|
|
624
|
+
if auth_token:
|
|
625
|
+
logger(logging.INFO, 'Using a token to delete on RSE %s', rse.name)
|
|
626
|
+
prot = rsemgr.create_protocol(rse.info, 'delete', scheme=scheme, auth_token=auth_token, logger=logger)
|
|
627
|
+
else:
|
|
628
|
+
logger(logging.WARNING, 'Failed to procure a token to delete on RSE %s', rse.name)
|
|
601
629
|
for file_replicas in chunks(replicas, chunk_size):
|
|
602
630
|
# Refresh heartbeat
|
|
603
631
|
_, total_workers, logger = heartbeat_handler.live(payload=hb_payload)
|
|
@@ -620,8 +648,8 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
620
648
|
|
|
621
649
|
# Then finally delete the replicas
|
|
622
650
|
del_start = time.time()
|
|
623
|
-
delete_replicas(rse_id=rse.id, files=deleted_files)
|
|
624
|
-
logger(logging.DEBUG, 'delete_replicas
|
|
651
|
+
delete_replicas(rse_id=rse.id, files=deleted_files) # type: ignore (argument missing: session)
|
|
652
|
+
logger(logging.DEBUG, 'delete_replicas succeeded on %s : %s replicas in %s seconds', rse.name, len(deleted_files), time.time() - del_start)
|
|
625
653
|
METRICS.counter('deletion.done').inc(len(deleted_files))
|
|
626
654
|
except RSEProtocolNotSupported:
|
|
627
655
|
logger(logging.WARNING, 'Protocol %s not supported on %s', scheme, rse.name)
|
|
@@ -635,14 +663,28 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
|
|
|
635
663
|
return rses_with_more_work
|
|
636
664
|
|
|
637
665
|
|
|
638
|
-
def stop(signum:
|
|
666
|
+
def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
|
|
639
667
|
"""
|
|
640
668
|
Graceful exit.
|
|
641
669
|
"""
|
|
642
670
|
GRACEFUL_STOP.set()
|
|
643
671
|
|
|
644
672
|
|
|
645
|
-
def run(
|
|
673
|
+
def run(
|
|
674
|
+
threads: int = 1,
|
|
675
|
+
chunk_size: int = 100,
|
|
676
|
+
once: bool = False,
|
|
677
|
+
greedy: bool = False,
|
|
678
|
+
rses: Optional["Sequence[str]"] = None,
|
|
679
|
+
scheme: Optional[str] = None,
|
|
680
|
+
exclude_rses: Optional[str] = None,
|
|
681
|
+
include_rses: Optional[str] = None,
|
|
682
|
+
vos: Optional["Sequence[str]"] = None,
|
|
683
|
+
delay_seconds: int = 0,
|
|
684
|
+
sleep_time: int = 60,
|
|
685
|
+
auto_exclude_threshold: int = 100,
|
|
686
|
+
auto_exclude_timeout: int = 600
|
|
687
|
+
) -> None:
|
|
646
688
|
"""
|
|
647
689
|
Starts up the reaper threads.
|
|
648
690
|
|