rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,46 +17,44 @@ import logging
|
|
|
18
17
|
import re
|
|
19
18
|
import threading
|
|
20
19
|
import time
|
|
21
|
-
from collections.abc import Callable
|
|
22
20
|
from datetime import datetime
|
|
23
|
-
from json import
|
|
24
|
-
from typing import TYPE_CHECKING
|
|
21
|
+
from json import dumps, loads
|
|
22
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
25
23
|
|
|
26
24
|
import rucio.db.sqla.util
|
|
27
25
|
from rucio.common.config import config_get
|
|
26
|
+
from rucio.common.constants import RseAttr
|
|
28
27
|
from rucio.common.exception import (
|
|
29
28
|
DatabaseException,
|
|
30
|
-
InvalidReplicationRule,
|
|
31
29
|
DuplicateRule,
|
|
32
|
-
InvalidRSEExpression,
|
|
33
|
-
InsufficientTargetRSEs,
|
|
34
30
|
InsufficientAccountLimit,
|
|
35
|
-
|
|
31
|
+
InsufficientTargetRSEs,
|
|
32
|
+
InvalidReplicationRule,
|
|
33
|
+
InvalidRSEExpression,
|
|
36
34
|
InvalidRuleWeight,
|
|
35
|
+
RSEOverQuota,
|
|
37
36
|
StagingAreaRuleRequiresLifetime,
|
|
38
|
-
SubscriptionWrongParameter,
|
|
39
37
|
SubscriptionNotFound,
|
|
38
|
+
SubscriptionWrongParameter,
|
|
40
39
|
)
|
|
41
40
|
from rucio.common.logging import setup_logging
|
|
42
41
|
from rucio.common.stopwatch import Stopwatch
|
|
43
|
-
from rucio.common.types import InternalAccount
|
|
42
|
+
from rucio.common.types import InternalAccount, InternalScope, LoggerFunction
|
|
44
43
|
from rucio.common.utils import chunks
|
|
45
|
-
from rucio.core.did import list_new_dids, set_new_dids
|
|
44
|
+
from rucio.core.did import get_metadata, list_new_dids, set_new_dids
|
|
46
45
|
from rucio.core.monitor import MetricManager
|
|
47
|
-
from rucio.core.rse import
|
|
46
|
+
from rucio.core.rse import get_rse_id, list_rse_attributes, list_rses, rse_exists
|
|
48
47
|
from rucio.core.rse_expression_parser import parse_expression
|
|
49
48
|
from rucio.core.rse_selector import resolve_rse_expression
|
|
50
|
-
from rucio.core.rule import add_rule,
|
|
49
|
+
from rucio.core.rule import add_rule, get_rule, list_rules
|
|
51
50
|
from rucio.core.subscription import list_subscriptions, update_subscription
|
|
52
51
|
from rucio.daemons.common import run_daemon
|
|
53
52
|
from rucio.db.sqla.constants import DIDType, SubscriptionState
|
|
54
53
|
|
|
55
54
|
if TYPE_CHECKING:
|
|
56
55
|
from types import FrameType
|
|
57
|
-
from typing import Optional
|
|
58
56
|
|
|
59
57
|
from rucio.daemons.common import HeartbeatHandler
|
|
60
|
-
from rucio.common.types import InternalScope
|
|
61
58
|
|
|
62
59
|
METRICS = MetricManager(module=__name__)
|
|
63
60
|
graceful_stop = threading.Event()
|
|
@@ -70,7 +67,7 @@ def __get_rule_dict(rule_dict: dict, subscription: dict) -> dict:
|
|
|
70
67
|
"""
|
|
71
68
|
Internal method to clean and enrich the rule_dict coming from the subscription.
|
|
72
69
|
|
|
73
|
-
:param rule_dict: The rule
|
|
70
|
+
:param rule_dict: The rule dictionary coming from a subscription.
|
|
74
71
|
:param subscription: The subscription associated to the rule.
|
|
75
72
|
:return: A dictionary that contains all the parameters associated to the rule.
|
|
76
73
|
"""
|
|
@@ -128,7 +125,7 @@ def __split_rule_select_rses(
|
|
|
128
125
|
rse_expression: str,
|
|
129
126
|
copies: int,
|
|
130
127
|
blocklisted_rse_id: list,
|
|
131
|
-
logger:
|
|
128
|
+
logger: LoggerFunction,
|
|
132
129
|
) -> tuple[list, bool, bool]:
|
|
133
130
|
"""
|
|
134
131
|
Internal method to create a list of RSEs that match RSE expression for subscriptions with split_rule.
|
|
@@ -213,9 +210,9 @@ def __split_rule_select_rses(
|
|
|
213
210
|
return selected_rses, create_rule, wont_reevaluate
|
|
214
211
|
|
|
215
212
|
|
|
216
|
-
def get_subscriptions(logger:
|
|
213
|
+
def get_subscriptions(logger: LoggerFunction = logging.log) -> list[dict]:
|
|
217
214
|
"""
|
|
218
|
-
A method to extract the list of active subscriptions and
|
|
215
|
+
A method to extract the list of active subscriptions and exclude the one that have bad RSE expression.
|
|
219
216
|
:param logger: The logger.
|
|
220
217
|
:return: The list of active subscriptions.
|
|
221
218
|
"""
|
|
@@ -292,13 +289,17 @@ def get_subscriptions(logger: Callable = logging.log) -> list[dict]:
|
|
|
292
289
|
return subscriptions
|
|
293
290
|
|
|
294
291
|
|
|
295
|
-
def __is_matching_subscription(
|
|
292
|
+
def __is_matching_subscription(
|
|
293
|
+
subscription: dict[str, Any],
|
|
294
|
+
did: dict[str, Any],
|
|
295
|
+
metadata: dict[str, Any]
|
|
296
|
+
) -> bool:
|
|
296
297
|
"""
|
|
297
298
|
Internal method to identify if a DID matches a subscription.
|
|
298
299
|
|
|
299
300
|
:param subscription: The subscription dictionary.
|
|
300
301
|
:param did: The DID dictionary
|
|
301
|
-
:param metadata: The metadata
|
|
302
|
+
:param metadata: The metadata dictionary for the DID
|
|
302
303
|
:return: True/False
|
|
303
304
|
"""
|
|
304
305
|
if metadata["hidden"]:
|
|
@@ -381,7 +382,12 @@ def __is_matching_subscription(subscription, did, metadata):
|
|
|
381
382
|
return True
|
|
382
383
|
|
|
383
384
|
|
|
384
|
-
def select_algorithm(
|
|
385
|
+
def select_algorithm(
|
|
386
|
+
algorithm: str,
|
|
387
|
+
rule_ids: list[str],
|
|
388
|
+
params: dict[str, Any],
|
|
389
|
+
logger: LoggerFunction
|
|
390
|
+
) -> dict:
|
|
385
391
|
"""
|
|
386
392
|
Method used in case of chained subscriptions
|
|
387
393
|
|
|
@@ -400,7 +406,7 @@ def select_algorithm(algorithm: str, rule_ids: list, params: dict, logger: "Call
|
|
|
400
406
|
rse_id = get_rse_id(rse, vo=vo)
|
|
401
407
|
rse_attributes = list_rse_attributes(rse_id)
|
|
402
408
|
if algorithm == "associated_site":
|
|
403
|
-
associated_sites = rse_attributes.get(
|
|
409
|
+
associated_sites = rse_attributes.get(RseAttr.ASSOCIATED_SITES, None)
|
|
404
410
|
associated_site_idx = params.get("associated_site_idx", None)
|
|
405
411
|
if not associated_site_idx:
|
|
406
412
|
raise SubscriptionWrongParameter(
|
|
@@ -418,7 +424,7 @@ def select_algorithm(algorithm: str, rule_ids: list, params: dict, logger: "Call
|
|
|
418
424
|
"weight": None,
|
|
419
425
|
}
|
|
420
426
|
if algorithm == "exclude_site":
|
|
421
|
-
site = rse_attributes.get(
|
|
427
|
+
site = rse_attributes.get(RseAttr.SITE, None)
|
|
422
428
|
rse_expression = params['rse_expression'] + '\\site=%s' % site
|
|
423
429
|
(
|
|
424
430
|
selected_rses,
|
|
@@ -713,7 +719,10 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
|
|
|
713
719
|
|
|
714
720
|
|
|
715
721
|
def run(
|
|
716
|
-
threads: int = 1,
|
|
722
|
+
threads: int = 1,
|
|
723
|
+
bulk: int = 100,
|
|
724
|
+
once: bool = False,
|
|
725
|
+
sleep_time: int = 60
|
|
717
726
|
) -> None:
|
|
718
727
|
"""
|
|
719
728
|
Starts up the transmogrifier threads.
|
|
@@ -746,7 +755,7 @@ def run(
|
|
|
746
755
|
]
|
|
747
756
|
|
|
748
757
|
|
|
749
|
-
def stop(signum:
|
|
758
|
+
def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
|
|
750
759
|
"""
|
|
751
760
|
Graceful exit.
|
|
752
761
|
"""
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -30,13 +29,14 @@ from typing import TYPE_CHECKING
|
|
|
30
29
|
from sqlalchemy.exc import DatabaseError
|
|
31
30
|
|
|
32
31
|
import rucio.db.sqla.util
|
|
33
|
-
from rucio.common.exception import DatabaseException,
|
|
32
|
+
from rucio.common.exception import DatabaseException, RuleNotFound, UnsupportedOperation
|
|
34
33
|
from rucio.common.logging import setup_logging
|
|
35
34
|
from rucio.common.types import InternalAccount
|
|
36
35
|
from rucio.common.utils import chunks
|
|
37
|
-
from rucio.core.did import
|
|
36
|
+
from rucio.core.did import delete_dids, list_expired_dids
|
|
38
37
|
from rucio.core.monitor import MetricManager
|
|
39
|
-
from rucio.daemons.common import
|
|
38
|
+
from rucio.daemons.common import HeartbeatHandler, run_daemon
|
|
39
|
+
from rucio.db.sqla.constants import MYSQL_LOCK_NOWAIT_REGEX, ORACLE_RESOURCE_BUSY_REGEX, PSQL_LOCK_NOT_AVAILABLE_REGEX
|
|
40
40
|
|
|
41
41
|
if TYPE_CHECKING:
|
|
42
42
|
from types import FrameType
|
|
@@ -49,7 +49,7 @@ graceful_stop = threading.Event()
|
|
|
49
49
|
DAEMON_NAME = 'undertaker'
|
|
50
50
|
|
|
51
51
|
|
|
52
|
-
def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10):
|
|
52
|
+
def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10) -> None:
|
|
53
53
|
"""
|
|
54
54
|
Main loop to select and delete dids.
|
|
55
55
|
"""
|
|
@@ -68,7 +68,7 @@ def undertaker(once: bool = False, sleep_time: int = 60, chunk_size: int = 10):
|
|
|
68
68
|
)
|
|
69
69
|
|
|
70
70
|
|
|
71
|
-
def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_handler: HeartbeatHandler, **_kwargs):
|
|
71
|
+
def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_handler: HeartbeatHandler, **_kwargs) -> None:
|
|
72
72
|
worker_number, total_workers, logger = heartbeat_handler.live()
|
|
73
73
|
|
|
74
74
|
try:
|
|
@@ -96,9 +96,9 @@ def run_once(paused_dids: dict[tuple, datetime], chunk_size: int, heartbeat_hand
|
|
|
96
96
|
except RuleNotFound as error:
|
|
97
97
|
logger(logging.ERROR, error)
|
|
98
98
|
except (DatabaseException, DatabaseError, UnsupportedOperation) as e:
|
|
99
|
-
if match(
|
|
99
|
+
if match(ORACLE_RESOURCE_BUSY_REGEX, str(e.args[0])) or match(PSQL_LOCK_NOT_AVAILABLE_REGEX, str(e.args[0])) or match(MYSQL_LOCK_NOWAIT_REGEX, str(e.args[0])):
|
|
100
100
|
for did in chunk:
|
|
101
|
-
paused_dids[(did['scope'], did['name'])] = datetime.utcnow() + timedelta(seconds=randint(600, 2400))
|
|
101
|
+
paused_dids[(did['scope'], did['name'])] = datetime.utcnow() + timedelta(seconds=randint(600, 2400)) # noqa: S311
|
|
102
102
|
METRICS.counter('delete_dids.exceptions.{exception}').labels(exception='LocksDetected').inc()
|
|
103
103
|
logger(logging.WARNING, 'Locks detected for chunk')
|
|
104
104
|
else:
|
|
@@ -114,14 +114,14 @@ def stop(signum: "Optional[int]" = None, frame: "Optional[FrameType]" = None) ->
|
|
|
114
114
|
graceful_stop.set()
|
|
115
115
|
|
|
116
116
|
|
|
117
|
-
def run(once: bool = False, total_workers: int = 1, chunk_size: int = 10, sleep_time: int = 60):
|
|
117
|
+
def run(once: bool = False, total_workers: int = 1, chunk_size: int = 10, sleep_time: int = 60) -> None:
|
|
118
118
|
"""
|
|
119
119
|
Starts up the undertaker threads.
|
|
120
120
|
"""
|
|
121
121
|
setup_logging(process_name=DAEMON_NAME)
|
|
122
122
|
|
|
123
123
|
if rucio.db.sqla.util.is_old_db():
|
|
124
|
-
raise DatabaseException(
|
|
124
|
+
raise DatabaseException("Database was not updated, daemon won't start")
|
|
125
125
|
|
|
126
126
|
if once:
|
|
127
127
|
undertaker(once)
|
rucio/db/__init__.py
CHANGED
rucio/db/sqla/__init__.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -13,10 +12,25 @@
|
|
|
13
12
|
# See the License for the specific language governing permissions and
|
|
14
13
|
# limitations under the License.
|
|
15
14
|
|
|
15
|
+
from typing import TYPE_CHECKING, Optional, TypeVar
|
|
16
|
+
|
|
16
17
|
from sqlalchemy.sql.expression import bindparam, text
|
|
17
18
|
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from sqlalchemy.orm import Session
|
|
21
|
+
from sqlalchemy.orm.query import RowReturningQuery
|
|
22
|
+
from sqlalchemy.sql.selectable import Select
|
|
23
|
+
|
|
24
|
+
Q = TypeVar('Q', RowReturningQuery, Select)
|
|
25
|
+
|
|
18
26
|
|
|
19
|
-
def filter_thread_work(
|
|
27
|
+
def filter_thread_work(
|
|
28
|
+
session: "Session",
|
|
29
|
+
query: "Q",
|
|
30
|
+
total_threads: Optional[int],
|
|
31
|
+
thread_id: Optional[int],
|
|
32
|
+
hash_variable: Optional[str] = None
|
|
33
|
+
) -> "Q":
|
|
20
34
|
""" Filters a query to partition thread workloads based on the thread id and total number of threads """
|
|
21
35
|
if thread_id is not None and total_threads is not None and (total_threads - 1) > 0:
|
|
22
36
|
if session.bind.dialect.name == 'oracle':
|
rucio/db/sqla/constants.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -20,6 +19,16 @@ from enum import Enum
|
|
|
20
19
|
|
|
21
20
|
OBSOLETE = datetime(year=1970, month=1, day=1) # Tombstone value to mark obsolete replicas
|
|
22
21
|
|
|
22
|
+
# Regex patterns
|
|
23
|
+
|
|
24
|
+
ORACLE_CONNECTION_LOST_CONTACT_REGEX = r".*ORA-03135.*"
|
|
25
|
+
ORACLE_DEADLOCK_DETECTED_REGEX = r".*ORA-00060.*"
|
|
26
|
+
ORACLE_RESOURCE_BUSY_REGEX = r".*ORA-00054.*"
|
|
27
|
+
ORACLE_UNIQUE_CONSTRAINT_VIOLATED_REGEX = r".*ORA-00001.*"
|
|
28
|
+
PSQL_LOCK_NOT_AVAILABLE_REGEX = r".*55P03.*"
|
|
29
|
+
MYSQL_LOCK_NOWAIT_REGEX = r".*3572.*"
|
|
30
|
+
MYSQL_LOCK_WAIT_TIMEOUT_EXCEEDED = "ERROR 1205 (HY000)"
|
|
31
|
+
|
|
23
32
|
|
|
24
33
|
# The enum values below are the actual strings stored in the database -- these must be string types.
|
|
25
34
|
# This is done explicitly via values_callable to SQLAlchemy enums in models.py and alembic scripts,
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add eol_at in rules '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '0437a40dbfd1'
|
|
26
23
|
down_revision = 'a5f6f6e928a7'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,8 +18,7 @@ import datetime
|
|
|
19
18
|
|
|
20
19
|
import sqlalchemy as sa
|
|
21
20
|
from alembic import context
|
|
22
|
-
from alembic.op import
|
|
23
|
-
create_check_constraint, create_index, drop_table)
|
|
21
|
+
from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
|
|
24
22
|
|
|
25
23
|
from rucio.db.sqla.types import GUID
|
|
26
24
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' added stuck_at column to rules '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '102efcf145f4'
|
|
26
23
|
down_revision = '70587619328'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,8 +17,7 @@ import datetime
|
|
|
18
17
|
|
|
19
18
|
import sqlalchemy as sa
|
|
20
19
|
from alembic import context
|
|
21
|
-
from alembic.op import
|
|
22
|
-
create_index, create_check_constraint, drop_table)
|
|
20
|
+
from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
|
|
23
21
|
|
|
24
22
|
from rucio.db.sqla.constants import TransferLimitDirection
|
|
25
23
|
from rucio.db.sqla.types import GUID
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,9 +15,8 @@
|
|
|
16
15
|
''' cleanup distances table '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
|
-
from alembic.op import add_column,
|
|
19
|
+
from alembic.op import add_column, alter_column, drop_column
|
|
22
20
|
|
|
23
21
|
# Alembic revision identifiers
|
|
24
22
|
revision = '140fef722e91'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add request external_host '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '14ec5aeb64cf'
|
|
26
23
|
down_revision = '52fd9f4916fa'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,9 +15,7 @@
|
|
|
16
15
|
''' add request_type to requests idx '''
|
|
17
16
|
|
|
18
17
|
from alembic import context
|
|
19
|
-
from alembic.op import
|
|
20
|
-
drop_constraint, drop_index)
|
|
21
|
-
|
|
18
|
+
from alembic.op import create_foreign_key, create_index, create_unique_constraint, drop_constraint, drop_index
|
|
22
19
|
|
|
23
20
|
# Alembic revision identifiers
|
|
24
21
|
revision = '156fb5b5a14'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,7 +17,6 @@
|
|
|
18
17
|
from alembic import context
|
|
19
18
|
from alembic.op import create_index, drop_index
|
|
20
19
|
|
|
21
|
-
|
|
22
20
|
# Alembic revision identifiers
|
|
23
21
|
revision = '16a0aca82e12'
|
|
24
22
|
down_revision = None
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,7 +17,6 @@
|
|
|
18
17
|
from alembic import context
|
|
19
18
|
from alembic.op import create_index, drop_index
|
|
20
19
|
|
|
21
|
-
|
|
22
20
|
# Alembic revision identifiers
|
|
23
21
|
revision = '1a80adff031a'
|
|
24
22
|
down_revision = '3ad36e2268b0'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,8 +18,7 @@ import datetime
|
|
|
19
18
|
|
|
20
19
|
import sqlalchemy as sa
|
|
21
20
|
from alembic import context
|
|
22
|
-
from alembic.op import
|
|
23
|
-
create_check_constraint, drop_table)
|
|
21
|
+
from alembic.op import create_check_constraint, create_foreign_key, create_primary_key, create_table, drop_table
|
|
24
22
|
|
|
25
23
|
from rucio.common.schema import get_schema_value
|
|
26
24
|
from rucio.db.sqla.types import GUID
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add bytes column to bad_replicas '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '1f46c5f240ac'
|
|
26
23
|
down_revision = '688ef1840840'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -85,7 +84,7 @@ def upgrade():
|
|
|
85
84
|
)
|
|
86
85
|
|
|
87
86
|
conn.execute(
|
|
88
|
-
sa.insert(
|
|
87
|
+
sa.insert(get_rse_attr_association()).from_select(
|
|
89
88
|
["rse_id", "key", "value", "created_at", "updated_at"], select_stmt
|
|
90
89
|
)
|
|
91
90
|
)
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add availability column to table RSEs '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '22cf51430c78'
|
|
26
23
|
down_revision = '49a21b4d4357'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -19,8 +18,7 @@ import datetime
|
|
|
19
18
|
|
|
20
19
|
import sqlalchemy as sa
|
|
21
20
|
from alembic import context
|
|
22
|
-
from alembic.op import
|
|
23
|
-
create_check_constraint, create_index, drop_table)
|
|
21
|
+
from alembic.op import create_check_constraint, create_foreign_key, create_index, create_primary_key, create_table, drop_table
|
|
24
22
|
|
|
25
23
|
from rucio.db.sqla.types import GUID
|
|
26
24
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,9 +15,7 @@
|
|
|
16
15
|
''' remove unique constraint on requests '''
|
|
17
16
|
|
|
18
17
|
from alembic import context
|
|
19
|
-
from alembic.op import
|
|
20
|
-
drop_constraint, create_unique_constraint)
|
|
21
|
-
|
|
18
|
+
from alembic.op import create_foreign_key, create_index, create_unique_constraint, drop_constraint, drop_index
|
|
22
19
|
|
|
23
20
|
# Alembic revision identifiers
|
|
24
21
|
revision = '25821a8a45a3'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,7 +17,6 @@
|
|
|
18
17
|
from alembic import context
|
|
19
18
|
from alembic.op import create_index, drop_index
|
|
20
19
|
|
|
21
|
-
|
|
22
20
|
# Alembic revision identifiers
|
|
23
21
|
revision = '25fc855625cf'
|
|
24
22
|
down_revision = '4a7182d9578b'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add repair_cnt to locks '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '269fee20dee9'
|
|
26
23
|
down_revision = '1d96f484df21'
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,11 +15,9 @@
|
|
|
16
15
|
''' add ignore_availability column to rules '''
|
|
17
16
|
|
|
18
17
|
import sqlalchemy as sa
|
|
19
|
-
|
|
20
18
|
from alembic import context
|
|
21
19
|
from alembic.op import add_column, drop_column
|
|
22
20
|
|
|
23
|
-
|
|
24
21
|
# Alembic revision identifiers
|
|
25
22
|
revision = '271a46ea6244'
|
|
26
23
|
down_revision = 'd6dceb1de2d'
|