rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/core/rule.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -15,53 +14,74 @@
|
|
|
15
14
|
|
|
16
15
|
import json
|
|
17
16
|
import logging
|
|
18
|
-
from
|
|
17
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
18
|
+
from configparser import NoOptionError, NoSectionError
|
|
19
19
|
from copy import deepcopy
|
|
20
20
|
from datetime import datetime, timedelta
|
|
21
21
|
from os import path
|
|
22
22
|
from re import match
|
|
23
23
|
from string import Template
|
|
24
|
-
from typing import Any, Optional
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
from
|
|
28
|
-
from sqlalchemy import
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union
|
|
25
|
+
|
|
26
|
+
from dogpile.cache.api import NoValue
|
|
27
|
+
from sqlalchemy import delete, desc, select, update
|
|
28
|
+
from sqlalchemy.exc import (
|
|
29
|
+
IntegrityError,
|
|
30
|
+
NoResultFound, # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.exc.NoResultFound.html
|
|
31
|
+
StatementError,
|
|
32
|
+
)
|
|
31
33
|
from sqlalchemy.sql import func
|
|
32
|
-
from sqlalchemy.sql.expression import and_,
|
|
34
|
+
from sqlalchemy.sql.expression import and_, false, null, or_, true, tuple_
|
|
33
35
|
|
|
34
36
|
import rucio.core.did
|
|
35
37
|
import rucio.core.lock # import get_replica_locks, get_files_and_replica_locks_of_dataset
|
|
36
38
|
import rucio.core.replica # import get_and_lock_file_replicas, get_and_lock_file_replicas_for_dataset
|
|
37
39
|
from rucio.common.cache import make_region_memcached
|
|
38
40
|
from rucio.common.config import config_get
|
|
39
|
-
from rucio.common.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
from rucio.common.constants import RseAttr
|
|
42
|
+
from rucio.common.exception import (
|
|
43
|
+
DataIdentifierNotFound,
|
|
44
|
+
DuplicateRule,
|
|
45
|
+
InputValidationError,
|
|
46
|
+
InsufficientAccountLimit,
|
|
47
|
+
InsufficientTargetRSEs,
|
|
48
|
+
InvalidObject,
|
|
49
|
+
InvalidReplicationRule,
|
|
50
|
+
InvalidRSEExpression,
|
|
51
|
+
InvalidRuleWeight,
|
|
52
|
+
InvalidSourceReplicaExpression,
|
|
53
|
+
InvalidValueForKey,
|
|
54
|
+
ManualRuleApprovalBlocked,
|
|
55
|
+
ReplicationRuleCreationTemporaryFailed,
|
|
56
|
+
RequestNotFound,
|
|
57
|
+
RSEOverQuota,
|
|
58
|
+
RSEWriteBlocked,
|
|
59
|
+
RucioException,
|
|
60
|
+
RuleNotFound,
|
|
61
|
+
RuleReplaceFailed,
|
|
62
|
+
StagingAreaRuleRequiresLifetime,
|
|
63
|
+
UndefinedPolicy,
|
|
64
|
+
UnsupportedOperation,
|
|
65
|
+
)
|
|
66
|
+
from rucio.common.plugins import PolicyPackageAlgorithms
|
|
67
|
+
from rucio.common.policy import get_scratchdisk_lifetime, policy_filter
|
|
47
68
|
from rucio.common.schema import validate_schema
|
|
48
|
-
from rucio.common.types import InternalScope,
|
|
49
|
-
from rucio.common.utils import
|
|
50
|
-
from rucio.core import account_counter, rse_counter
|
|
51
|
-
from rucio.core
|
|
52
|
-
from rucio.core
|
|
69
|
+
from rucio.common.types import DIDDict, InternalAccount, InternalScope, LoggerFunction, RuleDict
|
|
70
|
+
from rucio.common.utils import chunks, sizefmt, str_to_date
|
|
71
|
+
from rucio.core import account_counter, rse_counter
|
|
72
|
+
from rucio.core import request as request_core
|
|
73
|
+
from rucio.core import transfer as transfer_core
|
|
74
|
+
from rucio.core.account import get_account, has_account_attribute
|
|
53
75
|
from rucio.core.lifetime_exception import define_eol
|
|
54
76
|
from rucio.core.message import add_message
|
|
55
77
|
from rucio.core.monitor import MetricManager
|
|
56
|
-
from rucio.core.rse import
|
|
78
|
+
from rucio.core.rse import get_rse, get_rse_name, get_rse_usage, list_rse_attributes
|
|
57
79
|
from rucio.core.rse_expression_parser import parse_expression
|
|
58
80
|
from rucio.core.rse_selector import RSESelector
|
|
59
|
-
from rucio.core.rule_grouping import
|
|
60
|
-
from rucio.db.sqla import
|
|
61
|
-
from rucio.db.sqla.constants import
|
|
62
|
-
|
|
63
|
-
RequestType, RuleNotification, OBSOLETE, RSEType)
|
|
64
|
-
from rucio.db.sqla.session import read_session, transactional_session, stream_session
|
|
81
|
+
from rucio.core.rule_grouping import apply_rule, apply_rule_grouping, create_transfer_dict, repair_stuck_locks_and_apply_rule_grouping
|
|
82
|
+
from rucio.db.sqla import filter_thread_work, models
|
|
83
|
+
from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, DIDAvailability, DIDReEvaluation, DIDType, LockState, ReplicaState, RequestType, RSEType, RuleGrouping, RuleNotification, RuleState
|
|
84
|
+
from rucio.db.sqla.session import read_session, stream_session, transactional_session
|
|
65
85
|
|
|
66
86
|
if TYPE_CHECKING:
|
|
67
87
|
from sqlalchemy.orm import Session
|
|
@@ -69,13 +89,106 @@ if TYPE_CHECKING:
|
|
|
69
89
|
|
|
70
90
|
REGION = make_region_memcached(expiration_time=900)
|
|
71
91
|
METRICS = MetricManager(module=__name__)
|
|
92
|
+
AutoApproveT = TypeVar('AutoApproveT', bound='AutoApprove')
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class AutoApprove(PolicyPackageAlgorithms):
|
|
96
|
+
"""
|
|
97
|
+
Handle automatic approval algorithms for replication rules
|
|
98
|
+
"""
|
|
99
|
+
|
|
100
|
+
_algorithm_type = 'auto_approve'
|
|
101
|
+
|
|
102
|
+
def __init__(self, rule: models.ReplicationRule, did: models.DataIdentifier, session: 'Session') -> None:
|
|
103
|
+
super().__init__()
|
|
104
|
+
self.rule = rule
|
|
105
|
+
self.did = did
|
|
106
|
+
self.session = session
|
|
107
|
+
self.register("default", self.default)
|
|
108
|
+
|
|
109
|
+
def evaluate(self) -> bool:
|
|
110
|
+
"""
|
|
111
|
+
Evaluate the auto-approve algorithm
|
|
112
|
+
"""
|
|
113
|
+
return self.get_configured_algorithm()(self.rule, self.did, self.session)
|
|
114
|
+
|
|
115
|
+
@classmethod
|
|
116
|
+
def get_configured_algorithm(cls: type[AutoApproveT]) -> Callable[[models.ReplicationRule, models.DataIdentifier, 'Session'], bool]:
|
|
117
|
+
"""
|
|
118
|
+
Get the configured auto-approve algorithm
|
|
119
|
+
"""
|
|
120
|
+
try:
|
|
121
|
+
configured_algorithm: str = str(config_get('rules', cls._algorithm_type, default='default'))
|
|
122
|
+
except (NoOptionError, NoSectionError, RuntimeError):
|
|
123
|
+
configured_algorithm = 'default'
|
|
124
|
+
|
|
125
|
+
return super()._get_one_algorithm(cls._algorithm_type, configured_algorithm)
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def register(cls: type[AutoApproveT], name: str, fn_auto_approve: Callable[[models.ReplicationRule, models.DataIdentifier, 'Session'], bool]) -> None:
|
|
129
|
+
"""
|
|
130
|
+
Register a new auto-approve algorithm
|
|
131
|
+
"""
|
|
132
|
+
algorithm_dict = {name: fn_auto_approve}
|
|
133
|
+
super()._register(cls._algorithm_type, algorithm_dict)
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def default(rule: models.ReplicationRule, did: models.DataIdentifier, session: 'Session') -> bool:
|
|
137
|
+
"""
|
|
138
|
+
Default auto-approve algorithm
|
|
139
|
+
"""
|
|
140
|
+
rse_expression = rule['rse_expression']
|
|
141
|
+
vo = rule['account'].vo
|
|
142
|
+
|
|
143
|
+
rses = parse_expression(rse_expression, filter_={'vo': vo}, session=session)
|
|
144
|
+
|
|
145
|
+
auto_approve = False
|
|
146
|
+
# Block manual approval for multi-rse rules
|
|
147
|
+
if len(rses) > 1:
|
|
148
|
+
raise InvalidReplicationRule('Ask approval is not allowed for rules with multiple RSEs')
|
|
149
|
+
if len(rses) == 1 and not did.is_open and did.bytes is not None and did.length is not None:
|
|
150
|
+
# This rule can be considered for auto-approval:
|
|
151
|
+
rse_attr = list_rse_attributes(rse_id=rses[0]['id'], session=session)
|
|
152
|
+
auto_approve = False
|
|
153
|
+
if RseAttr.AUTO_APPROVE_BYTES in rse_attr and RseAttr.AUTO_APPROVE_FILES in rse_attr:
|
|
154
|
+
if did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)) and did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES)):
|
|
155
|
+
auto_approve = True
|
|
156
|
+
elif did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES, -1)):
|
|
157
|
+
auto_approve = True
|
|
158
|
+
elif did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES, -1)):
|
|
159
|
+
auto_approve = True
|
|
160
|
+
|
|
161
|
+
return auto_approve
|
|
72
162
|
|
|
73
163
|
|
|
74
164
|
@transactional_session
|
|
75
|
-
def add_rule(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
165
|
+
def add_rule(
|
|
166
|
+
dids: Sequence[DIDDict],
|
|
167
|
+
account: InternalAccount,
|
|
168
|
+
copies: int,
|
|
169
|
+
rse_expression: str,
|
|
170
|
+
grouping: Literal['ALL', 'DATASET', 'NONE'],
|
|
171
|
+
weight: Optional[str],
|
|
172
|
+
lifetime: Optional[int],
|
|
173
|
+
locked: bool,
|
|
174
|
+
subscription_id: Optional[str],
|
|
175
|
+
source_replica_expression: Optional[str] = None,
|
|
176
|
+
activity: str = 'User Subscriptions',
|
|
177
|
+
notify: Optional[Literal['Y', 'N', 'C', 'P']] = None,
|
|
178
|
+
purge_replicas: bool = False,
|
|
179
|
+
ignore_availability: bool = False,
|
|
180
|
+
comment: Optional[str] = None,
|
|
181
|
+
ask_approval: bool = False,
|
|
182
|
+
asynchronous: bool = False,
|
|
183
|
+
ignore_account_limit: bool = False,
|
|
184
|
+
priority: int = 3,
|
|
185
|
+
delay_injection: Optional[int] = None,
|
|
186
|
+
split_container: bool = False,
|
|
187
|
+
meta: Optional[dict[str, Any]] = None,
|
|
188
|
+
*,
|
|
189
|
+
session: "Session",
|
|
190
|
+
logger: LoggerFunction = logging.log
|
|
191
|
+
) -> list[str]:
|
|
79
192
|
"""
|
|
80
193
|
Adds a replication rule for every did in dids
|
|
81
194
|
|
|
@@ -114,7 +227,7 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
114
227
|
|
|
115
228
|
rule_ids = []
|
|
116
229
|
|
|
117
|
-
|
|
230
|
+
grouping_value = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
|
|
118
231
|
|
|
119
232
|
with METRICS.timer('add_rule.total'):
|
|
120
233
|
# 1. Resolve the rse_expression into a list of RSE-ids
|
|
@@ -143,14 +256,14 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
143
256
|
# Block manual approval if RSE does not allow it
|
|
144
257
|
if ask_approval:
|
|
145
258
|
for rse in rses:
|
|
146
|
-
if list_rse_attributes(rse_id=rse['id'], session=session).get(
|
|
259
|
+
if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.BLOCK_MANUAL_APPROVAL, False):
|
|
147
260
|
raise ManualRuleApprovalBlocked()
|
|
148
261
|
|
|
149
262
|
if source_replica_expression:
|
|
150
263
|
try:
|
|
151
264
|
source_rses = parse_expression(source_replica_expression, filter_={'vo': vo}, session=session)
|
|
152
|
-
except InvalidRSEExpression:
|
|
153
|
-
raise InvalidSourceReplicaExpression
|
|
265
|
+
except InvalidRSEExpression as exc:
|
|
266
|
+
raise InvalidSourceReplicaExpression from exc
|
|
154
267
|
else:
|
|
155
268
|
source_rses = []
|
|
156
269
|
|
|
@@ -160,43 +273,69 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
160
273
|
|
|
161
274
|
expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None
|
|
162
275
|
|
|
163
|
-
|
|
276
|
+
notify_value = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(notify or '', RuleNotification.NO)
|
|
164
277
|
|
|
165
278
|
for elem in dids:
|
|
166
279
|
# 3. Get the did
|
|
167
280
|
with METRICS.timer('add_rule.get_did'):
|
|
168
281
|
try:
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
282
|
+
stmt = select(
|
|
283
|
+
models.DataIdentifier
|
|
284
|
+
).where(
|
|
285
|
+
and_(models.DataIdentifier.scope == elem['scope'],
|
|
286
|
+
models.DataIdentifier.name == elem['name'])
|
|
287
|
+
)
|
|
288
|
+
did = session.execute(stmt).scalar_one()
|
|
289
|
+
except NoResultFound as exc:
|
|
290
|
+
raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
|
|
173
291
|
except TypeError as error:
|
|
174
|
-
raise InvalidObject(error.args)
|
|
292
|
+
raise InvalidObject(error.args) from error # https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/raise-missing-from.html
|
|
175
293
|
|
|
176
294
|
# 3.1 If the did is a constituent, relay the rule to the archive
|
|
177
295
|
if did.did_type == DIDType.FILE and did.constituent:
|
|
178
296
|
# Check if a single replica of this DID exists; Do not put rule on file if there are only replicas on TAPE
|
|
179
|
-
|
|
180
|
-
.
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
297
|
+
stmt = select(
|
|
298
|
+
func.count()
|
|
299
|
+
).select_from(
|
|
300
|
+
models.RSEFileAssociation
|
|
301
|
+
).join(
|
|
302
|
+
models.RSE,
|
|
303
|
+
models.RSEFileAssociation.rse_id == models.RSE.id
|
|
304
|
+
).where(
|
|
305
|
+
and_(models.RSEFileAssociation.scope == did.scope,
|
|
306
|
+
models.RSEFileAssociation.name == did.name,
|
|
307
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
308
|
+
models.RSE.rse_type != RSEType.TAPE)
|
|
309
|
+
)
|
|
310
|
+
replica_cnt = session.execute(stmt).scalar()
|
|
311
|
+
|
|
184
312
|
if replica_cnt == 0: # Put the rule on the archive
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
313
|
+
stmt = select(
|
|
314
|
+
models.ConstituentAssociation
|
|
315
|
+
).join(
|
|
316
|
+
models.RSEFileAssociation,
|
|
317
|
+
and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
|
|
318
|
+
models.ConstituentAssociation.name == models.RSEFileAssociation.name)
|
|
319
|
+
).where(
|
|
320
|
+
and_(models.ConstituentAssociation.child_scope == did.scope,
|
|
321
|
+
models.ConstituentAssociation.child_name == did.name)
|
|
322
|
+
)
|
|
323
|
+
archive = session.execute(stmt).scalars().first()
|
|
190
324
|
if archive is not None:
|
|
191
325
|
elem['name'] = archive.name
|
|
192
326
|
elem['scope'] = archive.scope
|
|
193
327
|
try:
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
328
|
+
stmt = select(
|
|
329
|
+
models.DataIdentifier
|
|
330
|
+
).where(
|
|
331
|
+
and_(models.DataIdentifier.scope == elem['scope'],
|
|
332
|
+
models.DataIdentifier.name == elem['name'])
|
|
333
|
+
)
|
|
334
|
+
did = session.execute(stmt).scalar_one()
|
|
335
|
+
except NoResultFound as exc:
|
|
336
|
+
raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
|
|
198
337
|
except TypeError as error:
|
|
199
|
-
raise InvalidObject(error.args)
|
|
338
|
+
raise InvalidObject(error.args) from error
|
|
200
339
|
else: # Put the rule on the constituent directly
|
|
201
340
|
pass
|
|
202
341
|
|
|
@@ -219,13 +358,13 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
219
358
|
copies=copies,
|
|
220
359
|
rse_expression=rse_expression,
|
|
221
360
|
locked=locked,
|
|
222
|
-
grouping=
|
|
361
|
+
grouping=grouping_value,
|
|
223
362
|
expires_at=expires_at,
|
|
224
363
|
weight=weight,
|
|
225
364
|
source_replica_expression=source_replica_expression,
|
|
226
365
|
activity=activity,
|
|
227
366
|
subscription_id=subscription_id,
|
|
228
|
-
notification=
|
|
367
|
+
notification=notify_value,
|
|
229
368
|
purge_replicas=purge_replicas,
|
|
230
369
|
ignore_availability=ignore_availability,
|
|
231
370
|
comments=comment,
|
|
@@ -243,31 +382,20 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
243
382
|
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
|
|
244
383
|
or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
|
|
245
384
|
or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]):
|
|
246
|
-
raise DuplicateRule(error.args[0])
|
|
247
|
-
raise InvalidReplicationRule(error.args[0])
|
|
385
|
+
raise DuplicateRule(error.args[0]) from error
|
|
386
|
+
raise InvalidReplicationRule(error.args[0]) from error
|
|
248
387
|
rule_ids.append(new_rule.id)
|
|
249
388
|
|
|
250
389
|
if ask_approval:
|
|
251
390
|
new_rule.state = RuleState.WAITING_APPROVAL
|
|
252
|
-
#
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
if did.bytes < int(rse_attr.get('auto_approve_bytes')) and did.length < int(rse_attr.get('auto_approve_bytes')):
|
|
261
|
-
auto_approve = True
|
|
262
|
-
elif did.bytes < int(rse_attr.get('auto_approve_bytes', -1)):
|
|
263
|
-
auto_approve = True
|
|
264
|
-
elif did.length < int(rse_attr.get('auto_approve_files', -1)):
|
|
265
|
-
auto_approve = True
|
|
266
|
-
if auto_approve:
|
|
267
|
-
logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
|
|
268
|
-
logger(logging.DEBUG, "Created rule %s for injection", str(new_rule.id))
|
|
269
|
-
approve_rule(rule_id=new_rule.id, notify_approvers=False, session=session)
|
|
270
|
-
continue
|
|
391
|
+
# Use the new rule as the argument here
|
|
392
|
+
auto_approver = AutoApprove(new_rule, did, session=session)
|
|
393
|
+
if auto_approver.evaluate():
|
|
394
|
+
logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
|
|
395
|
+
logger(logging.DEBUG, "Created rule %s for injection", str(new_rule.id))
|
|
396
|
+
approve_rule(rule_id=new_rule.id, notify_approvers=False, session=session)
|
|
397
|
+
continue
|
|
398
|
+
|
|
271
399
|
logger(logging.DEBUG, "Created rule %s in waiting for approval", str(new_rule.id))
|
|
272
400
|
__create_rule_approval_email(rule=new_rule, session=session)
|
|
273
401
|
continue
|
|
@@ -297,17 +425,31 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
297
425
|
try:
|
|
298
426
|
apply_rule(did, new_rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
|
|
299
427
|
except IntegrityError as error:
|
|
300
|
-
raise ReplicationRuleCreationTemporaryFailed(error.args[0])
|
|
428
|
+
raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
|
|
301
429
|
|
|
302
430
|
if new_rule.locks_stuck_cnt > 0:
|
|
303
431
|
new_rule.state = RuleState.STUCK
|
|
304
432
|
new_rule.error = 'MissingSourceReplica'
|
|
305
433
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
306
|
-
|
|
434
|
+
stmt = update(
|
|
435
|
+
models.DatasetLock
|
|
436
|
+
).where(
|
|
437
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
438
|
+
).values({
|
|
439
|
+
models.DatasetLock.state: LockState.STUCK
|
|
440
|
+
})
|
|
441
|
+
session.execute(stmt)
|
|
307
442
|
elif new_rule.locks_replicating_cnt == 0:
|
|
308
443
|
new_rule.state = RuleState.OK
|
|
309
444
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
310
|
-
|
|
445
|
+
stmt = update(
|
|
446
|
+
models.DatasetLock
|
|
447
|
+
).where(
|
|
448
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
449
|
+
).values({
|
|
450
|
+
models.DatasetLock.state: LockState.OK
|
|
451
|
+
})
|
|
452
|
+
session.execute(stmt)
|
|
311
453
|
session.flush()
|
|
312
454
|
if new_rule.notification == RuleNotification.YES:
|
|
313
455
|
generate_email_for_rule_ok_notification(rule=new_rule, session=session)
|
|
@@ -315,7 +457,14 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
315
457
|
else:
|
|
316
458
|
new_rule.state = RuleState.REPLICATING
|
|
317
459
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
318
|
-
|
|
460
|
+
stmt = update(
|
|
461
|
+
models.DatasetLock
|
|
462
|
+
).where(
|
|
463
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
464
|
+
).values({
|
|
465
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
466
|
+
})
|
|
467
|
+
session.execute(stmt)
|
|
319
468
|
|
|
320
469
|
# Add rule to History
|
|
321
470
|
insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
|
|
@@ -327,13 +476,19 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
|
|
|
327
476
|
|
|
328
477
|
|
|
329
478
|
@transactional_session
|
|
330
|
-
def add_rules(
|
|
479
|
+
def add_rules(
|
|
480
|
+
dids: Sequence[DIDDict],
|
|
481
|
+
rules: Sequence[RuleDict],
|
|
482
|
+
*,
|
|
483
|
+
session: "Session",
|
|
484
|
+
logger: LoggerFunction = logging.log
|
|
485
|
+
) -> dict[tuple[InternalScope, str], list[str]]:
|
|
331
486
|
"""
|
|
332
487
|
Adds a list of replication rules to every did in dids
|
|
333
488
|
|
|
334
489
|
:params dids: List of data identifiers.
|
|
335
490
|
:param rules: List of dictionaries defining replication rules.
|
|
336
|
-
{account, copies, rse_expression, grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity,
|
|
491
|
+
{account, copies, rse_expression, grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas}
|
|
337
492
|
:param session: The database session in use.
|
|
338
493
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
339
494
|
:returns: Dictionary (scope, name) with list of created rule ids
|
|
@@ -368,37 +523,61 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
368
523
|
# 2. Get the did
|
|
369
524
|
with METRICS.timer('add_rules.get_did'):
|
|
370
525
|
try:
|
|
371
|
-
|
|
372
|
-
models.DataIdentifier
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
526
|
+
stmt = select(
|
|
527
|
+
models.DataIdentifier
|
|
528
|
+
).where(
|
|
529
|
+
and_(models.DataIdentifier.scope == elem['scope'],
|
|
530
|
+
models.DataIdentifier.name == elem['name'])
|
|
531
|
+
)
|
|
532
|
+
did = session.execute(stmt).scalar_one()
|
|
533
|
+
except NoResultFound as exc:
|
|
534
|
+
raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
|
|
376
535
|
except TypeError as error:
|
|
377
|
-
raise InvalidObject(error.args)
|
|
536
|
+
raise InvalidObject(error.args) from error
|
|
378
537
|
|
|
379
538
|
# 2.1 If the did is a constituent, relay the rule to the archive
|
|
380
539
|
if did.did_type == DIDType.FILE and did.constituent: # Check if a single replica of this DID exists
|
|
381
|
-
|
|
382
|
-
.
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
540
|
+
stmt = select(
|
|
541
|
+
func.count()
|
|
542
|
+
).select_from(
|
|
543
|
+
models.RSEFileAssociation
|
|
544
|
+
).join(
|
|
545
|
+
models.RSE,
|
|
546
|
+
models.RSEFileAssociation.rse_id == models.RSE.id
|
|
547
|
+
).where(
|
|
548
|
+
and_(models.RSEFileAssociation.scope == did.scope,
|
|
549
|
+
models.RSEFileAssociation.name == did.name,
|
|
550
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
551
|
+
models.RSE.rse_type != RSEType.TAPE)
|
|
552
|
+
)
|
|
553
|
+
replica_cnt = session.execute(stmt).scalar()
|
|
386
554
|
if replica_cnt == 0: # Put the rule on the archive
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
555
|
+
stmt = select(
|
|
556
|
+
models.ConstituentAssociation
|
|
557
|
+
).join(
|
|
558
|
+
models.RSEFileAssociation,
|
|
559
|
+
and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
|
|
560
|
+
models.ConstituentAssociation.name == models.RSEFileAssociation.name)
|
|
561
|
+
).where(
|
|
562
|
+
and_(models.ConstituentAssociation.child_scope == did.scope,
|
|
563
|
+
models.ConstituentAssociation.child_name == did.name)
|
|
564
|
+
)
|
|
565
|
+
archive = session.execute(stmt).scalars().first()
|
|
392
566
|
if archive is not None:
|
|
393
567
|
elem['name'] = archive.name
|
|
394
568
|
elem['scope'] = archive.scope
|
|
395
569
|
try:
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
570
|
+
stmt = select(
|
|
571
|
+
models.DataIdentifier
|
|
572
|
+
).where(
|
|
573
|
+
and_(models.DataIdentifier.scope == elem['scope'],
|
|
574
|
+
models.DataIdentifier.name == elem['name'])
|
|
575
|
+
)
|
|
576
|
+
did = session.execute(stmt).scalar_one()
|
|
577
|
+
except NoResultFound as exc:
|
|
578
|
+
raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
|
|
400
579
|
except TypeError as error:
|
|
401
|
-
raise InvalidObject(error.args)
|
|
580
|
+
raise InvalidObject(error.args) from error
|
|
402
581
|
else: # Put the rule on the constituent directly
|
|
403
582
|
pass
|
|
404
583
|
|
|
@@ -445,7 +624,7 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
445
624
|
# Block manual approval if RSE does not allow it
|
|
446
625
|
if rule.get('ask_approval', False):
|
|
447
626
|
for rse in rses:
|
|
448
|
-
if list_rse_attributes(rse_id=rse['id'], session=session).get(
|
|
627
|
+
if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.BLOCK_MANUAL_APPROVAL, False):
|
|
449
628
|
raise ManualRuleApprovalBlocked()
|
|
450
629
|
|
|
451
630
|
if rule.get('source_replica_expression'):
|
|
@@ -459,11 +638,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
459
638
|
|
|
460
639
|
# 4. Create the replication rule
|
|
461
640
|
with METRICS.timer('add_rules.create_rule'):
|
|
462
|
-
grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(rule.get('grouping'), RuleGrouping.DATASET)
|
|
641
|
+
grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(str(rule.get('grouping')), RuleGrouping.DATASET)
|
|
463
642
|
|
|
464
|
-
|
|
643
|
+
rule_lifetime: Optional[int] = rule.get('lifetime')
|
|
644
|
+
expires_at: Optional[datetime] = datetime.utcnow() + timedelta(seconds=rule_lifetime) if rule_lifetime is not None else None
|
|
465
645
|
|
|
466
|
-
notify = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(rule.get('notify')
|
|
646
|
+
notify = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS, None: RuleNotification.NO}.get(rule.get('notify'))
|
|
467
647
|
|
|
468
648
|
if rule.get('meta') is not None:
|
|
469
649
|
try:
|
|
@@ -498,10 +678,10 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
498
678
|
new_rule.save(session=session)
|
|
499
679
|
except IntegrityError as error:
|
|
500
680
|
if match('.*ORA-00001.*', str(error.args[0])):
|
|
501
|
-
raise DuplicateRule(error.args[0])
|
|
681
|
+
raise DuplicateRule(error.args[0]) from error
|
|
502
682
|
elif str(error.args[0]) == '(IntegrityError) UNIQUE constraint failed: rules.scope, rules.name, rules.account, rules.rse_expression, rules.copies':
|
|
503
|
-
raise DuplicateRule(error.args[0])
|
|
504
|
-
raise InvalidReplicationRule(error.args[0])
|
|
683
|
+
raise DuplicateRule(error.args[0]) from error
|
|
684
|
+
raise InvalidReplicationRule(error.args[0]) from error
|
|
505
685
|
|
|
506
686
|
rule_ids[(did.scope, did.name)].append(new_rule.id)
|
|
507
687
|
|
|
@@ -514,12 +694,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
514
694
|
# This rule can be considered for auto-approval:
|
|
515
695
|
rse_attr = list_rse_attributes(rse_id=rses[0]['id'], session=session)
|
|
516
696
|
auto_approve = False
|
|
517
|
-
if
|
|
518
|
-
if did.bytes < int(rse_attr.get(
|
|
697
|
+
if RseAttr.AUTO_APPROVE_BYTES in rse_attr and RseAttr.AUTO_APPROVE_FILES in rse_attr:
|
|
698
|
+
if did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)) and did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)):
|
|
519
699
|
auto_approve = True
|
|
520
|
-
elif did.bytes < int(rse_attr.get(
|
|
700
|
+
elif did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES, -1)):
|
|
521
701
|
auto_approve = True
|
|
522
|
-
elif did.length < int(rse_attr.get(
|
|
702
|
+
elif did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES, -1)):
|
|
523
703
|
auto_approve = True
|
|
524
704
|
if auto_approve:
|
|
525
705
|
logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
|
|
@@ -557,17 +737,31 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
557
737
|
source_rses=[rse['id'] for rse in source_rses],
|
|
558
738
|
session=session)
|
|
559
739
|
except IntegrityError as error:
|
|
560
|
-
raise ReplicationRuleCreationTemporaryFailed(error.args[0])
|
|
740
|
+
raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
|
|
561
741
|
|
|
562
742
|
if new_rule.locks_stuck_cnt > 0:
|
|
563
743
|
new_rule.state = RuleState.STUCK
|
|
564
744
|
new_rule.error = 'MissingSourceReplica'
|
|
565
745
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
566
|
-
|
|
746
|
+
stmt = update(
|
|
747
|
+
models.DatasetLock
|
|
748
|
+
).where(
|
|
749
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
750
|
+
).values({
|
|
751
|
+
models.DatasetLock.state: LockState.STUCK
|
|
752
|
+
})
|
|
753
|
+
session.execute(stmt)
|
|
567
754
|
elif new_rule.locks_replicating_cnt == 0:
|
|
568
755
|
new_rule.state = RuleState.OK
|
|
569
756
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
570
|
-
|
|
757
|
+
stmt = update(
|
|
758
|
+
models.DatasetLock
|
|
759
|
+
).where(
|
|
760
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
761
|
+
).values({
|
|
762
|
+
models.DatasetLock.state: LockState.OK
|
|
763
|
+
})
|
|
764
|
+
session.execute(stmt)
|
|
571
765
|
session.flush()
|
|
572
766
|
if new_rule.notification == RuleNotification.YES:
|
|
573
767
|
generate_email_for_rule_ok_notification(rule=new_rule, session=session)
|
|
@@ -575,7 +769,14 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
575
769
|
else:
|
|
576
770
|
new_rule.state = RuleState.REPLICATING
|
|
577
771
|
if new_rule.grouping != RuleGrouping.NONE:
|
|
578
|
-
|
|
772
|
+
stmt = update(
|
|
773
|
+
models.DatasetLock
|
|
774
|
+
).where(
|
|
775
|
+
models.DatasetLock.rule_id == new_rule.id
|
|
776
|
+
).values({
|
|
777
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
778
|
+
})
|
|
779
|
+
session.execute(stmt)
|
|
579
780
|
|
|
580
781
|
# Add rule to History
|
|
581
782
|
insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
|
|
@@ -586,7 +787,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
|
|
|
586
787
|
|
|
587
788
|
|
|
588
789
|
@transactional_session
|
|
589
|
-
def inject_rule(
|
|
790
|
+
def inject_rule(
|
|
791
|
+
rule_id: str,
|
|
792
|
+
*,
|
|
793
|
+
session: "Session",
|
|
794
|
+
logger: LoggerFunction = logging.log
|
|
795
|
+
) -> None:
|
|
590
796
|
"""
|
|
591
797
|
Inject a replication rule.
|
|
592
798
|
|
|
@@ -598,9 +804,16 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
598
804
|
"""
|
|
599
805
|
|
|
600
806
|
try:
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
807
|
+
stmt = select(
|
|
808
|
+
models.ReplicationRule
|
|
809
|
+
).where(
|
|
810
|
+
models.ReplicationRule.id == rule_id
|
|
811
|
+
).with_for_update(
|
|
812
|
+
nowait=True
|
|
813
|
+
)
|
|
814
|
+
rule = session.execute(stmt).scalar_one()
|
|
815
|
+
except NoResultFound as exc:
|
|
816
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
604
817
|
|
|
605
818
|
# Check if rule will expire in the next 5 minutes:
|
|
606
819
|
if rule.child_rule_id is None and rule.expires_at is not None and rule.expires_at < datetime.utcnow() + timedelta(seconds=300):
|
|
@@ -615,7 +828,13 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
615
828
|
# Remove duplicates from the list of dictionaries
|
|
616
829
|
dids = [dict(t) for t in {tuple(d.items()) for d in dids}]
|
|
617
830
|
# Remove dids which already have a similar rule
|
|
618
|
-
|
|
831
|
+
stmt = select(
|
|
832
|
+
models.ReplicationRule.id
|
|
833
|
+
).where(
|
|
834
|
+
and_(models.ReplicationRule.account == rule.account,
|
|
835
|
+
models.ReplicationRule.rse_expression == rule.rse_expression)
|
|
836
|
+
)
|
|
837
|
+
dids = [did for did in dids if session.execute(stmt.where(and_(models.ReplicationRule.scope == did['scope'], models.ReplicationRule.name == did['name']))).scalar_one_or_none() is None]
|
|
619
838
|
if rule.expires_at:
|
|
620
839
|
lifetime = (rule.expires_at - datetime.utcnow()).days * 24 * 3600 + (rule.expires_at - datetime.utcnow()).seconds
|
|
621
840
|
else:
|
|
@@ -664,29 +883,48 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
664
883
|
# 3. Get the did
|
|
665
884
|
with METRICS.timer('inject_rule.get_did'):
|
|
666
885
|
try:
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
886
|
+
stmt = select(
|
|
887
|
+
models.DataIdentifier
|
|
888
|
+
).where(
|
|
889
|
+
and_(models.DataIdentifier.scope == rule.scope,
|
|
890
|
+
models.DataIdentifier.name == rule.name)
|
|
891
|
+
)
|
|
892
|
+
did = session.execute(stmt).scalar_one()
|
|
893
|
+
except NoResultFound as exc:
|
|
894
|
+
raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (rule.scope, rule.name)) from exc
|
|
671
895
|
except TypeError as error:
|
|
672
|
-
raise InvalidObject(error.args)
|
|
896
|
+
raise InvalidObject(error.args) from error
|
|
673
897
|
|
|
674
898
|
# 4. Apply the rule
|
|
675
899
|
with METRICS.timer('inject_rule.apply_rule'):
|
|
676
900
|
try:
|
|
677
901
|
apply_rule(did, rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
|
|
678
902
|
except IntegrityError as error:
|
|
679
|
-
raise ReplicationRuleCreationTemporaryFailed(error.args[0])
|
|
903
|
+
raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
|
|
680
904
|
|
|
681
905
|
if rule.locks_stuck_cnt > 0:
|
|
682
906
|
rule.state = RuleState.STUCK
|
|
683
907
|
rule.error = 'MissingSourceReplica'
|
|
684
908
|
if rule.grouping != RuleGrouping.NONE:
|
|
685
|
-
|
|
909
|
+
stmt = update(
|
|
910
|
+
models.DatasetLock
|
|
911
|
+
).where(
|
|
912
|
+
models.DatasetLock.rule_id == rule.id
|
|
913
|
+
).values({
|
|
914
|
+
models.DatasetLock.state: LockState.STUCK
|
|
915
|
+
})
|
|
916
|
+
session.execute(stmt)
|
|
686
917
|
elif rule.locks_replicating_cnt == 0:
|
|
687
918
|
rule.state = RuleState.OK
|
|
688
919
|
if rule.grouping != RuleGrouping.NONE:
|
|
689
|
-
|
|
920
|
+
stmt = update(
|
|
921
|
+
models.DatasetLock
|
|
922
|
+
).where(
|
|
923
|
+
models.DatasetLock.rule_id == rule.id
|
|
924
|
+
).values({
|
|
925
|
+
models.DatasetLock.state: LockState.OK
|
|
926
|
+
})
|
|
927
|
+
session.execute(stmt)
|
|
690
928
|
session.flush()
|
|
691
929
|
if rule.notification == RuleNotification.YES:
|
|
692
930
|
generate_email_for_rule_ok_notification(rule=rule, session=session)
|
|
@@ -696,7 +934,14 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
696
934
|
else:
|
|
697
935
|
rule.state = RuleState.REPLICATING
|
|
698
936
|
if rule.grouping != RuleGrouping.NONE:
|
|
699
|
-
|
|
937
|
+
stmt = update(
|
|
938
|
+
models.DatasetLock
|
|
939
|
+
).where(
|
|
940
|
+
models.DatasetLock.rule_id == rule.id
|
|
941
|
+
).values({
|
|
942
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
943
|
+
})
|
|
944
|
+
session.execute(stmt)
|
|
700
945
|
|
|
701
946
|
# Add rule to History
|
|
702
947
|
insert_rule_history(rule=rule, recent=True, longterm=True, session=session)
|
|
@@ -705,7 +950,11 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
705
950
|
|
|
706
951
|
|
|
707
952
|
@stream_session
|
|
708
|
-
def list_rules(
|
|
953
|
+
def list_rules(
|
|
954
|
+
filters: Optional[dict[str, Any]] = None,
|
|
955
|
+
*,
|
|
956
|
+
session: "Session"
|
|
957
|
+
) -> Iterator[dict[str, Any]]:
|
|
709
958
|
"""
|
|
710
959
|
List replication rules.
|
|
711
960
|
|
|
@@ -724,7 +973,7 @@ def list_rules(filters={}, *, session: "Session"):
|
|
|
724
973
|
models.ReplicationRule.name == models.DataIdentifier.name
|
|
725
974
|
)
|
|
726
975
|
)
|
|
727
|
-
if filters:
|
|
976
|
+
if filters is not None:
|
|
728
977
|
for (key, value) in filters.items():
|
|
729
978
|
if key in ['account', 'scope']:
|
|
730
979
|
if '*' in value.internal:
|
|
@@ -763,12 +1012,16 @@ def list_rules(filters={}, *, session: "Session"):
|
|
|
763
1012
|
d = rule.to_dict()
|
|
764
1013
|
d['bytes'] = data_identifier_bytes
|
|
765
1014
|
yield d
|
|
766
|
-
except StatementError:
|
|
767
|
-
raise RucioException('Badly formatted input (IDs?)')
|
|
1015
|
+
except StatementError as exc:
|
|
1016
|
+
raise RucioException('Badly formatted input (IDs?)') from exc
|
|
768
1017
|
|
|
769
1018
|
|
|
770
1019
|
@stream_session
|
|
771
|
-
def list_rule_history(
|
|
1020
|
+
def list_rule_history(
|
|
1021
|
+
rule_id: str,
|
|
1022
|
+
*,
|
|
1023
|
+
session: "Session"
|
|
1024
|
+
) -> Iterator[dict[str, Any]]:
|
|
772
1025
|
"""
|
|
773
1026
|
List the rule history of a rule.
|
|
774
1027
|
|
|
@@ -777,21 +1030,32 @@ def list_rule_history(rule_id, *, session: "Session"):
|
|
|
777
1030
|
:raises: RucioException
|
|
778
1031
|
"""
|
|
779
1032
|
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
1033
|
+
stmt = select(
|
|
1034
|
+
models.ReplicationRuleHistory.updated_at,
|
|
1035
|
+
models.ReplicationRuleHistory.state,
|
|
1036
|
+
models.ReplicationRuleHistory.locks_ok_cnt,
|
|
1037
|
+
models.ReplicationRuleHistory.locks_stuck_cnt,
|
|
1038
|
+
models.ReplicationRuleHistory.locks_replicating_cnt
|
|
1039
|
+
).where(
|
|
1040
|
+
models.ReplicationRuleHistory.id == rule_id
|
|
1041
|
+
).order_by(
|
|
1042
|
+
models.ReplicationRuleHistory.updated_at
|
|
1043
|
+
)
|
|
785
1044
|
|
|
786
1045
|
try:
|
|
787
|
-
for rule in
|
|
788
|
-
yield
|
|
789
|
-
except StatementError:
|
|
790
|
-
raise RucioException('Badly formatted input (IDs?)')
|
|
1046
|
+
for rule in session.execute(stmt).yield_per(5):
|
|
1047
|
+
yield rule._asdict()
|
|
1048
|
+
except StatementError as exc:
|
|
1049
|
+
raise RucioException('Badly formatted input (IDs?)') from exc
|
|
791
1050
|
|
|
792
1051
|
|
|
793
1052
|
@stream_session
|
|
794
|
-
def list_rule_full_history(
|
|
1053
|
+
def list_rule_full_history(
|
|
1054
|
+
scope: InternalScope,
|
|
1055
|
+
name: str,
|
|
1056
|
+
*,
|
|
1057
|
+
session: "Session"
|
|
1058
|
+
) -> Iterator[dict[str, Any]]:
|
|
795
1059
|
"""
|
|
796
1060
|
List the rule history of a DID.
|
|
797
1061
|
|
|
@@ -801,26 +1065,36 @@ def list_rule_full_history(scope, name, *, session: "Session"):
|
|
|
801
1065
|
:raises: RucioException
|
|
802
1066
|
"""
|
|
803
1067
|
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
1068
|
+
stmt = select(
|
|
1069
|
+
models.ReplicationRuleHistory.id.label('rule_id'),
|
|
1070
|
+
models.ReplicationRuleHistory.created_at,
|
|
1071
|
+
models.ReplicationRuleHistory.updated_at,
|
|
1072
|
+
models.ReplicationRuleHistory.rse_expression,
|
|
1073
|
+
models.ReplicationRuleHistory.state,
|
|
1074
|
+
models.ReplicationRuleHistory.account,
|
|
1075
|
+
models.ReplicationRuleHistory.locks_ok_cnt,
|
|
1076
|
+
models.ReplicationRuleHistory.locks_stuck_cnt,
|
|
1077
|
+
models.ReplicationRuleHistory.locks_replicating_cnt
|
|
1078
|
+
).with_hint(
|
|
1079
|
+
models.ReplicationRuleHistory, 'INDEX(RULES_HISTORY_SCOPENAME_IDX)', 'oracle'
|
|
1080
|
+
).where(
|
|
1081
|
+
and_(models.ReplicationRuleHistory.scope == scope,
|
|
1082
|
+
models.ReplicationRuleHistory.name == name)
|
|
1083
|
+
).order_by(
|
|
1084
|
+
models.ReplicationRuleHistory.created_at,
|
|
1085
|
+
models.ReplicationRuleHistory.updated_at
|
|
1086
|
+
)
|
|
1087
|
+
for rule in session.execute(stmt).yield_per(5):
|
|
1088
|
+
yield rule._asdict()
|
|
820
1089
|
|
|
821
1090
|
|
|
822
1091
|
@stream_session
|
|
823
|
-
def list_associated_rules_for_file(
|
|
1092
|
+
def list_associated_rules_for_file(
|
|
1093
|
+
scope: InternalScope,
|
|
1094
|
+
name: str,
|
|
1095
|
+
*,
|
|
1096
|
+
session: "Session"
|
|
1097
|
+
) -> Iterator[dict[str, Any]]:
|
|
824
1098
|
"""
|
|
825
1099
|
List replication rules a file is affected from.
|
|
826
1100
|
|
|
@@ -829,22 +1103,44 @@ def list_associated_rules_for_file(scope, name, *, session: "Session"):
|
|
|
829
1103
|
:param session: The database session in use.
|
|
830
1104
|
:raises: RucioException
|
|
831
1105
|
"""
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
1106
|
+
rucio.core.did.get_did(scope=scope, name=name, session=session) # Check if the did actually exists
|
|
1107
|
+
stmt = select(
|
|
1108
|
+
models.ReplicationRule,
|
|
1109
|
+
models.DataIdentifier.bytes
|
|
1110
|
+
).distinct(
|
|
1111
|
+
).join(
|
|
1112
|
+
models.ReplicaLock,
|
|
1113
|
+
models.ReplicationRule.id == models.ReplicaLock.rule_id
|
|
1114
|
+
).join(
|
|
1115
|
+
models.DataIdentifier,
|
|
1116
|
+
and_(models.ReplicationRule.scope == models.DataIdentifier.scope,
|
|
1117
|
+
models.ReplicationRule.name == models.DataIdentifier.name)
|
|
1118
|
+
).with_hint(
|
|
1119
|
+
models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
|
|
1120
|
+
).where(
|
|
1121
|
+
and_(models.ReplicaLock.scope == scope,
|
|
1122
|
+
models.ReplicaLock.name == name)
|
|
1123
|
+
)
|
|
838
1124
|
try:
|
|
839
|
-
for rule in
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1125
|
+
for rule, data_identifier_bytes in session.execute(stmt).yield_per(5):
|
|
1126
|
+
d = rule.to_dict()
|
|
1127
|
+
d['bytes'] = data_identifier_bytes
|
|
1128
|
+
yield d
|
|
1129
|
+
except StatementError as exc:
|
|
1130
|
+
raise RucioException('Badly formatted input (IDs?)') from exc
|
|
843
1131
|
|
|
844
1132
|
|
|
845
1133
|
@transactional_session
|
|
846
|
-
def delete_rule(
|
|
847
|
-
|
|
1134
|
+
def delete_rule(
|
|
1135
|
+
rule_id: str,
|
|
1136
|
+
purge_replicas: Optional[bool] = None,
|
|
1137
|
+
soft: bool = False,
|
|
1138
|
+
delete_parent: bool = False,
|
|
1139
|
+
nowait: bool = False,
|
|
1140
|
+
*,
|
|
1141
|
+
session: "Session",
|
|
1142
|
+
ignore_rule_lock: bool = False
|
|
1143
|
+
) -> None:
|
|
848
1144
|
"""
|
|
849
1145
|
Delete a replication rule.
|
|
850
1146
|
|
|
@@ -861,11 +1157,16 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
|
|
|
861
1157
|
|
|
862
1158
|
with METRICS.timer('delete_rule.total'):
|
|
863
1159
|
try:
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
1160
|
+
stmt = select(
|
|
1161
|
+
models.ReplicationRule
|
|
1162
|
+
).where(
|
|
1163
|
+
models.ReplicationRule.id == rule_id
|
|
1164
|
+
).with_for_update(
|
|
1165
|
+
nowait=nowait
|
|
1166
|
+
)
|
|
1167
|
+
rule = session.execute(stmt).scalar_one()
|
|
1168
|
+
except NoResultFound as exc:
|
|
1169
|
+
raise RuleNotFound('No rule with the id %s found' % rule_id) from exc
|
|
869
1170
|
if rule.locked and not ignore_rule_lock:
|
|
870
1171
|
raise UnsupportedOperation('The replication rule is locked and has to be unlocked before it can be deleted.')
|
|
871
1172
|
|
|
@@ -885,15 +1186,21 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
|
|
|
885
1186
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
886
1187
|
return
|
|
887
1188
|
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1189
|
+
stmt = select(
|
|
1190
|
+
models.ReplicaLock
|
|
1191
|
+
).where(
|
|
1192
|
+
models.ReplicaLock.rule_id == rule_id
|
|
1193
|
+
).with_for_update(
|
|
1194
|
+
nowait=nowait
|
|
1195
|
+
)
|
|
1196
|
+
results = session.execute(stmt).yield_per(100)
|
|
891
1197
|
|
|
892
1198
|
# Remove locks, set tombstone if applicable
|
|
893
1199
|
transfers_to_delete = [] # [{'scope': , 'name':, 'rse_id':}]
|
|
894
1200
|
account_counter_decreases = {} # {'rse_id': [file_size, file_size, file_size]}
|
|
895
1201
|
|
|
896
|
-
for
|
|
1202
|
+
for result in results:
|
|
1203
|
+
lock = result[0]
|
|
897
1204
|
if __delete_lock_and_update_replica(lock=lock, purge_replicas=rule.purge_replicas,
|
|
898
1205
|
nowait=nowait, session=session):
|
|
899
1206
|
transfers_to_delete.append({'scope': lock.scope, 'name': lock.name, 'rse_id': lock.rse_id})
|
|
@@ -902,9 +1209,14 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
|
|
|
902
1209
|
account_counter_decreases[lock.rse_id].append(lock.bytes)
|
|
903
1210
|
|
|
904
1211
|
# Delete the DatasetLocks
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
1212
|
+
stmt = delete(
|
|
1213
|
+
models.DatasetLock
|
|
1214
|
+
).where(
|
|
1215
|
+
models.DatasetLock.rule_id == rule_id
|
|
1216
|
+
).execution_options(
|
|
1217
|
+
synchronize_session=False
|
|
1218
|
+
)
|
|
1219
|
+
session.execute(stmt)
|
|
908
1220
|
|
|
909
1221
|
# Decrease account_counters
|
|
910
1222
|
for rse_id in account_counter_decreases.keys():
|
|
@@ -927,7 +1239,12 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
|
|
|
927
1239
|
|
|
928
1240
|
|
|
929
1241
|
@transactional_session
|
|
930
|
-
def repair_rule(
|
|
1242
|
+
def repair_rule(
|
|
1243
|
+
rule_id: str,
|
|
1244
|
+
*,
|
|
1245
|
+
session: "Session",
|
|
1246
|
+
logger: LoggerFunction = logging.log
|
|
1247
|
+
) -> None:
|
|
931
1248
|
"""
|
|
932
1249
|
Repair a STUCK replication rule.
|
|
933
1250
|
|
|
@@ -937,8 +1254,8 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
937
1254
|
"""
|
|
938
1255
|
|
|
939
1256
|
# Rule error cases:
|
|
940
|
-
# (A) A rule
|
|
941
|
-
# (B) A rule
|
|
1257
|
+
# (A) A rule gets an exception on rule-creation. This can only be the MissingSourceReplica exception.
|
|
1258
|
+
# (B) A rule gets an error when re-evaluated: InvalidRSEExpression, InvalidRuleWeight, InsufficientTargetRSEs, RSEWriteBlocked
|
|
942
1259
|
# InsufficientAccountLimit. The re-evaluation has to be done again and potential missing locks have to be
|
|
943
1260
|
# created.
|
|
944
1261
|
# (C) Transfers fail and mark locks (and the rule) as STUCK. All STUCK locks have to be repaired.
|
|
@@ -946,7 +1263,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
946
1263
|
|
|
947
1264
|
# start_time = time.time()
|
|
948
1265
|
try:
|
|
949
|
-
|
|
1266
|
+
stmt = select(
|
|
1267
|
+
models.ReplicationRule
|
|
1268
|
+
).where(
|
|
1269
|
+
models.ReplicationRule.id == rule_id
|
|
1270
|
+
).with_for_update(
|
|
1271
|
+
nowait=True
|
|
1272
|
+
)
|
|
1273
|
+
rule = session.execute(stmt).scalar_one()
|
|
950
1274
|
rule.updated_at = datetime.utcnow()
|
|
951
1275
|
|
|
952
1276
|
# Check if rule is longer than 2 weeks in STUCK
|
|
@@ -978,7 +1302,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
978
1302
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
979
1303
|
# Try to update the DatasetLocks
|
|
980
1304
|
if rule.grouping != RuleGrouping.NONE:
|
|
981
|
-
|
|
1305
|
+
stmt = update(
|
|
1306
|
+
models.DatasetLock
|
|
1307
|
+
).where(
|
|
1308
|
+
models.DatasetLock.rule_id == rule.id
|
|
1309
|
+
).values({
|
|
1310
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1311
|
+
})
|
|
1312
|
+
session.execute(stmt)
|
|
982
1313
|
logger(logging.DEBUG, '%s while repairing rule %s', str(error), rule_id)
|
|
983
1314
|
return
|
|
984
1315
|
|
|
@@ -998,7 +1329,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
998
1329
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
999
1330
|
# Try to update the DatasetLocks
|
|
1000
1331
|
if rule.grouping != RuleGrouping.NONE:
|
|
1001
|
-
|
|
1332
|
+
stmt = update(
|
|
1333
|
+
models.DatasetLock
|
|
1334
|
+
).where(
|
|
1335
|
+
models.DatasetLock.rule_id == rule.id
|
|
1336
|
+
).values({
|
|
1337
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1338
|
+
})
|
|
1339
|
+
session.execute(stmt)
|
|
1002
1340
|
logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
|
|
1003
1341
|
return
|
|
1004
1342
|
|
|
@@ -1007,19 +1345,32 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1007
1345
|
rule.locks_ok_cnt = 0
|
|
1008
1346
|
rule.locks_replicating_cnt = 0
|
|
1009
1347
|
rule.locks_stuck_cnt = 0
|
|
1010
|
-
|
|
1348
|
+
stmt = select(
|
|
1349
|
+
models.ReplicaLock.state,
|
|
1350
|
+
func.count(models.ReplicaLock.state).label('state_counter')
|
|
1351
|
+
).where(
|
|
1352
|
+
models.ReplicaLock.rule_id == rule.id
|
|
1353
|
+
).group_by(
|
|
1354
|
+
models.ReplicaLock.state
|
|
1355
|
+
)
|
|
1356
|
+
rule_counts = session.execute(stmt).all()
|
|
1011
1357
|
for count in rule_counts:
|
|
1012
|
-
if count
|
|
1013
|
-
rule.locks_ok_cnt = count
|
|
1014
|
-
elif count
|
|
1015
|
-
rule.locks_replicating_cnt = count
|
|
1016
|
-
elif count
|
|
1017
|
-
rule.locks_stuck_cnt = count
|
|
1358
|
+
if count.state == LockState.OK:
|
|
1359
|
+
rule.locks_ok_cnt = count.state_counter
|
|
1360
|
+
elif count.state == LockState.REPLICATING:
|
|
1361
|
+
rule.locks_replicating_cnt = count.state_counter
|
|
1362
|
+
elif count.state == LockState.STUCK:
|
|
1363
|
+
rule.locks_stuck_cnt = count.state_counter
|
|
1018
1364
|
logger(logging.DEBUG, "Finished resetting counters for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
1019
1365
|
|
|
1020
1366
|
# Get the did
|
|
1021
|
-
|
|
1022
|
-
|
|
1367
|
+
stmt = select(
|
|
1368
|
+
models.DataIdentifier
|
|
1369
|
+
).where(
|
|
1370
|
+
and_(models.DataIdentifier.scope == rule.scope,
|
|
1371
|
+
models.DataIdentifier.name == rule.name)
|
|
1372
|
+
)
|
|
1373
|
+
did = session.execute(stmt).scalar_one()
|
|
1023
1374
|
|
|
1024
1375
|
# Detect if there is something wrong with the dataset and
|
|
1025
1376
|
# make the decisison on soft or hard repair.
|
|
@@ -1064,7 +1415,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1064
1415
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
1065
1416
|
# Try to update the DatasetLocks
|
|
1066
1417
|
if rule.grouping != RuleGrouping.NONE:
|
|
1067
|
-
|
|
1418
|
+
stmt = update(
|
|
1419
|
+
models.DatasetLock
|
|
1420
|
+
).where(
|
|
1421
|
+
models.DatasetLock.rule_id == rule.id
|
|
1422
|
+
).values({
|
|
1423
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1424
|
+
})
|
|
1425
|
+
session.execute(stmt)
|
|
1068
1426
|
logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
|
|
1069
1427
|
return
|
|
1070
1428
|
|
|
@@ -1074,11 +1432,7 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1074
1432
|
if hard_repair:
|
|
1075
1433
|
__find_surplus_locks_and_remove_them(datasetfiles=datasetfiles,
|
|
1076
1434
|
locks=locks,
|
|
1077
|
-
replicas=replicas,
|
|
1078
|
-
source_replicas=source_replicas,
|
|
1079
|
-
rseselector=rseselector,
|
|
1080
1435
|
rule=rule,
|
|
1081
|
-
source_rses=[rse['id'] for rse in source_rses],
|
|
1082
1436
|
session=session)
|
|
1083
1437
|
|
|
1084
1438
|
session.flush()
|
|
@@ -1101,13 +1455,32 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1101
1455
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
1102
1456
|
# Try to update the DatasetLocks
|
|
1103
1457
|
if rule.grouping != RuleGrouping.NONE:
|
|
1104
|
-
|
|
1458
|
+
stmt = update(
|
|
1459
|
+
models.DatasetLock
|
|
1460
|
+
).where(
|
|
1461
|
+
models.DatasetLock.rule_id == rule.id
|
|
1462
|
+
).values({
|
|
1463
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1464
|
+
})
|
|
1465
|
+
session.execute(stmt)
|
|
1105
1466
|
logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
|
|
1106
1467
|
return
|
|
1107
1468
|
|
|
1108
1469
|
# Delete Datasetlocks which are not relevant anymore
|
|
1109
|
-
|
|
1110
|
-
|
|
1470
|
+
stmt = select(
|
|
1471
|
+
models.ReplicaLock.rse_id
|
|
1472
|
+
).distinct(
|
|
1473
|
+
).where(
|
|
1474
|
+
models.ReplicaLock.rule_id == rule.id
|
|
1475
|
+
)
|
|
1476
|
+
validated_datasetlock_rse_ids = session.execute(stmt).scalars().all()
|
|
1477
|
+
|
|
1478
|
+
stmt = select(
|
|
1479
|
+
models.DatasetLock
|
|
1480
|
+
).where(
|
|
1481
|
+
models.DatasetLock.rule_id == rule.id
|
|
1482
|
+
)
|
|
1483
|
+
dataset_locks = session.execute(stmt).scalars().all()
|
|
1111
1484
|
for dataset_lock in dataset_locks:
|
|
1112
1485
|
if dataset_lock.rse_id not in validated_datasetlock_rse_ids:
|
|
1113
1486
|
dataset_lock.delete(session=session)
|
|
@@ -1119,7 +1492,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1119
1492
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
1120
1493
|
# Try to update the DatasetLocks
|
|
1121
1494
|
if rule.grouping != RuleGrouping.NONE:
|
|
1122
|
-
|
|
1495
|
+
stmt = update(
|
|
1496
|
+
models.DatasetLock
|
|
1497
|
+
).where(
|
|
1498
|
+
models.DatasetLock.rule_id == rule.id
|
|
1499
|
+
).values({
|
|
1500
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1501
|
+
})
|
|
1502
|
+
session.execute(stmt)
|
|
1123
1503
|
# TODO: Increase some kind of Stuck Counter here, The rule should at some point be SUSPENDED
|
|
1124
1504
|
return
|
|
1125
1505
|
|
|
@@ -1133,7 +1513,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1133
1513
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
1134
1514
|
# Try to update the DatasetLocks
|
|
1135
1515
|
if rule.grouping != RuleGrouping.NONE:
|
|
1136
|
-
|
|
1516
|
+
stmt = update(
|
|
1517
|
+
models.DatasetLock
|
|
1518
|
+
).where(
|
|
1519
|
+
models.DatasetLock.rule_id == rule.id
|
|
1520
|
+
).values({
|
|
1521
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
1522
|
+
})
|
|
1523
|
+
session.execute(stmt)
|
|
1137
1524
|
return
|
|
1138
1525
|
|
|
1139
1526
|
rule.state = RuleState.OK
|
|
@@ -1143,7 +1530,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1143
1530
|
logger(logging.INFO, 'Rule %s [%d/%d/%d] state=OK', str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
1144
1531
|
|
|
1145
1532
|
if rule.grouping != RuleGrouping.NONE:
|
|
1146
|
-
|
|
1533
|
+
stmt = update(
|
|
1534
|
+
models.DatasetLock
|
|
1535
|
+
).where(
|
|
1536
|
+
models.DatasetLock.rule_id == rule.id
|
|
1537
|
+
).values({
|
|
1538
|
+
models.DatasetLock.state: LockState.OK
|
|
1539
|
+
})
|
|
1540
|
+
session.execute(stmt)
|
|
1147
1541
|
session.flush()
|
|
1148
1542
|
if rule.notification == RuleNotification.YES:
|
|
1149
1543
|
generate_email_for_rule_ok_notification(rule=rule, session=session)
|
|
@@ -1159,7 +1553,11 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
|
|
|
1159
1553
|
|
|
1160
1554
|
|
|
1161
1555
|
@read_session
|
|
1162
|
-
def get_rule(
|
|
1556
|
+
def get_rule(
|
|
1557
|
+
rule_id: str,
|
|
1558
|
+
*,
|
|
1559
|
+
session: "Session"
|
|
1560
|
+
) -> dict[str, Any]:
|
|
1163
1561
|
"""
|
|
1164
1562
|
Get a specific replication rule.
|
|
1165
1563
|
|
|
@@ -1169,16 +1567,26 @@ def get_rule(rule_id, *, session: "Session"):
|
|
|
1169
1567
|
"""
|
|
1170
1568
|
|
|
1171
1569
|
try:
|
|
1172
|
-
|
|
1570
|
+
stmt = select(
|
|
1571
|
+
models.ReplicationRule
|
|
1572
|
+
).where(
|
|
1573
|
+
models.ReplicationRule.id == rule_id
|
|
1574
|
+
)
|
|
1575
|
+
rule = session.execute(stmt).scalar_one()
|
|
1173
1576
|
return rule.to_dict()
|
|
1174
|
-
except NoResultFound:
|
|
1175
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
1176
|
-
except StatementError:
|
|
1177
|
-
raise RucioException('Badly formatted rule id (%s)' % (rule_id))
|
|
1577
|
+
except NoResultFound as exc:
|
|
1578
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
1579
|
+
except StatementError as exc:
|
|
1580
|
+
raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
|
|
1178
1581
|
|
|
1179
1582
|
|
|
1180
1583
|
@transactional_session
|
|
1181
|
-
def update_rule(
|
|
1584
|
+
def update_rule(
|
|
1585
|
+
rule_id: str,
|
|
1586
|
+
options: dict[str, Any],
|
|
1587
|
+
*,
|
|
1588
|
+
session: "Session"
|
|
1589
|
+
) -> None:
|
|
1182
1590
|
"""
|
|
1183
1591
|
Update a rules options.
|
|
1184
1592
|
|
|
@@ -1291,9 +1699,9 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
|
|
|
1291
1699
|
locktype
|
|
1292
1700
|
).where(
|
|
1293
1701
|
locktype.rule_id == rule.id
|
|
1294
|
-
).values(
|
|
1295
|
-
account
|
|
1296
|
-
)
|
|
1702
|
+
).values({
|
|
1703
|
+
locktype.account: options['account']
|
|
1704
|
+
})
|
|
1297
1705
|
session.execute(query)
|
|
1298
1706
|
|
|
1299
1707
|
# Update counters
|
|
@@ -1361,18 +1769,18 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
|
|
|
1361
1769
|
).where(
|
|
1362
1770
|
models.ReplicationRule.id == rid,
|
|
1363
1771
|
models.ReplicationRule.state != RuleState.SUSPENDED
|
|
1364
|
-
).values(
|
|
1365
|
-
state
|
|
1366
|
-
)
|
|
1772
|
+
).values({
|
|
1773
|
+
models.ReplicationRule.state: RuleState.STUCK
|
|
1774
|
+
})
|
|
1367
1775
|
session.execute(query)
|
|
1368
1776
|
|
|
1369
1777
|
query = update(
|
|
1370
1778
|
models.DatasetLock
|
|
1371
1779
|
).where(
|
|
1372
1780
|
models.DatasetLock.rule_id == rid
|
|
1373
|
-
).values(
|
|
1374
|
-
state
|
|
1375
|
-
)
|
|
1781
|
+
).values({
|
|
1782
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1783
|
+
})
|
|
1376
1784
|
session.execute(query)
|
|
1377
1785
|
|
|
1378
1786
|
if options['state'].lower() == 'suspended':
|
|
@@ -1387,18 +1795,18 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
|
|
|
1387
1795
|
).where(
|
|
1388
1796
|
models.ReplicaLock.rule_id == rule.id,
|
|
1389
1797
|
models.ReplicaLock.state == LockState.REPLICATING
|
|
1390
|
-
).values(
|
|
1391
|
-
state
|
|
1392
|
-
)
|
|
1798
|
+
).values({
|
|
1799
|
+
models.ReplicaLock.state: LockState.STUCK
|
|
1800
|
+
})
|
|
1393
1801
|
session.execute(query)
|
|
1394
1802
|
|
|
1395
1803
|
query = update(
|
|
1396
1804
|
models.DatasetLock
|
|
1397
1805
|
).where(
|
|
1398
1806
|
models.DatasetLock.rule_id == rule_id
|
|
1399
|
-
).values(
|
|
1400
|
-
state
|
|
1401
|
-
)
|
|
1807
|
+
).values({
|
|
1808
|
+
models.DatasetLock.state: LockState.STUCK
|
|
1809
|
+
})
|
|
1402
1810
|
session.execute(query)
|
|
1403
1811
|
|
|
1404
1812
|
elif key == 'cancel_requests':
|
|
@@ -1409,8 +1817,8 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
|
|
|
1409
1817
|
rule.priority = options[key]
|
|
1410
1818
|
transfers_to_update = request_core.update_requests_priority(priority=options[key], filter_={'rule_id': rule_id}, session=session)
|
|
1411
1819
|
transfer_core.update_transfer_priority(transfers_to_update)
|
|
1412
|
-
except Exception:
|
|
1413
|
-
raise UnsupportedOperation('The FTS Requests are already in a final state.')
|
|
1820
|
+
except Exception as exc:
|
|
1821
|
+
raise UnsupportedOperation('The FTS Requests are already in a final state.') from exc
|
|
1414
1822
|
|
|
1415
1823
|
elif key == 'child_rule_id':
|
|
1416
1824
|
# Check if the child rule has the same scope/name as the parent rule
|
|
@@ -1470,17 +1878,23 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
|
|
|
1470
1878
|
or match('.*IntegrityError.*UNIQUE constraint failed.*', str(error.args[0])) \
|
|
1471
1879
|
or match('.*1062.*Duplicate entry.*for key.*', str(error.args[0])) \
|
|
1472
1880
|
or match('.*IntegrityError.*columns? .*not unique.*', str(error.args[0])):
|
|
1473
|
-
raise DuplicateRule(error.args[0])
|
|
1881
|
+
raise DuplicateRule(error.args[0]) from error
|
|
1474
1882
|
else:
|
|
1475
1883
|
raise error
|
|
1476
|
-
except NoResultFound:
|
|
1477
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
1478
|
-
except StatementError as
|
|
1479
|
-
raise RucioException(f"A StatementError occurred while processing rule {rule_id}") from
|
|
1884
|
+
except NoResultFound as exc:
|
|
1885
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
1886
|
+
except StatementError as exc:
|
|
1887
|
+
raise RucioException(f"A StatementError occurred while processing rule {rule_id}") from exc
|
|
1480
1888
|
|
|
1481
1889
|
|
|
1482
1890
|
@transactional_session
|
|
1483
|
-
def reduce_rule(
|
|
1891
|
+
def reduce_rule(
|
|
1892
|
+
rule_id: str,
|
|
1893
|
+
copies: int,
|
|
1894
|
+
exclude_expression: Optional[str] = None,
|
|
1895
|
+
*,
|
|
1896
|
+
session: "Session"
|
|
1897
|
+
) -> str:
|
|
1484
1898
|
"""
|
|
1485
1899
|
Reduce the number of copies for a rule by atomically replacing the rule.
|
|
1486
1900
|
|
|
@@ -1491,7 +1905,12 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
|
|
|
1491
1905
|
:raises: RuleReplaceFailed, RuleNotFound
|
|
1492
1906
|
"""
|
|
1493
1907
|
try:
|
|
1494
|
-
|
|
1908
|
+
stmt = select(
|
|
1909
|
+
models.ReplicationRule
|
|
1910
|
+
).where(
|
|
1911
|
+
models.ReplicationRule.id == rule_id
|
|
1912
|
+
)
|
|
1913
|
+
rule = session.execute(stmt).scalar_one()
|
|
1495
1914
|
|
|
1496
1915
|
if copies >= rule.copies:
|
|
1497
1916
|
raise RuleReplaceFailed('Copies of the new rule must be smaller than the old rule.')
|
|
@@ -1531,7 +1950,12 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
|
|
|
1531
1950
|
|
|
1532
1951
|
session.flush()
|
|
1533
1952
|
|
|
1534
|
-
|
|
1953
|
+
stmt = select(
|
|
1954
|
+
models.ReplicationRule
|
|
1955
|
+
).where(
|
|
1956
|
+
models.ReplicationRule.id == new_rule_id[0]
|
|
1957
|
+
)
|
|
1958
|
+
new_rule = session.execute(stmt).scalar_one()
|
|
1535
1959
|
|
|
1536
1960
|
if new_rule.state != RuleState.OK:
|
|
1537
1961
|
raise RuleReplaceFailed('The replacement of the rule failed.')
|
|
@@ -1541,12 +1965,18 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
|
|
|
1541
1965
|
|
|
1542
1966
|
return new_rule_id[0]
|
|
1543
1967
|
|
|
1544
|
-
except NoResultFound:
|
|
1545
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
1968
|
+
except NoResultFound as exc:
|
|
1969
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
1546
1970
|
|
|
1547
1971
|
|
|
1548
1972
|
@transactional_session
|
|
1549
|
-
def move_rule(
|
|
1973
|
+
def move_rule(
|
|
1974
|
+
rule_id: str,
|
|
1975
|
+
rse_expression: str,
|
|
1976
|
+
override: Optional[dict[str, Any]] = None,
|
|
1977
|
+
*,
|
|
1978
|
+
session: "Session"
|
|
1979
|
+
) -> str:
|
|
1550
1980
|
"""
|
|
1551
1981
|
Move a replication rule to another RSE and, once done, delete the original one.
|
|
1552
1982
|
|
|
@@ -1559,7 +1989,12 @@ def move_rule(rule_id: str, rse_expression: str, override: Optional[dict[str, An
|
|
|
1559
1989
|
override = override or {}
|
|
1560
1990
|
|
|
1561
1991
|
try:
|
|
1562
|
-
|
|
1992
|
+
stmt = select(
|
|
1993
|
+
models.ReplicationRule
|
|
1994
|
+
).where(
|
|
1995
|
+
models.ReplicationRule.id == rule_id
|
|
1996
|
+
)
|
|
1997
|
+
rule = session.execute(stmt).scalar_one()
|
|
1563
1998
|
|
|
1564
1999
|
if rule.child_rule_id:
|
|
1565
2000
|
raise RuleReplaceFailed('The rule must not have a child rule.')
|
|
@@ -1608,14 +2043,20 @@ def move_rule(rule_id: str, rse_expression: str, override: Optional[dict[str, An
|
|
|
1608
2043
|
|
|
1609
2044
|
return new_rule_id[0]
|
|
1610
2045
|
|
|
1611
|
-
except StatementError:
|
|
1612
|
-
raise RucioException('Badly formatted rule id (%s)' % (rule_id))
|
|
1613
|
-
except NoResultFound:
|
|
1614
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
2046
|
+
except StatementError as exc:
|
|
2047
|
+
raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
|
|
2048
|
+
except NoResultFound as exc:
|
|
2049
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
1615
2050
|
|
|
1616
2051
|
|
|
1617
2052
|
@transactional_session
|
|
1618
|
-
def re_evaluate_did(
|
|
2053
|
+
def re_evaluate_did(
|
|
2054
|
+
scope: InternalScope,
|
|
2055
|
+
name: str,
|
|
2056
|
+
rule_evaluation_action: DIDReEvaluation,
|
|
2057
|
+
*,
|
|
2058
|
+
session: "Session"
|
|
2059
|
+
) -> None:
|
|
1619
2060
|
"""
|
|
1620
2061
|
Re-Evaluates a did.
|
|
1621
2062
|
|
|
@@ -1627,10 +2068,15 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
|
|
|
1627
2068
|
"""
|
|
1628
2069
|
|
|
1629
2070
|
try:
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
2071
|
+
stmt = select(
|
|
2072
|
+
models.DataIdentifier
|
|
2073
|
+
).where(
|
|
2074
|
+
and_(models.DataIdentifier.scope == scope,
|
|
2075
|
+
models.DataIdentifier.name == name)
|
|
2076
|
+
)
|
|
2077
|
+
did = session.execute(stmt).scalar_one()
|
|
2078
|
+
except NoResultFound as exc:
|
|
2079
|
+
raise DataIdentifierNotFound() from exc
|
|
1634
2080
|
|
|
1635
2081
|
if rule_evaluation_action == DIDReEvaluation.ATTACH:
|
|
1636
2082
|
__evaluate_did_attach(did, session=session)
|
|
@@ -1639,13 +2085,16 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
|
|
|
1639
2085
|
|
|
1640
2086
|
# Update size and length of did
|
|
1641
2087
|
if session.bind.dialect.name == 'oracle':
|
|
1642
|
-
stmt =
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
2088
|
+
stmt = select(
|
|
2089
|
+
func.sum(models.DataIdentifierAssociation.bytes),
|
|
2090
|
+
func.count(1)
|
|
2091
|
+
).with_hint(
|
|
2092
|
+
models.DataIdentifierAssociation, 'INDEX(CONTENTS CONTENTS_PK)', 'oracle'
|
|
2093
|
+
).where(
|
|
2094
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
2095
|
+
models.DataIdentifierAssociation.name == name)
|
|
2096
|
+
)
|
|
2097
|
+
for bytes_, length in session.execute(stmt):
|
|
1649
2098
|
did.bytes = bytes_
|
|
1650
2099
|
did.length = length
|
|
1651
2100
|
|
|
@@ -1657,7 +2106,14 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
|
|
|
1657
2106
|
|
|
1658
2107
|
|
|
1659
2108
|
@read_session
|
|
1660
|
-
def get_updated_dids(
|
|
2109
|
+
def get_updated_dids(
|
|
2110
|
+
total_workers: int,
|
|
2111
|
+
worker_number: int,
|
|
2112
|
+
limit: int = 100,
|
|
2113
|
+
blocked_dids: Optional[Sequence[tuple[str, str]]] = None,
|
|
2114
|
+
*,
|
|
2115
|
+
session: "Session"
|
|
2116
|
+
) -> list[tuple[str, InternalScope, str, DIDReEvaluation]]:
|
|
1661
2117
|
"""
|
|
1662
2118
|
Get updated dids.
|
|
1663
2119
|
|
|
@@ -1667,21 +2123,23 @@ def get_updated_dids(total_workers, worker_number, limit=100, blocked_dids=[], *
|
|
|
1667
2123
|
:param blocked_dids: Blocked dids to filter.
|
|
1668
2124
|
:param session: Database session in use.
|
|
1669
2125
|
"""
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
2126
|
+
blocked_dids = blocked_dids or []
|
|
2127
|
+
stmt = select(
|
|
2128
|
+
models.UpdatedDID.id,
|
|
2129
|
+
models.UpdatedDID.scope,
|
|
2130
|
+
models.UpdatedDID.name,
|
|
2131
|
+
models.UpdatedDID.rule_evaluation_action
|
|
2132
|
+
)
|
|
2133
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
1676
2134
|
|
|
1677
2135
|
# Remove blocked dids from query, but only do the first 30 ones, not to overload the query
|
|
1678
2136
|
if blocked_dids:
|
|
1679
2137
|
chunk = list(chunks(blocked_dids, 30))[0]
|
|
1680
|
-
|
|
2138
|
+
stmt = stmt.where(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
|
|
1681
2139
|
|
|
1682
2140
|
if limit:
|
|
1683
|
-
fetched_dids =
|
|
1684
|
-
filtered_dids = [did for did in fetched_dids if (did.scope, did.name) not in blocked_dids]
|
|
2141
|
+
fetched_dids = session.execute(stmt.order_by(models.UpdatedDID.created_at).limit(limit)).all()
|
|
2142
|
+
filtered_dids = [did._tuple() for did in fetched_dids if (did.scope, did.name) not in blocked_dids]
|
|
1685
2143
|
if len(fetched_dids) == limit and not filtered_dids:
|
|
1686
2144
|
return get_updated_dids(total_workers=total_workers,
|
|
1687
2145
|
worker_number=worker_number,
|
|
@@ -1691,11 +2149,23 @@ def get_updated_dids(total_workers, worker_number, limit=100, blocked_dids=[], *
|
|
|
1691
2149
|
else:
|
|
1692
2150
|
return filtered_dids
|
|
1693
2151
|
else:
|
|
1694
|
-
return [did for did in
|
|
2152
|
+
return [did._tuple() for did in session.execute(stmt.order_by(models.UpdatedDID.created_at)).all() if (did.scope, did.name) not in blocked_dids]
|
|
1695
2153
|
|
|
1696
2154
|
|
|
1697
2155
|
@read_session
|
|
1698
|
-
def get_rules_beyond_eol(
|
|
2156
|
+
def get_rules_beyond_eol(
|
|
2157
|
+
date_check: datetime,
|
|
2158
|
+
worker_number: int,
|
|
2159
|
+
total_workers: int, *,
|
|
2160
|
+
session: "Session"
|
|
2161
|
+
) -> list[tuple[InternalScope,
|
|
2162
|
+
str,
|
|
2163
|
+
str,
|
|
2164
|
+
bool,
|
|
2165
|
+
str,
|
|
2166
|
+
Optional[datetime],
|
|
2167
|
+
Optional[datetime],
|
|
2168
|
+
InternalAccount]]:
|
|
1699
2169
|
"""
|
|
1700
2170
|
Get rules which have eol_at before a certain date.
|
|
1701
2171
|
|
|
@@ -1704,22 +2174,32 @@ def get_rules_beyond_eol(date_check, worker_number, total_workers, *, session: "
|
|
|
1704
2174
|
:param total_workers: Number of total workers.
|
|
1705
2175
|
:param session: Database session in use.
|
|
1706
2176
|
"""
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
2177
|
+
stmt = select(
|
|
2178
|
+
models.ReplicationRule.scope,
|
|
2179
|
+
models.ReplicationRule.name,
|
|
2180
|
+
models.ReplicationRule.rse_expression,
|
|
2181
|
+
models.ReplicationRule.locked,
|
|
2182
|
+
models.ReplicationRule.id,
|
|
2183
|
+
models.ReplicationRule.eol_at,
|
|
2184
|
+
models.ReplicationRule.expires_at,
|
|
2185
|
+
models.ReplicationRule.account
|
|
2186
|
+
).where(
|
|
2187
|
+
models.ReplicationRule.eol_at < date_check
|
|
2188
|
+
)
|
|
1716
2189
|
|
|
1717
|
-
|
|
1718
|
-
return [
|
|
2190
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
2191
|
+
return [row._tuple() for row in session.execute(stmt).all()]
|
|
1719
2192
|
|
|
1720
2193
|
|
|
1721
2194
|
@read_session
|
|
1722
|
-
def get_expired_rules(
|
|
2195
|
+
def get_expired_rules(
|
|
2196
|
+
total_workers: int,
|
|
2197
|
+
worker_number: int,
|
|
2198
|
+
limit: int = 100,
|
|
2199
|
+
blocked_rules: Optional[Sequence[str]] = None,
|
|
2200
|
+
*,
|
|
2201
|
+
session: "Session"
|
|
2202
|
+
) -> list[tuple[str, str]]:
|
|
1723
2203
|
"""
|
|
1724
2204
|
Get expired rules.
|
|
1725
2205
|
|
|
@@ -1730,18 +2210,26 @@ def get_expired_rules(total_workers, worker_number, limit=100, blocked_rules=[],
|
|
|
1730
2210
|
:param session: Database session in use.
|
|
1731
2211
|
"""
|
|
1732
2212
|
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
2213
|
+
blocked_rules = blocked_rules or []
|
|
2214
|
+
stmt = select(
|
|
2215
|
+
models.ReplicationRule.id,
|
|
2216
|
+
models.ReplicationRule.rse_expression
|
|
2217
|
+
).with_hint(
|
|
2218
|
+
models.ReplicationRule, 'INDEX(RULES RULES_EXPIRES_AT_IDX)', 'oracle'
|
|
2219
|
+
).where(
|
|
2220
|
+
and_(models.ReplicationRule.expires_at < datetime.utcnow(),
|
|
2221
|
+
models.ReplicationRule.locked == false(),
|
|
2222
|
+
models.ReplicationRule.child_rule_id == null())
|
|
2223
|
+
).order_by(
|
|
2224
|
+
models.ReplicationRule.expires_at
|
|
2225
|
+
)
|
|
2226
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
1740
2227
|
|
|
1741
2228
|
if limit:
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
2229
|
+
stmt = stmt.limit(limit)
|
|
2230
|
+
result = session.execute(stmt).all()
|
|
2231
|
+
filtered_rules = [rule._tuple() for rule in result if rule.id not in blocked_rules]
|
|
2232
|
+
if len(result) == limit and not filtered_rules:
|
|
1745
2233
|
return get_expired_rules(total_workers=total_workers,
|
|
1746
2234
|
worker_number=worker_number,
|
|
1747
2235
|
limit=None,
|
|
@@ -1750,11 +2238,18 @@ def get_expired_rules(total_workers, worker_number, limit=100, blocked_rules=[],
|
|
|
1750
2238
|
else:
|
|
1751
2239
|
return filtered_rules
|
|
1752
2240
|
else:
|
|
1753
|
-
return [rule for rule in
|
|
2241
|
+
return [rule._tuple() for rule in session.execute(stmt).all() if rule.id not in blocked_rules]
|
|
1754
2242
|
|
|
1755
2243
|
|
|
1756
2244
|
@read_session
|
|
1757
|
-
def get_injected_rules(
|
|
2245
|
+
def get_injected_rules(
|
|
2246
|
+
total_workers: int,
|
|
2247
|
+
worker_number: int,
|
|
2248
|
+
limit: int = 100,
|
|
2249
|
+
blocked_rules: Optional[Sequence[str]] = None,
|
|
2250
|
+
*,
|
|
2251
|
+
session: "Session"
|
|
2252
|
+
) -> list[str]:
|
|
1758
2253
|
"""
|
|
1759
2254
|
Get rules to be injected.
|
|
1760
2255
|
|
|
@@ -1765,18 +2260,24 @@ def get_injected_rules(total_workers, worker_number, limit=100, blocked_rules=[]
|
|
|
1765
2260
|
:param session: Database session in use.
|
|
1766
2261
|
"""
|
|
1767
2262
|
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
2263
|
+
blocked_rules = blocked_rules or []
|
|
2264
|
+
stmt = select(
|
|
2265
|
+
models.ReplicationRule.id
|
|
2266
|
+
).with_hint(
|
|
2267
|
+
models.ReplicationRule, 'INDEX(RULES RULES_STATE_IDX)', 'oracle'
|
|
2268
|
+
).where(
|
|
2269
|
+
and_(models.ReplicationRule.state == RuleState.INJECT,
|
|
2270
|
+
models.ReplicationRule.created_at <= datetime.utcnow())
|
|
2271
|
+
).order_by(
|
|
2272
|
+
models.ReplicationRule.created_at
|
|
2273
|
+
)
|
|
2274
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
1775
2275
|
|
|
1776
2276
|
if limit:
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
2277
|
+
stmt = stmt.limit(limit)
|
|
2278
|
+
result = session.execute(stmt).scalars().all()
|
|
2279
|
+
filtered_rules = [rule for rule in result if rule not in blocked_rules]
|
|
2280
|
+
if len(result) == limit and not filtered_rules:
|
|
1780
2281
|
return get_injected_rules(total_workers=total_workers,
|
|
1781
2282
|
worker_number=worker_number,
|
|
1782
2283
|
limit=None,
|
|
@@ -1785,11 +2286,19 @@ def get_injected_rules(total_workers, worker_number, limit=100, blocked_rules=[]
|
|
|
1785
2286
|
else:
|
|
1786
2287
|
return filtered_rules
|
|
1787
2288
|
else:
|
|
1788
|
-
return [rule for rule in
|
|
2289
|
+
return [rule for rule in session.execute(stmt).scalars().all() if rule not in blocked_rules]
|
|
1789
2290
|
|
|
1790
2291
|
|
|
1791
2292
|
@read_session
|
|
1792
|
-
def get_stuck_rules(
|
|
2293
|
+
def get_stuck_rules(
|
|
2294
|
+
total_workers: int,
|
|
2295
|
+
worker_number: int,
|
|
2296
|
+
delta: int = 600,
|
|
2297
|
+
limit: int = 10,
|
|
2298
|
+
blocked_rules: Optional[Sequence[str]] = None,
|
|
2299
|
+
*,
|
|
2300
|
+
session: "Session"
|
|
2301
|
+
) -> list[str]:
|
|
1793
2302
|
"""
|
|
1794
2303
|
Get stuck rules.
|
|
1795
2304
|
|
|
@@ -1800,21 +2309,27 @@ def get_stuck_rules(total_workers, worker_number, delta=600, limit=10, blocked_r
|
|
|
1800
2309
|
:param blocked_rules: Blocked rules to filter out.
|
|
1801
2310
|
:param session: Database session in use.
|
|
1802
2311
|
"""
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
2312
|
+
blocked_rules = blocked_rules or []
|
|
2313
|
+
stmt = select(
|
|
2314
|
+
models.ReplicationRule.id
|
|
2315
|
+
).with_hint(
|
|
2316
|
+
models.ReplicationRule, 'INDEX(RULES RULES_STATE_IDX)', 'oracle'
|
|
2317
|
+
).where(
|
|
2318
|
+
and_(models.ReplicationRule.state == RuleState.STUCK,
|
|
2319
|
+
models.ReplicationRule.updated_at < datetime.utcnow() - timedelta(seconds=delta),
|
|
2320
|
+
or_(models.ReplicationRule.expires_at == null(),
|
|
2321
|
+
models.ReplicationRule.expires_at > datetime.utcnow(),
|
|
2322
|
+
models.ReplicationRule.locked == true()))
|
|
2323
|
+
).order_by(
|
|
2324
|
+
models.ReplicationRule.updated_at
|
|
2325
|
+
)
|
|
2326
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
1813
2327
|
|
|
1814
2328
|
if limit:
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
2329
|
+
stmt = stmt.limit(limit)
|
|
2330
|
+
result = session.execute(stmt).scalars().all()
|
|
2331
|
+
filtered_rules = [rule for rule in result if rule not in blocked_rules]
|
|
2332
|
+
if len(result) == limit and not filtered_rules:
|
|
1818
2333
|
return get_stuck_rules(total_workers=total_workers,
|
|
1819
2334
|
worker_number=worker_number,
|
|
1820
2335
|
delta=delta,
|
|
@@ -1824,22 +2339,39 @@ def get_stuck_rules(total_workers, worker_number, delta=600, limit=10, blocked_r
|
|
|
1824
2339
|
else:
|
|
1825
2340
|
return filtered_rules
|
|
1826
2341
|
else:
|
|
1827
|
-
return [rule for rule in
|
|
2342
|
+
return [rule for rule in session.execute(stmt).scalars().all() if rule not in blocked_rules]
|
|
1828
2343
|
|
|
1829
2344
|
|
|
1830
2345
|
@transactional_session
|
|
1831
|
-
def delete_updated_did(
|
|
2346
|
+
def delete_updated_did(
|
|
2347
|
+
id_: str,
|
|
2348
|
+
*,
|
|
2349
|
+
session: "Session"
|
|
2350
|
+
) -> None:
|
|
1832
2351
|
"""
|
|
1833
2352
|
Delete an updated_did by id.
|
|
1834
2353
|
|
|
1835
2354
|
:param id_: Id of the row not to delete.
|
|
1836
2355
|
:param session: The database session in use.
|
|
1837
2356
|
"""
|
|
1838
|
-
|
|
2357
|
+
stmt = delete(
|
|
2358
|
+
models.UpdatedDID
|
|
2359
|
+
).where(
|
|
2360
|
+
models.UpdatedDID.id == id_
|
|
2361
|
+
)
|
|
2362
|
+
session.execute(stmt)
|
|
1839
2363
|
|
|
1840
2364
|
|
|
1841
2365
|
@transactional_session
|
|
1842
|
-
def update_rules_for_lost_replica(
|
|
2366
|
+
def update_rules_for_lost_replica(
|
|
2367
|
+
scope: InternalScope,
|
|
2368
|
+
name: str,
|
|
2369
|
+
rse_id: str,
|
|
2370
|
+
nowait: bool = False,
|
|
2371
|
+
*,
|
|
2372
|
+
session: "Session",
|
|
2373
|
+
logger: LoggerFunction = logging.log
|
|
2374
|
+
) -> None:
|
|
1843
2375
|
"""
|
|
1844
2376
|
Update rules if a file replica is lost.
|
|
1845
2377
|
|
|
@@ -1851,9 +2383,38 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1851
2383
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
1852
2384
|
"""
|
|
1853
2385
|
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
2386
|
+
stmt = select(
|
|
2387
|
+
models.ReplicaLock
|
|
2388
|
+
).where(
|
|
2389
|
+
and_(models.ReplicaLock.scope == scope,
|
|
2390
|
+
models.ReplicaLock.name == name,
|
|
2391
|
+
models.ReplicaLock.rse_id == rse_id)
|
|
2392
|
+
).with_for_update(
|
|
2393
|
+
nowait=nowait
|
|
2394
|
+
)
|
|
2395
|
+
locks = session.execute(stmt).scalars().all()
|
|
2396
|
+
|
|
2397
|
+
stmt = select(
|
|
2398
|
+
models.RSEFileAssociation
|
|
2399
|
+
).where(
|
|
2400
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2401
|
+
models.RSEFileAssociation.name == name,
|
|
2402
|
+
models.RSEFileAssociation.rse_id == rse_id)
|
|
2403
|
+
).with_for_update(
|
|
2404
|
+
nowait=nowait
|
|
2405
|
+
)
|
|
2406
|
+
replica = session.execute(stmt).scalar_one()
|
|
2407
|
+
|
|
2408
|
+
stmt = select(
|
|
2409
|
+
models.Request
|
|
2410
|
+
).where(
|
|
2411
|
+
and_(models.Request.scope == scope,
|
|
2412
|
+
models.Request.name == name,
|
|
2413
|
+
models.Request.dest_rse_id == rse_id)
|
|
2414
|
+
).with_for_update(
|
|
2415
|
+
nowait=nowait
|
|
2416
|
+
)
|
|
2417
|
+
requests = session.execute(stmt).scalars().all()
|
|
1857
2418
|
|
|
1858
2419
|
rse = get_rse_name(rse_id, session=session)
|
|
1859
2420
|
|
|
@@ -1867,7 +2428,14 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1867
2428
|
session.delete(request)
|
|
1868
2429
|
|
|
1869
2430
|
for lock in locks:
|
|
1870
|
-
|
|
2431
|
+
stmt = select(
|
|
2432
|
+
models.ReplicationRule
|
|
2433
|
+
).where(
|
|
2434
|
+
models.ReplicationRule.id == lock.rule_id
|
|
2435
|
+
).with_for_update(
|
|
2436
|
+
nowait=nowait
|
|
2437
|
+
)
|
|
2438
|
+
rule = session.execute(stmt).scalar_one()
|
|
1871
2439
|
rule_state_before = rule.state
|
|
1872
2440
|
replica.lock_cnt -= 1
|
|
1873
2441
|
if lock.state == LockState.OK:
|
|
@@ -1884,7 +2452,14 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1884
2452
|
elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
|
|
1885
2453
|
rule.state = RuleState.OK
|
|
1886
2454
|
if rule.grouping != RuleGrouping.NONE:
|
|
1887
|
-
|
|
2455
|
+
stmt = update(
|
|
2456
|
+
models.DatasetLock
|
|
2457
|
+
).where(
|
|
2458
|
+
models.DatasetLock.rule_id == rule.id
|
|
2459
|
+
).values({
|
|
2460
|
+
models.DatasetLock.state: LockState.OK
|
|
2461
|
+
})
|
|
2462
|
+
session.execute(stmt)
|
|
1888
2463
|
session.flush()
|
|
1889
2464
|
if rule_state_before != RuleState.OK:
|
|
1890
2465
|
generate_rule_notifications(rule=rule, session=session)
|
|
@@ -1902,8 +2477,28 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1902
2477
|
|
|
1903
2478
|
replica.tombstone = OBSOLETE
|
|
1904
2479
|
replica.state = ReplicaState.UNAVAILABLE
|
|
1905
|
-
|
|
1906
|
-
|
|
2480
|
+
stmt = update(
|
|
2481
|
+
models.DataIdentifier
|
|
2482
|
+
).where(
|
|
2483
|
+
and_(models.DataIdentifier.scope == scope,
|
|
2484
|
+
models.DataIdentifier.name == name)
|
|
2485
|
+
).values({
|
|
2486
|
+
models.DataIdentifier.availability: DIDAvailability.LOST
|
|
2487
|
+
})
|
|
2488
|
+
session.execute(stmt)
|
|
2489
|
+
|
|
2490
|
+
stmt = update(
|
|
2491
|
+
models.BadReplica
|
|
2492
|
+
).where(
|
|
2493
|
+
and_(models.BadReplica.scope == scope,
|
|
2494
|
+
models.BadReplica.name == name,
|
|
2495
|
+
models.BadReplica.rse_id == rse_id,
|
|
2496
|
+
models.BadReplica.state == BadFilesStatus.BAD)
|
|
2497
|
+
).values({
|
|
2498
|
+
models.BadReplica.state: BadFilesStatus.LOST,
|
|
2499
|
+
models.BadReplica.updated_at: datetime.utcnow()
|
|
2500
|
+
})
|
|
2501
|
+
session.execute(stmt)
|
|
1907
2502
|
for dts in datasets:
|
|
1908
2503
|
logger(logging.INFO, 'File %s:%s bad at site %s is completely lost from dataset %s:%s. Will be marked as LOST and detached', scope, name, rse, dts['scope'], dts['name'])
|
|
1909
2504
|
rucio.core.did.detach_dids(scope=dts['scope'], name=dts['name'], dids=[{'scope': scope, 'name': name}], session=session)
|
|
@@ -1919,7 +2514,15 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1919
2514
|
|
|
1920
2515
|
|
|
1921
2516
|
@transactional_session
|
|
1922
|
-
def update_rules_for_bad_replica(
|
|
2517
|
+
def update_rules_for_bad_replica(
|
|
2518
|
+
scope: InternalScope,
|
|
2519
|
+
name: str,
|
|
2520
|
+
rse_id: str,
|
|
2521
|
+
nowait: bool = False,
|
|
2522
|
+
*,
|
|
2523
|
+
session: "Session",
|
|
2524
|
+
logger: LoggerFunction = logging.log
|
|
2525
|
+
) -> None:
|
|
1923
2526
|
"""
|
|
1924
2527
|
Update rules if a file replica is bad and has to be recreated.
|
|
1925
2528
|
|
|
@@ -1930,15 +2533,40 @@ def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1930
2533
|
:param session: The database session in use.
|
|
1931
2534
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
1932
2535
|
"""
|
|
2536
|
+
stmt = select(
|
|
2537
|
+
models.ReplicaLock
|
|
2538
|
+
).where(
|
|
2539
|
+
and_(models.ReplicaLock.scope == scope,
|
|
2540
|
+
models.ReplicaLock.name == name,
|
|
2541
|
+
models.ReplicaLock.rse_id == rse_id)
|
|
2542
|
+
).with_for_update(
|
|
2543
|
+
nowait=nowait
|
|
2544
|
+
)
|
|
2545
|
+
locks = session.execute(stmt).scalars().all()
|
|
1933
2546
|
|
|
1934
|
-
|
|
1935
|
-
|
|
2547
|
+
stmt = select(
|
|
2548
|
+
models.RSEFileAssociation
|
|
2549
|
+
).where(
|
|
2550
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2551
|
+
models.RSEFileAssociation.name == name,
|
|
2552
|
+
models.RSEFileAssociation.rse_id == rse_id)
|
|
2553
|
+
).with_for_update(
|
|
2554
|
+
nowait=nowait
|
|
2555
|
+
)
|
|
2556
|
+
replica = session.execute(stmt).scalar_one()
|
|
1936
2557
|
|
|
1937
2558
|
nlock = 0
|
|
1938
2559
|
datasets = []
|
|
1939
2560
|
for lock in locks:
|
|
1940
2561
|
nlock += 1
|
|
1941
|
-
|
|
2562
|
+
stmt = select(
|
|
2563
|
+
models.ReplicationRule
|
|
2564
|
+
).where(
|
|
2565
|
+
models.ReplicationRule.id == lock.rule_id
|
|
2566
|
+
).with_for_update(
|
|
2567
|
+
nowait=nowait
|
|
2568
|
+
)
|
|
2569
|
+
rule = session.execute(stmt).scalar_one()
|
|
1942
2570
|
# If source replica expression exists, we remove it
|
|
1943
2571
|
if rule.source_replica_expression:
|
|
1944
2572
|
rule.source_replica_expression = None
|
|
@@ -1981,19 +2609,50 @@ def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, *, session:
|
|
|
1981
2609
|
else:
|
|
1982
2610
|
rule.state = RuleState.REPLICATING
|
|
1983
2611
|
if rule.grouping != RuleGrouping.NONE:
|
|
1984
|
-
|
|
2612
|
+
stmt = update(
|
|
2613
|
+
models.DatasetLock
|
|
2614
|
+
).where(
|
|
2615
|
+
models.DatasetLock.rule_id == rule.id
|
|
2616
|
+
).values({
|
|
2617
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
2618
|
+
})
|
|
2619
|
+
session.execute(stmt)
|
|
1985
2620
|
# Insert rule history
|
|
1986
2621
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
1987
2622
|
if nlock:
|
|
1988
|
-
|
|
2623
|
+
stmt = update(
|
|
2624
|
+
models.RSEFileAssociation
|
|
2625
|
+
).where(
|
|
2626
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2627
|
+
models.RSEFileAssociation.name == name,
|
|
2628
|
+
models.RSEFileAssociation.rse_id == rse_id)
|
|
2629
|
+
).values({
|
|
2630
|
+
models.RSEFileAssociation.state: ReplicaState.COPYING
|
|
2631
|
+
})
|
|
2632
|
+
session.execute(stmt)
|
|
1989
2633
|
else:
|
|
1990
2634
|
logger(logging.INFO, 'File %s:%s at site %s has no locks. Will be deleted now.', scope, name, get_rse_name(rse_id=rse_id, session=session))
|
|
1991
2635
|
tombstone = OBSOLETE
|
|
1992
|
-
|
|
2636
|
+
stmt = update(
|
|
2637
|
+
models.RSEFileAssociation
|
|
2638
|
+
).where(
|
|
2639
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2640
|
+
models.RSEFileAssociation.name == name,
|
|
2641
|
+
models.RSEFileAssociation.rse_id == rse_id)
|
|
2642
|
+
).values({
|
|
2643
|
+
models.RSEFileAssociation.state: ReplicaState.UNAVAILABLE,
|
|
2644
|
+
models.RSEFileAssociation.tombstone: tombstone
|
|
2645
|
+
})
|
|
2646
|
+
session.execute(stmt)
|
|
1993
2647
|
|
|
1994
2648
|
|
|
1995
2649
|
@transactional_session
|
|
1996
|
-
def generate_rule_notifications(
|
|
2650
|
+
def generate_rule_notifications(
|
|
2651
|
+
rule: models.ReplicationRule,
|
|
2652
|
+
replicating_locks_before: Optional[int] = None,
|
|
2653
|
+
*,
|
|
2654
|
+
session: "Session"
|
|
2655
|
+
) -> None:
|
|
1997
2656
|
"""
|
|
1998
2657
|
Generate (If necessary) a callback for a rule (DATASETLOCK_OK, RULE_OK, DATASETLOCK_PROGRESS)
|
|
1999
2658
|
|
|
@@ -2047,7 +2706,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
|
|
|
2047
2706
|
if rule.grouping != RuleGrouping.NONE:
|
|
2048
2707
|
# Only send DATASETLOCK_OK callbacks for ALL/DATASET grouped rules
|
|
2049
2708
|
if rule.notification == RuleNotification.YES:
|
|
2050
|
-
|
|
2709
|
+
stmt = select(
|
|
2710
|
+
models.DatasetLock
|
|
2711
|
+
).where(
|
|
2712
|
+
models.DatasetLock.rule_id == rule.id
|
|
2713
|
+
)
|
|
2714
|
+
dataset_locks = session.execute(stmt).scalars().all()
|
|
2051
2715
|
for dataset_lock in dataset_locks:
|
|
2052
2716
|
payload = {'scope': dataset_lock.scope.external,
|
|
2053
2717
|
'name': dataset_lock.name,
|
|
@@ -2060,7 +2724,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
|
|
|
2060
2724
|
add_message(event_type='DATASETLOCK_OK', payload=payload, session=session)
|
|
2061
2725
|
|
|
2062
2726
|
elif rule.notification == RuleNotification.CLOSE:
|
|
2063
|
-
|
|
2727
|
+
stmt = select(
|
|
2728
|
+
models.DatasetLock
|
|
2729
|
+
).where(
|
|
2730
|
+
models.DatasetLock.rule_id == rule.id
|
|
2731
|
+
)
|
|
2732
|
+
dataset_locks = session.execute(stmt).scalars().all()
|
|
2064
2733
|
for dataset_lock in dataset_locks:
|
|
2065
2734
|
try:
|
|
2066
2735
|
did = rucio.core.did.get_did(scope=dataset_lock.scope, name=dataset_lock.name, session=session)
|
|
@@ -2082,7 +2751,7 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
|
|
|
2082
2751
|
pass
|
|
2083
2752
|
|
|
2084
2753
|
elif rule.state == RuleState.REPLICATING and rule.notification == RuleNotification.PROGRESS and replicating_locks_before:
|
|
2085
|
-
# For RuleNotification PROGRESS rules, also
|
|
2754
|
+
# For RuleNotification PROGRESS rules, also notify when REPLICATING thresholds are passed
|
|
2086
2755
|
if __progress_class(replicating_locks_before, total_locks) != __progress_class(rule.locks_replicating_cnt, total_locks):
|
|
2087
2756
|
try:
|
|
2088
2757
|
did = rucio.core.did.get_did(scope=rule.scope, name=rule.name, session=session)
|
|
@@ -2101,7 +2770,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
|
|
|
2101
2770
|
|
|
2102
2771
|
|
|
2103
2772
|
@transactional_session
|
|
2104
|
-
def generate_email_for_rule_ok_notification(
|
|
2773
|
+
def generate_email_for_rule_ok_notification(
|
|
2774
|
+
rule: models.ReplicationRule,
|
|
2775
|
+
*,
|
|
2776
|
+
session: "Session",
|
|
2777
|
+
logger: LoggerFunction = logging.log
|
|
2778
|
+
) -> None:
|
|
2105
2779
|
"""
|
|
2106
2780
|
Generate (If necessary) an eMail for a rule with notification mode Y.
|
|
2107
2781
|
|
|
@@ -2122,7 +2796,7 @@ def generate_email_for_rule_ok_notification(rule, *, session: "Session", logger=
|
|
|
2122
2796
|
try:
|
|
2123
2797
|
with open(template_path, 'r') as templatefile:
|
|
2124
2798
|
template = Template(templatefile.read())
|
|
2125
|
-
except
|
|
2799
|
+
except OSError as ex:
|
|
2126
2800
|
logger(logging.ERROR, "Couldn't open file '%s'", template_path, exc_info=ex)
|
|
2127
2801
|
return
|
|
2128
2802
|
|
|
@@ -2147,12 +2821,18 @@ def generate_email_for_rule_ok_notification(rule, *, session: "Session", logger=
|
|
|
2147
2821
|
add_message(event_type='email',
|
|
2148
2822
|
payload={'body': email_body,
|
|
2149
2823
|
'to': [email],
|
|
2150
|
-
'subject': '[RUCIO] Replication rule %s has been
|
|
2824
|
+
'subject': '[RUCIO] Replication rule %s has been successfully transferred' % (str(rule.id))},
|
|
2151
2825
|
session=session)
|
|
2152
2826
|
|
|
2153
2827
|
|
|
2154
2828
|
@transactional_session
|
|
2155
|
-
def insert_rule_history(
|
|
2829
|
+
def insert_rule_history(
|
|
2830
|
+
rule: models.ReplicationRule,
|
|
2831
|
+
recent: bool = True,
|
|
2832
|
+
longterm: bool = False,
|
|
2833
|
+
*,
|
|
2834
|
+
session: "Session"
|
|
2835
|
+
) -> None:
|
|
2156
2836
|
"""
|
|
2157
2837
|
Insert rule history to recent/longterm history.
|
|
2158
2838
|
|
|
@@ -2182,7 +2862,13 @@ def insert_rule_history(rule, recent=True, longterm=False, *, session: "Session"
|
|
|
2182
2862
|
|
|
2183
2863
|
|
|
2184
2864
|
@transactional_session
|
|
2185
|
-
def approve_rule(
|
|
2865
|
+
def approve_rule(
|
|
2866
|
+
rule_id: str,
|
|
2867
|
+
approver: str = '',
|
|
2868
|
+
notify_approvers: bool = True,
|
|
2869
|
+
*,
|
|
2870
|
+
session: "Session"
|
|
2871
|
+
) -> None:
|
|
2186
2872
|
"""
|
|
2187
2873
|
Approve a specific replication rule.
|
|
2188
2874
|
|
|
@@ -2194,7 +2880,12 @@ def approve_rule(rule_id, approver=None, notify_approvers=True, *, session: "Ses
|
|
|
2194
2880
|
"""
|
|
2195
2881
|
|
|
2196
2882
|
try:
|
|
2197
|
-
|
|
2883
|
+
stmt = select(
|
|
2884
|
+
models.ReplicationRule
|
|
2885
|
+
).where(
|
|
2886
|
+
models.ReplicationRule.id == rule_id
|
|
2887
|
+
)
|
|
2888
|
+
rule = session.execute(stmt).scalar_one()
|
|
2198
2889
|
if rule.state == RuleState.WAITING_APPROVAL:
|
|
2199
2890
|
rule.ignore_account_limit = True
|
|
2200
2891
|
rule.state = RuleState.INJECT
|
|
@@ -2235,14 +2926,20 @@ def approve_rule(rule_id, approver=None, notify_approvers=True, *, session: "Ses
|
|
|
2235
2926
|
'to': [recipient[0]],
|
|
2236
2927
|
'subject': 'Re: [RUCIO] Request to approve replication rule %s' % (str(rule.id))},
|
|
2237
2928
|
session=session)
|
|
2238
|
-
except NoResultFound:
|
|
2239
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
2240
|
-
except StatementError:
|
|
2241
|
-
raise RucioException('Badly formatted rule id (%s)' % (rule_id))
|
|
2929
|
+
except NoResultFound as exc:
|
|
2930
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
2931
|
+
except StatementError as exc:
|
|
2932
|
+
raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
|
|
2242
2933
|
|
|
2243
2934
|
|
|
2244
2935
|
@transactional_session
|
|
2245
|
-
def deny_rule(
|
|
2936
|
+
def deny_rule(
|
|
2937
|
+
rule_id: str,
|
|
2938
|
+
approver: str = '',
|
|
2939
|
+
reason: Optional[str] = None,
|
|
2940
|
+
*,
|
|
2941
|
+
session: "Session"
|
|
2942
|
+
) -> None:
|
|
2246
2943
|
"""
|
|
2247
2944
|
Deny a specific replication rule.
|
|
2248
2945
|
|
|
@@ -2254,7 +2951,12 @@ def deny_rule(rule_id, approver=None, reason=None, *, session: "Session"):
|
|
|
2254
2951
|
"""
|
|
2255
2952
|
|
|
2256
2953
|
try:
|
|
2257
|
-
|
|
2954
|
+
stmt = select(
|
|
2955
|
+
models.ReplicationRule
|
|
2956
|
+
).where(
|
|
2957
|
+
models.ReplicationRule.id == rule_id
|
|
2958
|
+
)
|
|
2959
|
+
rule = session.execute(stmt).scalar_one()
|
|
2258
2960
|
if rule.state == RuleState.WAITING_APPROVAL:
|
|
2259
2961
|
with open('%s/rule_denied_user.tmpl' % config_get('common', 'mailtemplatedir'), 'r') as templatefile:
|
|
2260
2962
|
template = Template(templatefile.read())
|
|
@@ -2294,26 +2996,35 @@ def deny_rule(rule_id, approver=None, reason=None, *, session: "Session"):
|
|
|
2294
2996
|
'to': [recipient[0]],
|
|
2295
2997
|
'subject': 'Re: [RUCIO] Request to approve replication rule %s' % (str(rule.id))},
|
|
2296
2998
|
session=session)
|
|
2297
|
-
except NoResultFound:
|
|
2298
|
-
raise RuleNotFound('No rule with the id %s found' % rule_id)
|
|
2299
|
-
except StatementError:
|
|
2300
|
-
raise RucioException('Badly formatted rule id (%s)' % rule_id)
|
|
2999
|
+
except NoResultFound as exc:
|
|
3000
|
+
raise RuleNotFound('No rule with the id %s found' % rule_id) from exc
|
|
3001
|
+
except StatementError as exc:
|
|
3002
|
+
raise RucioException('Badly formatted rule id (%s)' % rule_id) from exc
|
|
2301
3003
|
|
|
2302
3004
|
|
|
2303
3005
|
@transactional_session
|
|
2304
|
-
def examine_rule(
|
|
3006
|
+
def examine_rule(
|
|
3007
|
+
rule_id: str,
|
|
3008
|
+
*,
|
|
3009
|
+
session: "Session"
|
|
3010
|
+
) -> dict[str, Any]:
|
|
2305
3011
|
"""
|
|
2306
3012
|
Examine a replication rule for transfer errors.
|
|
2307
3013
|
|
|
2308
3014
|
:param rule_id: Replication rule id
|
|
2309
3015
|
:param session: Session of the db.
|
|
2310
|
-
:returns: Dictionary of
|
|
3016
|
+
:returns: Dictionary of information
|
|
2311
3017
|
"""
|
|
2312
3018
|
result = {'rule_error': None,
|
|
2313
3019
|
'transfers': []}
|
|
2314
3020
|
|
|
2315
3021
|
try:
|
|
2316
|
-
|
|
3022
|
+
stmt = select(
|
|
3023
|
+
models.ReplicationRule
|
|
3024
|
+
).where(
|
|
3025
|
+
models.ReplicationRule.id == rule_id
|
|
3026
|
+
)
|
|
3027
|
+
rule = session.execute(stmt).scalar_one()
|
|
2317
3028
|
if rule.state == RuleState.OK:
|
|
2318
3029
|
result['rule_error'] = 'This replication rule is OK'
|
|
2319
3030
|
elif rule.state == RuleState.REPLICATING:
|
|
@@ -2323,10 +3034,25 @@ def examine_rule(rule_id, *, session: "Session"):
|
|
|
2323
3034
|
else:
|
|
2324
3035
|
result['rule_error'] = rule.error
|
|
2325
3036
|
# Get the stuck locks
|
|
2326
|
-
|
|
3037
|
+
stmt = select(
|
|
3038
|
+
models.ReplicaLock
|
|
3039
|
+
).where(
|
|
3040
|
+
and_(models.ReplicaLock.rule_id == rule_id,
|
|
3041
|
+
models.ReplicaLock.state == LockState.STUCK)
|
|
3042
|
+
)
|
|
3043
|
+
stuck_locks = session.execute(stmt).scalars().all()
|
|
2327
3044
|
for lock in stuck_locks:
|
|
2328
3045
|
# Get the count of requests in the request_history for each lock
|
|
2329
|
-
|
|
3046
|
+
stmt = select(
|
|
3047
|
+
models.RequestHistory
|
|
3048
|
+
).where(
|
|
3049
|
+
and_(models.RequestHistory.scope == lock.scope,
|
|
3050
|
+
models.RequestHistory.name == lock.name,
|
|
3051
|
+
models.RequestHistory.dest_rse_id == lock.rse_id)
|
|
3052
|
+
).order_by(
|
|
3053
|
+
desc(models.RequestHistory.created_at)
|
|
3054
|
+
)
|
|
3055
|
+
transfers = session.execute(stmt).scalars().all()
|
|
2330
3056
|
transfer_cnt = len(transfers)
|
|
2331
3057
|
# Get the error of the last request that has been tried and also the SOURCE used for the last request
|
|
2332
3058
|
last_error, last_source, last_time, sources = None, None, None, []
|
|
@@ -2335,7 +3061,14 @@ def examine_rule(rule_id, *, session: "Session"):
|
|
|
2335
3061
|
last_error = last_request.state
|
|
2336
3062
|
last_time = last_request.created_at
|
|
2337
3063
|
last_source = None if last_request.source_rse_id is None else get_rse_name(rse_id=last_request.source_rse_id, session=session)
|
|
2338
|
-
|
|
3064
|
+
stmt = select(
|
|
3065
|
+
models.RSEFileAssociation
|
|
3066
|
+
).where(
|
|
3067
|
+
and_(models.RSEFileAssociation.scope == lock.scope,
|
|
3068
|
+
models.RSEFileAssociation.name == lock.name,
|
|
3069
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
|
|
3070
|
+
)
|
|
3071
|
+
available_replicas = session.execute(stmt).scalars().all()
|
|
2339
3072
|
|
|
2340
3073
|
for replica in available_replicas:
|
|
2341
3074
|
sources.append((get_rse_name(rse_id=replica.rse_id, session=session),
|
|
@@ -2351,14 +3084,18 @@ def examine_rule(rule_id, *, session: "Session"):
|
|
|
2351
3084
|
'sources': sources,
|
|
2352
3085
|
'last_time': last_time})
|
|
2353
3086
|
return result
|
|
2354
|
-
except NoResultFound:
|
|
2355
|
-
raise RuleNotFound('No rule with the id %s found' % (rule_id))
|
|
2356
|
-
except StatementError:
|
|
2357
|
-
raise RucioException('Badly formatted rule id (%s)' % (rule_id))
|
|
3087
|
+
except NoResultFound as exc:
|
|
3088
|
+
raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
|
|
3089
|
+
except StatementError as exc:
|
|
3090
|
+
raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
|
|
2358
3091
|
|
|
2359
3092
|
|
|
2360
3093
|
@transactional_session
|
|
2361
|
-
def get_evaluation_backlog(
|
|
3094
|
+
def get_evaluation_backlog(
|
|
3095
|
+
expiration_time: int = 600,
|
|
3096
|
+
*,
|
|
3097
|
+
session: "Session"
|
|
3098
|
+
) -> tuple[int, datetime]:
|
|
2362
3099
|
"""
|
|
2363
3100
|
Counts the number of entries in the rule evaluation backlog.
|
|
2364
3101
|
(Number of files to be evaluated)
|
|
@@ -2366,15 +3103,25 @@ def get_evaluation_backlog(expiration_time=600, *, session: "Session"):
|
|
|
2366
3103
|
:returns: Tuple (Count, Datetime of oldest entry)
|
|
2367
3104
|
"""
|
|
2368
3105
|
|
|
2369
|
-
|
|
2370
|
-
if
|
|
2371
|
-
|
|
3106
|
+
cached_backlog: Union[NoValue, tuple[int, datetime]] = REGION.get('rule_evaluation_backlog', expiration_time=expiration_time)
|
|
3107
|
+
if isinstance(cached_backlog, NoValue):
|
|
3108
|
+
stmt = select(
|
|
3109
|
+
func.count(models.UpdatedDID.created_at),
|
|
3110
|
+
func.min(models.UpdatedDID.created_at)
|
|
3111
|
+
)
|
|
3112
|
+
result = session.execute(stmt).one()._tuple()
|
|
2372
3113
|
REGION.set('rule_evaluation_backlog', result)
|
|
2373
|
-
|
|
3114
|
+
return result
|
|
3115
|
+
return cached_backlog
|
|
2374
3116
|
|
|
2375
3117
|
|
|
2376
3118
|
@transactional_session
|
|
2377
|
-
def release_parent_rule(
|
|
3119
|
+
def release_parent_rule(
|
|
3120
|
+
child_rule_id: str,
|
|
3121
|
+
remove_parent_expiration: bool = False,
|
|
3122
|
+
*,
|
|
3123
|
+
session: "Session"
|
|
3124
|
+
) -> None:
|
|
2378
3125
|
"""
|
|
2379
3126
|
Release a potential parent rule, because the child_rule is OK.
|
|
2380
3127
|
|
|
@@ -2385,8 +3132,14 @@ def release_parent_rule(child_rule_id, remove_parent_expiration=False, *, sessio
|
|
|
2385
3132
|
|
|
2386
3133
|
session.flush()
|
|
2387
3134
|
|
|
2388
|
-
|
|
2389
|
-
|
|
3135
|
+
stmt = select(
|
|
3136
|
+
models.ReplicationRule
|
|
3137
|
+
).with_hint(
|
|
3138
|
+
models.ReplicationRule, 'INDEX(RULES RULES_CHILD_RULE_ID_IDX)', 'oracle'
|
|
3139
|
+
).where(
|
|
3140
|
+
models.ReplicationRule.child_rule_id == child_rule_id
|
|
3141
|
+
)
|
|
3142
|
+
parent_rules = session.execute(stmt).scalars().all()
|
|
2390
3143
|
for rule in parent_rules:
|
|
2391
3144
|
if remove_parent_expiration:
|
|
2392
3145
|
rule.expires_at = None
|
|
@@ -2394,18 +3147,95 @@ def release_parent_rule(child_rule_id, remove_parent_expiration=False, *, sessio
|
|
|
2394
3147
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
2395
3148
|
|
|
2396
3149
|
|
|
3150
|
+
@stream_session
|
|
3151
|
+
def list_rules_for_rse_decommissioning(
|
|
3152
|
+
rse_id: str,
|
|
3153
|
+
*,
|
|
3154
|
+
session: "Session"
|
|
3155
|
+
) -> Iterator[dict[str, Any]]:
|
|
3156
|
+
"""Return a generator of rules at the RSE that is being decommissioned.
|
|
3157
|
+
|
|
3158
|
+
Decommissioning of an RSE involves deleting or moving away all rules that are
|
|
3159
|
+
locking the replicas that exist at the RSE. The rules can be enforcing
|
|
3160
|
+
dataset-level and/or file-level locks. Because rules are defined in terms of
|
|
3161
|
+
RSE expressions, we need to first identify the locks with the RSE id and make
|
|
3162
|
+
the list of rules that are enforcing such locks.
|
|
3163
|
+
This function has two yield statements corresponding to the two types
|
|
3164
|
+
(dataset-level and file-level) of locks. To avoid listing duplicates, the
|
|
3165
|
+
rules identified through the dataset-level locks are excluded from the
|
|
3166
|
+
second query using file-level locks.
|
|
3167
|
+
|
|
3168
|
+
:param rse_id: Id of the RSE being decommissioned.
|
|
3169
|
+
:param session: The database session in use.
|
|
3170
|
+
:returns: A generator that yields rule dictionaries.
|
|
3171
|
+
"""
|
|
3172
|
+
# Get rules with dataset locks first.
|
|
3173
|
+
query_rules_from_dataset_locks = select(
|
|
3174
|
+
models.ReplicationRule
|
|
3175
|
+
).distinct(
|
|
3176
|
+
).join_from(
|
|
3177
|
+
models.DatasetLock,
|
|
3178
|
+
models.ReplicationRule,
|
|
3179
|
+
models.DatasetLock.rule_id == models.ReplicationRule.id
|
|
3180
|
+
).where(
|
|
3181
|
+
models.DatasetLock.rse_id == rse_id
|
|
3182
|
+
)
|
|
3183
|
+
|
|
3184
|
+
for rule in session.execute(query_rules_from_dataset_locks).yield_per(5).scalars():
|
|
3185
|
+
yield rule.to_dict()
|
|
3186
|
+
|
|
3187
|
+
# Make a subquery from the previous query to be excluded from the next query
|
|
3188
|
+
dataset_rule_ids = query_rules_from_dataset_locks.with_only_columns(models.ReplicationRule.id)
|
|
3189
|
+
|
|
3190
|
+
# ReplicaLock ("locks") table is not indexed by RSE ID, so we instead go
|
|
3191
|
+
# through the RSEFileAssociation ("replicas") table.
|
|
3192
|
+
query_rules_from_replicas = select(
|
|
3193
|
+
models.ReplicationRule
|
|
3194
|
+
).prefix_with(
|
|
3195
|
+
'/*+ USE_NL(locks) LEADING(replicas locks) */',
|
|
3196
|
+
dialect='oracle'
|
|
3197
|
+
).distinct(
|
|
3198
|
+
).join_from(
|
|
3199
|
+
models.RSEFileAssociation,
|
|
3200
|
+
models.ReplicaLock,
|
|
3201
|
+
and_(models.RSEFileAssociation.scope == models.ReplicaLock.scope,
|
|
3202
|
+
models.RSEFileAssociation.name == models.ReplicaLock.name,
|
|
3203
|
+
models.RSEFileAssociation.rse_id == models.ReplicaLock.rse_id)
|
|
3204
|
+
).join(
|
|
3205
|
+
models.ReplicationRule,
|
|
3206
|
+
models.ReplicaLock.rule_id == models.ReplicationRule.id
|
|
3207
|
+
).where(
|
|
3208
|
+
models.RSEFileAssociation.rse_id == rse_id,
|
|
3209
|
+
models.ReplicaLock.rule_id.not_in(dataset_rule_ids)
|
|
3210
|
+
)
|
|
3211
|
+
|
|
3212
|
+
for rule in session.execute(query_rules_from_replicas).yield_per(5).scalars():
|
|
3213
|
+
yield rule.to_dict()
|
|
3214
|
+
|
|
3215
|
+
|
|
2397
3216
|
@transactional_session
|
|
2398
|
-
def __find_missing_locks_and_create_them(
|
|
3217
|
+
def __find_missing_locks_and_create_them(
|
|
3218
|
+
datasetfiles: Sequence[dict[str, Any]],
|
|
3219
|
+
locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
|
|
3220
|
+
replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
3221
|
+
source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
3222
|
+
rseselector: RSESelector,
|
|
3223
|
+
rule: models.ReplicationRule,
|
|
3224
|
+
source_rses: Sequence[str],
|
|
3225
|
+
*,
|
|
3226
|
+
session: "Session",
|
|
3227
|
+
logger: LoggerFunction = logging.log
|
|
3228
|
+
) -> None:
|
|
2399
3229
|
"""
|
|
2400
3230
|
Find missing locks for a rule and create them.
|
|
2401
3231
|
|
|
2402
|
-
:param datasetfiles:
|
|
3232
|
+
:param datasetfiles: Sequence of dicts holding all datasets and files.
|
|
2403
3233
|
:param locks: Dict holding locks.
|
|
2404
3234
|
:param replicas: Dict holding replicas.
|
|
2405
3235
|
:param source_replicas: Dict holding source replicas.
|
|
2406
3236
|
:param rseselector: The RSESelector to be used.
|
|
2407
3237
|
:param rule: The rule.
|
|
2408
|
-
:param source_rses: RSE ids for
|
|
3238
|
+
:param source_rses: RSE ids for eligible source RSEs.
|
|
2409
3239
|
:param session: Session of the db.
|
|
2410
3240
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
2411
3241
|
:raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
|
|
@@ -2442,17 +3272,20 @@ def __find_missing_locks_and_create_them(datasetfiles, locks, replicas, source_r
|
|
|
2442
3272
|
|
|
2443
3273
|
|
|
2444
3274
|
@transactional_session
|
|
2445
|
-
def __find_surplus_locks_and_remove_them(
|
|
3275
|
+
def __find_surplus_locks_and_remove_them(
|
|
3276
|
+
datasetfiles: Sequence[dict[str, Any]],
|
|
3277
|
+
locks: dict[tuple[InternalScope, str], list[models.ReplicaLock]],
|
|
3278
|
+
rule: models.ReplicationRule,
|
|
3279
|
+
*,
|
|
3280
|
+
session: "Session",
|
|
3281
|
+
logger: LoggerFunction = logging.log
|
|
3282
|
+
) -> None:
|
|
2446
3283
|
"""
|
|
2447
3284
|
Find surplocks locks for a rule and delete them.
|
|
2448
3285
|
|
|
2449
3286
|
:param datasetfiles: Dict holding all datasets and files.
|
|
2450
3287
|
:param locks: Dict holding locks.
|
|
2451
|
-
:param replicas: Dict holding replicas.
|
|
2452
|
-
:param source_replicas: Dict holding all source replicas.
|
|
2453
|
-
:param rseselector: The RSESelector to be used.
|
|
2454
3288
|
:param rule: The rule.
|
|
2455
|
-
:param source_rses: RSE ids for eglible source RSEs.
|
|
2456
3289
|
:param session: Session of the db.
|
|
2457
3290
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
2458
3291
|
:raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
|
|
@@ -2490,7 +3323,18 @@ def __find_surplus_locks_and_remove_them(datasetfiles, locks, replicas, source_r
|
|
|
2490
3323
|
|
|
2491
3324
|
|
|
2492
3325
|
@transactional_session
|
|
2493
|
-
def __find_stuck_locks_and_repair_them(
|
|
3326
|
+
def __find_stuck_locks_and_repair_them(
|
|
3327
|
+
datasetfiles: Sequence[dict[str, Any]],
|
|
3328
|
+
locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
|
|
3329
|
+
replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
3330
|
+
source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
3331
|
+
rseselector: RSESelector,
|
|
3332
|
+
rule: models.ReplicationRule,
|
|
3333
|
+
source_rses: Sequence[str],
|
|
3334
|
+
*,
|
|
3335
|
+
session: "Session",
|
|
3336
|
+
logger: LoggerFunction = logging.log
|
|
3337
|
+
) -> None:
|
|
2494
3338
|
"""
|
|
2495
3339
|
Find stuck locks for a rule and repair them.
|
|
2496
3340
|
|
|
@@ -2500,7 +3344,7 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
|
|
|
2500
3344
|
:param source_replicas: Dict holding source replicas.
|
|
2501
3345
|
:param rseselector: The RSESelector to be used.
|
|
2502
3346
|
:param rule: The rule.
|
|
2503
|
-
:param source_rses: RSE ids of
|
|
3347
|
+
:param source_rses: RSE ids of eligible source RSEs.
|
|
2504
3348
|
:param session: Session of the db.
|
|
2505
3349
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
2506
3350
|
:raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
|
|
@@ -2509,7 +3353,7 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
|
|
|
2509
3353
|
|
|
2510
3354
|
logger(logging.DEBUG, "Finding and repairing stuck locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
2511
3355
|
|
|
2512
|
-
replicas_to_create, locks_to_create, transfers_to_create
|
|
3356
|
+
replicas_to_create, locks_to_create, transfers_to_create, \
|
|
2513
3357
|
locks_to_delete = repair_stuck_locks_and_apply_rule_grouping(datasetfiles=datasetfiles,
|
|
2514
3358
|
locks=locks,
|
|
2515
3359
|
replicas=replicas,
|
|
@@ -2549,7 +3393,12 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
|
|
|
2549
3393
|
|
|
2550
3394
|
|
|
2551
3395
|
@transactional_session
|
|
2552
|
-
def __evaluate_did_detach(
|
|
3396
|
+
def __evaluate_did_detach(
|
|
3397
|
+
eval_did: models.DataIdentifier,
|
|
3398
|
+
*,
|
|
3399
|
+
session: "Session",
|
|
3400
|
+
logger: LoggerFunction = logging.log
|
|
3401
|
+
) -> None:
|
|
2553
3402
|
"""
|
|
2554
3403
|
Evaluate a parent did which has children removed.
|
|
2555
3404
|
|
|
@@ -2566,9 +3415,25 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2566
3415
|
parent_dids = rucio.core.did.list_all_parent_dids(scope=eval_did.scope, name=eval_did.name, session=session)
|
|
2567
3416
|
|
|
2568
3417
|
# Get all RR from parents and eval_did
|
|
2569
|
-
|
|
3418
|
+
stmt = select(
|
|
3419
|
+
models.ReplicationRule
|
|
3420
|
+
).where(
|
|
3421
|
+
and_(models.ReplicationRule.scope == eval_did.scope,
|
|
3422
|
+
models.ReplicationRule.name == eval_did.name)
|
|
3423
|
+
).with_for_update(
|
|
3424
|
+
nowait=True
|
|
3425
|
+
)
|
|
3426
|
+
rules = list(session.execute(stmt).scalars().all())
|
|
2570
3427
|
for did in parent_dids:
|
|
2571
|
-
|
|
3428
|
+
stmt = select(
|
|
3429
|
+
models.ReplicationRule
|
|
3430
|
+
).where(
|
|
3431
|
+
and_(models.ReplicationRule.scope == did['scope'],
|
|
3432
|
+
models.ReplicationRule.name == did['name'])
|
|
3433
|
+
).with_for_update(
|
|
3434
|
+
nowait=True
|
|
3435
|
+
)
|
|
3436
|
+
rules.extend(session.execute(stmt).scalars().all())
|
|
2572
3437
|
|
|
2573
3438
|
# Iterate rules and delete locks
|
|
2574
3439
|
transfers_to_delete = [] # [{'scope': , 'name':, 'rse_id':}]
|
|
@@ -2580,8 +3445,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2580
3445
|
files[(file['scope'], file['name'])] = True
|
|
2581
3446
|
logger(logging.DEBUG, "Removing locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
2582
3447
|
rule_locks_ok_cnt_before = rule.locks_ok_cnt
|
|
2583
|
-
|
|
2584
|
-
|
|
3448
|
+
stmt = select(
|
|
3449
|
+
models.ReplicaLock
|
|
3450
|
+
).where(
|
|
3451
|
+
models.ReplicaLock.rule_id == rule.id
|
|
3452
|
+
)
|
|
3453
|
+
for lock in session.execute(stmt).scalars().all():
|
|
2585
3454
|
if (lock.scope, lock.name) not in files:
|
|
2586
3455
|
if __delete_lock_and_update_replica(lock=lock, purge_replicas=force_epoch or rule.purge_replicas, nowait=True, session=session):
|
|
2587
3456
|
transfers_to_delete.append({'scope': lock.scope, 'name': lock.name, 'rse_id': lock.rse_id})
|
|
@@ -2602,7 +3471,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2602
3471
|
for ds in rucio.core.did.list_child_datasets(scope=rule.scope, name=rule.name, session=session):
|
|
2603
3472
|
child_datasets[(ds['scope'], ds['name'])] = True
|
|
2604
3473
|
logger(logging.DEBUG, "Removing dataset_locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
2605
|
-
|
|
3474
|
+
stmt = select(
|
|
3475
|
+
models.DatasetLock
|
|
3476
|
+
).where(
|
|
3477
|
+
models.DatasetLock.rule_id == rule.id
|
|
3478
|
+
)
|
|
3479
|
+
query = session.execute(stmt).scalars().all()
|
|
2606
3480
|
for ds_lock in query:
|
|
2607
3481
|
if (ds_lock.scope, ds_lock.name) not in child_datasets:
|
|
2608
3482
|
ds_lock.delete(flush=False, session=session)
|
|
@@ -2615,7 +3489,14 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2615
3489
|
elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
|
|
2616
3490
|
rule.state = RuleState.OK
|
|
2617
3491
|
if rule.grouping != RuleGrouping.NONE:
|
|
2618
|
-
|
|
3492
|
+
stmt = update(
|
|
3493
|
+
models.DatasetLock
|
|
3494
|
+
).where(
|
|
3495
|
+
models.DatasetLock.rule_id == rule.id
|
|
3496
|
+
).values({
|
|
3497
|
+
models.DatasetLock.state: LockState.OK
|
|
3498
|
+
})
|
|
3499
|
+
session.execute(stmt)
|
|
2619
3500
|
session.flush()
|
|
2620
3501
|
if rule_locks_ok_cnt_before != rule.locks_ok_cnt:
|
|
2621
3502
|
generate_rule_notifications(rule=rule, session=session)
|
|
@@ -2638,7 +3519,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2638
3519
|
|
|
2639
3520
|
|
|
2640
3521
|
@transactional_session
|
|
2641
|
-
def __oldest_file_under(
|
|
3522
|
+
def __oldest_file_under(
|
|
3523
|
+
scope: InternalScope,
|
|
3524
|
+
name: str,
|
|
3525
|
+
*,
|
|
3526
|
+
session: "Session"
|
|
3527
|
+
) -> Optional[tuple[InternalScope, str]]:
|
|
2642
3528
|
"""
|
|
2643
3529
|
Finds oldest file in oldest container/dataset in the container or the dataset, recursively.
|
|
2644
3530
|
Oldest means attached to its parent first.
|
|
@@ -2647,10 +3533,15 @@ def __oldest_file_under(scope, name, *, session: "Session"):
|
|
|
2647
3533
|
:param name: dataset or container name
|
|
2648
3534
|
:returns: tuple (scope, name) or None
|
|
2649
3535
|
"""
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
3536
|
+
stmt = select(
|
|
3537
|
+
models.DataIdentifierAssociation
|
|
3538
|
+
).where(
|
|
3539
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
3540
|
+
models.DataIdentifierAssociation.name == name)
|
|
3541
|
+
).order_by(
|
|
3542
|
+
models.DataIdentifierAssociation.created_at
|
|
3543
|
+
)
|
|
3544
|
+
children = session.execute(stmt).scalars().all()
|
|
2654
3545
|
for child in children:
|
|
2655
3546
|
if child.child_type == DIDType.FILE:
|
|
2656
3547
|
return child.child_scope, child.child_name
|
|
@@ -2662,9 +3553,14 @@ def __oldest_file_under(scope, name, *, session: "Session"):
|
|
|
2662
3553
|
|
|
2663
3554
|
|
|
2664
3555
|
@transactional_session
|
|
2665
|
-
def __evaluate_did_attach(
|
|
3556
|
+
def __evaluate_did_attach(
|
|
3557
|
+
eval_did: models.DataIdentifier,
|
|
3558
|
+
*,
|
|
3559
|
+
session: "Session",
|
|
3560
|
+
logger: LoggerFunction = logging.log
|
|
3561
|
+
) -> None:
|
|
2666
3562
|
"""
|
|
2667
|
-
Evaluate a parent did which has new
|
|
3563
|
+
Evaluate a parent did which has new children
|
|
2668
3564
|
|
|
2669
3565
|
:param eval_did: The did object in use.
|
|
2670
3566
|
:param session: The database session in use.
|
|
@@ -2681,12 +3577,16 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2681
3577
|
|
|
2682
3578
|
# Get immediate new child DID's
|
|
2683
3579
|
with METRICS.timer('evaluate_did_attach.list_new_child_dids'):
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
3580
|
+
stmt = select(
|
|
3581
|
+
models.DataIdentifierAssociation
|
|
3582
|
+
).with_hint(
|
|
3583
|
+
models.DataIdentifierAssociation, 'INDEX_RS_ASC(CONTENTS CONTENTS_PK)', 'oracle'
|
|
3584
|
+
).where(
|
|
3585
|
+
and_(models.DataIdentifierAssociation.scope == eval_did.scope,
|
|
3586
|
+
models.DataIdentifierAssociation.name == eval_did.name,
|
|
3587
|
+
models.DataIdentifierAssociation.rule_evaluation == true())
|
|
3588
|
+
)
|
|
3589
|
+
new_child_dids = session.execute(stmt).scalars().all()
|
|
2690
3590
|
if new_child_dids:
|
|
2691
3591
|
# Get all unsuspended RR from parents and eval_did
|
|
2692
3592
|
with METRICS.timer('evaluate_did_attach.get_rules'):
|
|
@@ -2696,12 +3596,17 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2696
3596
|
models.ReplicationRule.name == did['name']))
|
|
2697
3597
|
rule_clauses.append(and_(models.ReplicationRule.scope == eval_did.scope,
|
|
2698
3598
|
models.ReplicationRule.name == eval_did.name))
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
3599
|
+
stmt = select(
|
|
3600
|
+
models.ReplicationRule
|
|
3601
|
+
).where(
|
|
3602
|
+
and_(or_(*rule_clauses),
|
|
3603
|
+
models.ReplicationRule.state.not_in([RuleState.SUSPENDED,
|
|
3604
|
+
RuleState.WAITING_APPROVAL,
|
|
3605
|
+
RuleState.INJECT]))
|
|
3606
|
+
).with_for_update(
|
|
3607
|
+
nowait=True
|
|
3608
|
+
)
|
|
3609
|
+
rules = session.execute(stmt).scalars().all()
|
|
2705
3610
|
if rules:
|
|
2706
3611
|
# Resolve the new_child_dids to its locks
|
|
2707
3612
|
with METRICS.timer('evaluate_did_attach.resolve_did_to_locks_and_replicas'):
|
|
@@ -2754,7 +3659,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2754
3659
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
2755
3660
|
# Try to update the DatasetLocks
|
|
2756
3661
|
if rule.grouping != RuleGrouping.NONE:
|
|
2757
|
-
|
|
3662
|
+
stmt = update(
|
|
3663
|
+
models.DatasetLock
|
|
3664
|
+
).where(
|
|
3665
|
+
models.DatasetLock.rule_id == rule.id
|
|
3666
|
+
).values({
|
|
3667
|
+
models.DatasetLock.state: LockState.STUCK
|
|
3668
|
+
})
|
|
3669
|
+
session.execute(stmt)
|
|
2758
3670
|
continue
|
|
2759
3671
|
|
|
2760
3672
|
# 2. Create the RSE Selector
|
|
@@ -2773,7 +3685,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2773
3685
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
2774
3686
|
# Try to update the DatasetLocks
|
|
2775
3687
|
if rule.grouping != RuleGrouping.NONE:
|
|
2776
|
-
|
|
3688
|
+
stmt = update(
|
|
3689
|
+
models.DatasetLock
|
|
3690
|
+
).where(
|
|
3691
|
+
models.DatasetLock.rule_id == rule.id
|
|
3692
|
+
).values({
|
|
3693
|
+
models.DatasetLock.state: LockState.STUCK
|
|
3694
|
+
})
|
|
3695
|
+
session.execute(stmt)
|
|
2777
3696
|
continue
|
|
2778
3697
|
|
|
2779
3698
|
# 3. Apply the Replication rule to the Files
|
|
@@ -2816,7 +3735,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2816
3735
|
insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
2817
3736
|
# Try to update the DatasetLocks
|
|
2818
3737
|
if rule.grouping != RuleGrouping.NONE:
|
|
2819
|
-
|
|
3738
|
+
stmt = update(
|
|
3739
|
+
models.DatasetLock
|
|
3740
|
+
).where(
|
|
3741
|
+
models.DatasetLock.rule_id == rule.id
|
|
3742
|
+
).values({
|
|
3743
|
+
models.DatasetLock.state: LockState.STUCK
|
|
3744
|
+
})
|
|
3745
|
+
session.execute(stmt)
|
|
2820
3746
|
continue
|
|
2821
3747
|
|
|
2822
3748
|
# 4. Update the Rule State
|
|
@@ -2826,15 +3752,36 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2826
3752
|
if locks_stuck_before != rule.locks_stuck_cnt:
|
|
2827
3753
|
rule.state = RuleState.STUCK
|
|
2828
3754
|
rule.error = 'MissingSourceReplica'
|
|
2829
|
-
|
|
3755
|
+
stmt = update(
|
|
3756
|
+
models.DatasetLock
|
|
3757
|
+
).where(
|
|
3758
|
+
models.DatasetLock.rule_id == rule.id
|
|
3759
|
+
).values({
|
|
3760
|
+
models.DatasetLock.state: LockState.STUCK
|
|
3761
|
+
})
|
|
3762
|
+
session.execute(stmt)
|
|
2830
3763
|
elif rule.locks_replicating_cnt > 0:
|
|
2831
3764
|
rule.state = RuleState.REPLICATING
|
|
2832
3765
|
if rule.grouping != RuleGrouping.NONE:
|
|
2833
|
-
|
|
3766
|
+
stmt = update(
|
|
3767
|
+
models.DatasetLock
|
|
3768
|
+
).where(
|
|
3769
|
+
models.DatasetLock.rule_id == rule.id
|
|
3770
|
+
).values({
|
|
3771
|
+
models.DatasetLock.state: LockState.REPLICATING
|
|
3772
|
+
})
|
|
3773
|
+
session.execute(stmt)
|
|
2834
3774
|
elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
|
|
2835
3775
|
rule.state = RuleState.OK
|
|
2836
3776
|
if rule.grouping != RuleGrouping.NONE:
|
|
2837
|
-
|
|
3777
|
+
stmt = update(
|
|
3778
|
+
models.DatasetLock
|
|
3779
|
+
).where(
|
|
3780
|
+
models.DatasetLock.rule_id == rule.id
|
|
3781
|
+
).values({
|
|
3782
|
+
models.DatasetLock.state: LockState.OK
|
|
3783
|
+
})
|
|
3784
|
+
session.execute(stmt)
|
|
2838
3785
|
session.flush()
|
|
2839
3786
|
if rule_locks_ok_cnt_before < rule.locks_ok_cnt:
|
|
2840
3787
|
generate_rule_notifications(rule=rule, session=session)
|
|
@@ -2852,9 +3799,20 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
|
|
|
2852
3799
|
|
|
2853
3800
|
|
|
2854
3801
|
@transactional_session
|
|
2855
|
-
def __resolve_did_to_locks_and_replicas(
|
|
3802
|
+
def __resolve_did_to_locks_and_replicas(
|
|
3803
|
+
did: models.DataIdentifier,
|
|
3804
|
+
nowait: bool = False,
|
|
3805
|
+
restrict_rses: Optional[Sequence[str]] = None,
|
|
3806
|
+
source_rses: Optional[Sequence[str]] = None,
|
|
3807
|
+
only_stuck: bool = False,
|
|
3808
|
+
*,
|
|
3809
|
+
session: "Session"
|
|
3810
|
+
) -> tuple[list[dict[str, Any]],
|
|
3811
|
+
dict[tuple[str, str], models.ReplicaLock],
|
|
3812
|
+
dict[tuple[str, str], models.RSEFileAssociation],
|
|
3813
|
+
dict[tuple[str, str], str]]:
|
|
2856
3814
|
"""
|
|
2857
|
-
Resolves a did to its constituent
|
|
3815
|
+
Resolves a did to its constituent children and reads the locks and replicas of all the constituent files.
|
|
2858
3816
|
|
|
2859
3817
|
:param did: The db object of the did the rule is applied on.
|
|
2860
3818
|
:param nowait: Nowait parameter for the FOR UPDATE statement.
|
|
@@ -2862,7 +3820,7 @@ def __resolve_did_to_locks_and_replicas(did, nowait=False, restrict_rses=None, s
|
|
|
2862
3820
|
:param source_rses: Source rses for this rule. These replicas are not row-locked.
|
|
2863
3821
|
:param only_stuck: Get results only for STUCK locks, if True.
|
|
2864
3822
|
:param session: Session of the db.
|
|
2865
|
-
:returns: (datasetfiles, locks, replicas)
|
|
3823
|
+
:returns: (datasetfiles, locks, replicas, source_replicas)
|
|
2866
3824
|
"""
|
|
2867
3825
|
|
|
2868
3826
|
datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}]
|
|
@@ -2949,16 +3907,26 @@ def __resolve_did_to_locks_and_replicas(did, nowait=False, restrict_rses=None, s
|
|
|
2949
3907
|
|
|
2950
3908
|
|
|
2951
3909
|
@transactional_session
|
|
2952
|
-
def __resolve_dids_to_locks_and_replicas(
|
|
3910
|
+
def __resolve_dids_to_locks_and_replicas(
|
|
3911
|
+
dids: Sequence[models.DataIdentifierAssociation],
|
|
3912
|
+
nowait: bool = False,
|
|
3913
|
+
restrict_rses: Optional[Sequence[str]] = None,
|
|
3914
|
+
source_rses: Optional[Sequence[str]] = None,
|
|
3915
|
+
*,
|
|
3916
|
+
session: "Session"
|
|
3917
|
+
) -> tuple[list[dict[str, Any]],
|
|
3918
|
+
dict[tuple[str, str], models.ReplicaLock],
|
|
3919
|
+
dict[tuple[str, str], models.RSEFileAssociation],
|
|
3920
|
+
dict[tuple[str, str], str]]:
|
|
2953
3921
|
"""
|
|
2954
|
-
Resolves a list of dids to its constituent
|
|
3922
|
+
Resolves a list of dids to its constituent children and reads the locks and replicas of all the constituent files.
|
|
2955
3923
|
|
|
2956
|
-
:param dids: The list of
|
|
3924
|
+
:param dids: The list of DataIdentifierAssociation objects.
|
|
2957
3925
|
:param nowait: Nowait parameter for the FOR UPDATE statement.
|
|
2958
3926
|
:param restrict_rses: Possible rses of the rule, so only these replica/locks should be considered.
|
|
2959
3927
|
:param source_rses: Source rses for this rule. These replicas are not row-locked.
|
|
2960
3928
|
:param session: Session of the db.
|
|
2961
|
-
:returns: (datasetfiles, locks, replicas)
|
|
3929
|
+
:returns: (datasetfiles, locks, replicas, source_replicas)
|
|
2962
3930
|
"""
|
|
2963
3931
|
|
|
2964
3932
|
datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}]
|
|
@@ -2966,6 +3934,7 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
2966
3934
|
locks = {} # {(scope,name): [SQLAlchemy]}
|
|
2967
3935
|
replicas = {} # {(scope, name): [SQLAlchemy]}
|
|
2968
3936
|
source_replicas = {} # {(scope, name): [rse_id]
|
|
3937
|
+
restrict_rses = restrict_rses or []
|
|
2969
3938
|
|
|
2970
3939
|
if dids[0].child_type == DIDType.FILE:
|
|
2971
3940
|
# All the dids will be files!
|
|
@@ -3006,13 +3975,28 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
3006
3975
|
|
|
3007
3976
|
for lock_clause_chunk in lock_clause_chunks:
|
|
3008
3977
|
if locks_rse_clause:
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3978
|
+
stmt = select(
|
|
3979
|
+
models.ReplicaLock
|
|
3980
|
+
).with_hint(
|
|
3981
|
+
models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
|
|
3982
|
+
).where(
|
|
3983
|
+
and_(or_(*lock_clause_chunk),
|
|
3984
|
+
or_(*locks_rse_clause))
|
|
3985
|
+
).with_for_update(
|
|
3986
|
+
nowait=nowait
|
|
3987
|
+
)
|
|
3988
|
+
tmp_locks = session.execute(stmt).scalars().all()
|
|
3012
3989
|
else:
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3990
|
+
stmt = select(
|
|
3991
|
+
models.ReplicaLock
|
|
3992
|
+
).with_hint(
|
|
3993
|
+
models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
|
|
3994
|
+
).where(
|
|
3995
|
+
or_(*lock_clause_chunk)
|
|
3996
|
+
).with_for_update(
|
|
3997
|
+
nowait=nowait
|
|
3998
|
+
)
|
|
3999
|
+
tmp_locks = session.execute(stmt).scalars().all()
|
|
3016
4000
|
for lock in tmp_locks:
|
|
3017
4001
|
if (lock.scope, lock.name) not in locks:
|
|
3018
4002
|
locks[(lock.scope, lock.name)] = [lock]
|
|
@@ -3021,13 +4005,30 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
3021
4005
|
|
|
3022
4006
|
for replica_clause_chunk in replica_clause_chunks:
|
|
3023
4007
|
if replicas_rse_clause:
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
4008
|
+
stmt = select(
|
|
4009
|
+
models.RSEFileAssociation
|
|
4010
|
+
).with_hint(
|
|
4011
|
+
models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
|
|
4012
|
+
).where(
|
|
4013
|
+
and_(or_(*replica_clause_chunk),
|
|
4014
|
+
or_(*replicas_rse_clause),
|
|
4015
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
|
|
4016
|
+
).with_for_update(
|
|
4017
|
+
nowait=nowait
|
|
4018
|
+
)
|
|
4019
|
+
tmp_replicas = session.execute(stmt).scalars().all()
|
|
3027
4020
|
else:
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
4021
|
+
stmt = select(
|
|
4022
|
+
models.RSEFileAssociation
|
|
4023
|
+
).with_hint(
|
|
4024
|
+
models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
|
|
4025
|
+
).where(
|
|
4026
|
+
and_(or_(*replica_clause_chunk),
|
|
4027
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
|
|
4028
|
+
).with_for_update(
|
|
4029
|
+
nowait=nowait
|
|
4030
|
+
)
|
|
4031
|
+
tmp_replicas = session.execute(stmt).scalars().all()
|
|
3031
4032
|
for replica in tmp_replicas:
|
|
3032
4033
|
if (replica.scope, replica.name) not in replicas:
|
|
3033
4034
|
replicas[(replica.scope, replica.name)] = [replica]
|
|
@@ -3036,9 +4037,18 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
3036
4037
|
|
|
3037
4038
|
if source_rses:
|
|
3038
4039
|
for replica_clause_chunk in replica_clause_chunks:
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
4040
|
+
stmt = select(
|
|
4041
|
+
models.RSEFileAssociation.scope,
|
|
4042
|
+
models.RSEFileAssociation.name,
|
|
4043
|
+
models.RSEFileAssociation.rse_id
|
|
4044
|
+
).with_hint(
|
|
4045
|
+
models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
|
|
4046
|
+
).where(
|
|
4047
|
+
and_(or_(*replica_clause_chunk),
|
|
4048
|
+
or_(*source_replicas_rse_clause),
|
|
4049
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
|
|
4050
|
+
)
|
|
4051
|
+
tmp_source_replicas = session.execute(stmt).all()
|
|
3042
4052
|
for scope, name, rse_id in tmp_source_replicas:
|
|
3043
4053
|
if (scope, name) not in source_replicas:
|
|
3044
4054
|
source_replicas[(scope, name)] = [rse_id]
|
|
@@ -3047,7 +4057,13 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
3047
4057
|
else:
|
|
3048
4058
|
# The evaluate_dids will be containers and/or datasets
|
|
3049
4059
|
for did in dids:
|
|
3050
|
-
|
|
4060
|
+
stmt = select(
|
|
4061
|
+
models.DataIdentifier
|
|
4062
|
+
).where(
|
|
4063
|
+
and_(models.DataIdentifier.scope == did.child_scope,
|
|
4064
|
+
models.DataIdentifier.name == did.child_name)
|
|
4065
|
+
)
|
|
4066
|
+
real_did = session.execute(stmt).scalar_one()
|
|
3051
4067
|
tmp_datasetfiles, tmp_locks, tmp_replicas, tmp_source_replicas = __resolve_did_to_locks_and_replicas(did=real_did,
|
|
3052
4068
|
nowait=nowait,
|
|
3053
4069
|
restrict_rses=restrict_rses,
|
|
@@ -3061,7 +4077,19 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
|
|
|
3061
4077
|
|
|
3062
4078
|
|
|
3063
4079
|
@transactional_session
|
|
3064
|
-
def __create_locks_replicas_transfers(
|
|
4080
|
+
def __create_locks_replicas_transfers(
|
|
4081
|
+
datasetfiles: Sequence[dict[str, Any]],
|
|
4082
|
+
locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
|
|
4083
|
+
replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
4084
|
+
source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
|
|
4085
|
+
rseselector: RSESelector,
|
|
4086
|
+
rule: models.ReplicationRule,
|
|
4087
|
+
preferred_rse_ids: Optional[Sequence[str]] = None,
|
|
4088
|
+
source_rses: Optional[Sequence[str]] = None,
|
|
4089
|
+
*,
|
|
4090
|
+
session: "Session",
|
|
4091
|
+
logger: LoggerFunction = logging.log
|
|
4092
|
+
) -> None:
|
|
3065
4093
|
"""
|
|
3066
4094
|
Apply a created replication rule to a set of files
|
|
3067
4095
|
|
|
@@ -3072,13 +4100,15 @@ def __create_locks_replicas_transfers(datasetfiles, locks, replicas, source_repl
|
|
|
3072
4100
|
:param rseselector: The RSESelector to be used.
|
|
3073
4101
|
:param rule: The rule.
|
|
3074
4102
|
:param preferred_rse_ids: Preferred RSE's to select.
|
|
3075
|
-
:param source_rses: RSE ids of
|
|
4103
|
+
:param source_rses: RSE ids of eligible source replicas.
|
|
3076
4104
|
:param session: Session of the db.
|
|
3077
4105
|
:param logger: Optional decorated logger that can be passed from the calling daemons or servers.
|
|
3078
4106
|
:raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs, RSEOverQuota
|
|
3079
4107
|
:attention: This method modifies the contents of the locks and replicas input parameters.
|
|
3080
4108
|
"""
|
|
3081
4109
|
|
|
4110
|
+
preferred_rse_ids = preferred_rse_ids or []
|
|
4111
|
+
source_rses = source_rses or []
|
|
3082
4112
|
logger(logging.DEBUG, "Creating locks and replicas for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
|
|
3083
4113
|
|
|
3084
4114
|
replicas_to_create, locks_to_create, transfers_to_create = apply_rule_grouping(datasetfiles=datasetfiles,
|
|
@@ -3114,7 +4144,14 @@ def __create_locks_replicas_transfers(datasetfiles, locks, replicas, source_repl
|
|
|
3114
4144
|
|
|
3115
4145
|
|
|
3116
4146
|
@transactional_session
|
|
3117
|
-
def __delete_lock_and_update_replica(
|
|
4147
|
+
def __delete_lock_and_update_replica(
|
|
4148
|
+
lock: models.ReplicaLock,
|
|
4149
|
+
purge_replicas: bool = False,
|
|
4150
|
+
nowait: bool = False,
|
|
4151
|
+
*,
|
|
4152
|
+
session: "Session",
|
|
4153
|
+
logger: LoggerFunction = logging.log
|
|
4154
|
+
) -> bool:
|
|
3118
4155
|
"""
|
|
3119
4156
|
Delete a lock and update the associated replica.
|
|
3120
4157
|
|
|
@@ -3129,10 +4166,16 @@ def __delete_lock_and_update_replica(lock, purge_replicas=False, nowait=False, *
|
|
|
3129
4166
|
logger(logging.DEBUG, "Deleting lock %s:%s for rule %s", lock.scope, lock.name, str(lock.rule_id))
|
|
3130
4167
|
lock.delete(session=session, flush=False)
|
|
3131
4168
|
try:
|
|
3132
|
-
|
|
3133
|
-
models.RSEFileAssociation
|
|
3134
|
-
|
|
3135
|
-
models.RSEFileAssociation.
|
|
4169
|
+
stmt = select(
|
|
4170
|
+
models.RSEFileAssociation
|
|
4171
|
+
).where(
|
|
4172
|
+
and_(models.RSEFileAssociation.scope == lock.scope,
|
|
4173
|
+
models.RSEFileAssociation.name == lock.name,
|
|
4174
|
+
models.RSEFileAssociation.rse_id == lock.rse_id)
|
|
4175
|
+
).with_for_update(
|
|
4176
|
+
nowait=nowait
|
|
4177
|
+
)
|
|
4178
|
+
replica = session.execute(stmt).scalar_one()
|
|
3136
4179
|
replica.lock_cnt -= 1
|
|
3137
4180
|
if replica.lock_cnt == 0:
|
|
3138
4181
|
if purge_replicas:
|
|
@@ -3153,7 +4196,11 @@ def __delete_lock_and_update_replica(lock, purge_replicas=False, nowait=False, *
|
|
|
3153
4196
|
|
|
3154
4197
|
|
|
3155
4198
|
@transactional_session
|
|
3156
|
-
def __create_rule_approval_email(
|
|
4199
|
+
def __create_rule_approval_email(
|
|
4200
|
+
rule: models.ReplicationRule,
|
|
4201
|
+
*,
|
|
4202
|
+
session: "Session"
|
|
4203
|
+
) -> None:
|
|
3157
4204
|
"""
|
|
3158
4205
|
Create the rule notification email.
|
|
3159
4206
|
|
|
@@ -3239,11 +4286,16 @@ def __create_rule_approval_email(rule: "models.ReplicationRule", *, session: "Se
|
|
|
3239
4286
|
|
|
3240
4287
|
|
|
3241
4288
|
@transactional_session
|
|
3242
|
-
def _create_recipients_list(
|
|
4289
|
+
def _create_recipients_list(
|
|
4290
|
+
rse_expression: str,
|
|
4291
|
+
filter_: Optional[str] = None,
|
|
4292
|
+
*,
|
|
4293
|
+
session: "Session"
|
|
4294
|
+
) -> list[tuple[str, Union[str, InternalAccount]]]:
|
|
3243
4295
|
"""
|
|
3244
4296
|
Create a list of recipients for a notification email based on rse_expression.
|
|
3245
4297
|
|
|
3246
|
-
:param
|
|
4298
|
+
:param rse_expression: The rse_expression.
|
|
3247
4299
|
:param session: The database session in use.
|
|
3248
4300
|
"""
|
|
3249
4301
|
|
|
@@ -3253,8 +4305,8 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
|
|
|
3253
4305
|
# If there are accounts in the approvers-list of any of the RSEs only these should be used
|
|
3254
4306
|
for rse in parse_expression(rse_expression, filter_=filter_, session=session):
|
|
3255
4307
|
rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
|
|
3256
|
-
if rse_attr.get(
|
|
3257
|
-
for account in rse_attr.get(
|
|
4308
|
+
if rse_attr.get(RseAttr.RULE_APPROVERS):
|
|
4309
|
+
for account in rse_attr.get(RseAttr.RULE_APPROVERS).split(','):
|
|
3258
4310
|
account = InternalAccount(account)
|
|
3259
4311
|
try:
|
|
3260
4312
|
email = get_account(account=account, session=session).email
|
|
@@ -3267,12 +4319,12 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
|
|
|
3267
4319
|
if not recipients:
|
|
3268
4320
|
for rse in parse_expression(rse_expression, filter_=filter_, session=session):
|
|
3269
4321
|
rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
|
|
3270
|
-
if rse_attr.get(
|
|
4322
|
+
if rse_attr.get(RseAttr.TYPE, '') in ('LOCALGROUPDISK', 'LOCALGROUPTAPE'):
|
|
3271
4323
|
|
|
3272
4324
|
query = select(
|
|
3273
4325
|
models.AccountAttrAssociation.account
|
|
3274
4326
|
).where(
|
|
3275
|
-
models.AccountAttrAssociation.key == f'country-{rse_attr.get(
|
|
4327
|
+
models.AccountAttrAssociation.key == f'country-{rse_attr.get(RseAttr.COUNTRY, "")}',
|
|
3276
4328
|
models.AccountAttrAssociation.value == 'admin'
|
|
3277
4329
|
)
|
|
3278
4330
|
|
|
@@ -3288,12 +4340,12 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
|
|
|
3288
4340
|
if not recipients:
|
|
3289
4341
|
for rse in parse_expression(rse_expression, filter_=filter_, session=session):
|
|
3290
4342
|
rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
|
|
3291
|
-
if rse_attr.get(
|
|
4343
|
+
if rse_attr.get(RseAttr.TYPE, '') == 'GROUPDISK':
|
|
3292
4344
|
|
|
3293
4345
|
query = select(
|
|
3294
4346
|
models.AccountAttrAssociation.account
|
|
3295
4347
|
).where(
|
|
3296
|
-
models.AccountAttrAssociation.key == f'group-{rse_attr.get(
|
|
4348
|
+
models.AccountAttrAssociation.key == f'group-{rse_attr.get(RseAttr.PHYSGROUP, "")}',
|
|
3297
4349
|
models.AccountAttrAssociation.value == 'admin'
|
|
3298
4350
|
)
|
|
3299
4351
|
|
|
@@ -3332,7 +4384,13 @@ def __progress_class(replicating_locks, total_locks):
|
|
|
3332
4384
|
|
|
3333
4385
|
@policy_filter
|
|
3334
4386
|
@transactional_session
|
|
3335
|
-
def archive_localgroupdisk_datasets(
|
|
4387
|
+
def archive_localgroupdisk_datasets(
|
|
4388
|
+
scope: InternalScope,
|
|
4389
|
+
name: str,
|
|
4390
|
+
*,
|
|
4391
|
+
session: "Session",
|
|
4392
|
+
logger: LoggerFunction = logging.log
|
|
4393
|
+
) -> None:
|
|
3336
4394
|
"""
|
|
3337
4395
|
ATLAS policy to archive a dataset which has a replica on LOCALGROUPDISK
|
|
3338
4396
|
|
|
@@ -3400,7 +4458,13 @@ def archive_localgroupdisk_datasets(scope, name, *, session: "Session", logger=l
|
|
|
3400
4458
|
|
|
3401
4459
|
@policy_filter
|
|
3402
4460
|
@read_session
|
|
3403
|
-
def get_scratch_policy(
|
|
4461
|
+
def get_scratch_policy(
|
|
4462
|
+
account: InternalAccount,
|
|
4463
|
+
rses: Sequence[dict[str, Any]],
|
|
4464
|
+
lifetime: Optional[int],
|
|
4465
|
+
*,
|
|
4466
|
+
session: "Session"
|
|
4467
|
+
) -> Optional[int]:
|
|
3404
4468
|
"""
|
|
3405
4469
|
ATLAS policy for rules on SCRATCHDISK
|
|
3406
4470
|
|
|
@@ -3414,6 +4478,6 @@ def get_scratch_policy(account, rses, lifetime, *, session: "Session"):
|
|
|
3414
4478
|
# Check SCRATCHDISK Policy
|
|
3415
4479
|
if not has_account_attribute(account=account, key='admin', session=session) and (lifetime is None or lifetime > 60 * 60 * 24 * scratchdisk_lifetime):
|
|
3416
4480
|
# Check if one of the rses is a SCRATCHDISK:
|
|
3417
|
-
if [rse for rse in rses if list_rse_attributes(rse_id=rse['id'], session=session).get(
|
|
4481
|
+
if [rse for rse in rses if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.TYPE) == 'SCRATCHDISK']:
|
|
3418
4482
|
lifetime = 60 * 60 * 24 * scratchdisk_lifetime - 1
|
|
3419
4483
|
return lifetime
|