rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/core/lock.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -14,28 +13,30 @@
|
|
|
14
13
|
# limitations under the License.
|
|
15
14
|
|
|
16
15
|
import logging
|
|
16
|
+
from collections.abc import Iterable, Iterator
|
|
17
17
|
from datetime import datetime
|
|
18
|
-
from typing import TYPE_CHECKING
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
19
19
|
|
|
20
20
|
from sqlalchemy.exc import DatabaseError
|
|
21
|
-
from sqlalchemy.sql.expression import and_, or_
|
|
21
|
+
from sqlalchemy.sql.expression import and_, or_, select, true, update
|
|
22
22
|
|
|
23
23
|
import rucio.core.did
|
|
24
24
|
import rucio.core.rule
|
|
25
|
+
from rucio.common.constants import RseAttr
|
|
25
26
|
from rucio.common.exception import DataIdentifierNotFound
|
|
26
|
-
from rucio.common.types import InternalScope
|
|
27
|
+
from rucio.common.types import InternalScope, LoggerFunction
|
|
27
28
|
from rucio.core.lifetime_exception import define_eol
|
|
28
29
|
from rucio.core.rse import get_rse_attribute, get_rse_name
|
|
29
|
-
from rucio.db.sqla import
|
|
30
|
-
from rucio.db.sqla.constants import
|
|
31
|
-
from rucio.db.sqla.session import read_session,
|
|
30
|
+
from rucio.db.sqla import filter_thread_work, models
|
|
31
|
+
from rucio.db.sqla.constants import DIDType, LockState, RuleGrouping, RuleNotification, RuleState
|
|
32
|
+
from rucio.db.sqla.session import read_session, stream_session, transactional_session
|
|
32
33
|
|
|
33
34
|
if TYPE_CHECKING:
|
|
34
35
|
from sqlalchemy.orm import Session
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
@stream_session
|
|
38
|
-
def get_dataset_locks(scope, name, *, session: "Session"):
|
|
39
|
+
def get_dataset_locks(scope: InternalScope, name: str, *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
39
40
|
"""
|
|
40
41
|
Get the dataset locks of a dataset
|
|
41
42
|
|
|
@@ -45,17 +46,22 @@ def get_dataset_locks(scope, name, *, session: "Session"):
|
|
|
45
46
|
:return: List of dicts {'rse_id': ..., 'state': ...}
|
|
46
47
|
"""
|
|
47
48
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
49
|
+
stmt = select(
|
|
50
|
+
models.DatasetLock.rse_id,
|
|
51
|
+
models.DatasetLock.scope,
|
|
52
|
+
models.DatasetLock.name,
|
|
53
|
+
models.DatasetLock.rule_id,
|
|
54
|
+
models.DatasetLock.account,
|
|
55
|
+
models.DatasetLock.state,
|
|
56
|
+
models.DatasetLock.length,
|
|
57
|
+
models.DatasetLock.bytes,
|
|
58
|
+
models.DatasetLock.accessed_at
|
|
59
|
+
).where(
|
|
60
|
+
and_(models.DatasetLock.scope == scope,
|
|
61
|
+
models.DatasetLock.name == name)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
for rse_id, scope, name, rule_id, account, state, length, bytes_, accessed_at in session.execute(stmt).yield_per(500):
|
|
59
65
|
yield {'rse_id': rse_id,
|
|
60
66
|
'rse': get_rse_name(rse_id, session=session),
|
|
61
67
|
'scope': scope,
|
|
@@ -69,7 +75,7 @@ def get_dataset_locks(scope, name, *, session: "Session"):
|
|
|
69
75
|
|
|
70
76
|
|
|
71
77
|
@stream_session
|
|
72
|
-
def get_dataset_locks_bulk(dids, *, session: "Session"):
|
|
78
|
+
def get_dataset_locks_bulk(dids: Iterable[dict[str, Any]], *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
73
79
|
"""
|
|
74
80
|
Get the dataset locks of a list of datasets or containers, recursively
|
|
75
81
|
|
|
@@ -81,7 +87,8 @@ def get_dataset_locks_bulk(dids, *, session: "Session"):
|
|
|
81
87
|
|
|
82
88
|
for did in dids:
|
|
83
89
|
scope = did["scope"]
|
|
84
|
-
|
|
90
|
+
if not isinstance(scope, InternalScope):
|
|
91
|
+
raise ValueError("Scope must be passed as an InternalScope object.")
|
|
85
92
|
name = did["name"]
|
|
86
93
|
did_type = did.get("type")
|
|
87
94
|
if not did_type:
|
|
@@ -90,19 +97,21 @@ def get_dataset_locks_bulk(dids, *, session: "Session"):
|
|
|
90
97
|
except DataIdentifierNotFound:
|
|
91
98
|
continue
|
|
92
99
|
did_type = did_info["type"]
|
|
93
|
-
|
|
100
|
+
|
|
94
101
|
if did_type == DIDType.DATASET:
|
|
95
102
|
for lock_dict in get_dataset_locks(scope, name, session=session):
|
|
96
103
|
yield lock_dict
|
|
97
|
-
|
|
104
|
+
elif did_type == DIDType.CONTAINER:
|
|
98
105
|
for dataset_info in rucio.core.did.list_child_datasets(scope, name, session=session):
|
|
99
106
|
dataset_scope, dataset_name = dataset_info["scope"], dataset_info["name"]
|
|
100
107
|
for lock_dict in get_dataset_locks(dataset_scope, dataset_name, session=session):
|
|
101
108
|
yield lock_dict
|
|
109
|
+
else:
|
|
110
|
+
raise ValueError("Can only get locks for datasets and containers.")
|
|
102
111
|
|
|
103
112
|
|
|
104
113
|
@stream_session
|
|
105
|
-
def get_dataset_locks_by_rse_id(rse_id, *, session: "Session"):
|
|
114
|
+
def get_dataset_locks_by_rse_id(rse_id: str, *, session: "Session") -> Iterator[dict[str, Any]]:
|
|
106
115
|
"""
|
|
107
116
|
Get the dataset locks of an RSE.
|
|
108
117
|
|
|
@@ -110,18 +119,25 @@ def get_dataset_locks_by_rse_id(rse_id, *, session: "Session"):
|
|
|
110
119
|
:param session: The db session.
|
|
111
120
|
:return: List of dicts {'rse_id': ..., 'state': ...}
|
|
112
121
|
"""
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
stmt = select(
|
|
123
|
+
models.DatasetLock.rse_id,
|
|
124
|
+
models.DatasetLock.scope,
|
|
125
|
+
models.DatasetLock.name,
|
|
126
|
+
models.DatasetLock.rule_id,
|
|
127
|
+
models.DatasetLock.account,
|
|
128
|
+
models.DatasetLock.state,
|
|
129
|
+
models.DatasetLock.length,
|
|
130
|
+
models.DatasetLock.bytes,
|
|
131
|
+
models.DatasetLock.accessed_at
|
|
132
|
+
).with_hint(
|
|
133
|
+
models.DatasetLock,
|
|
134
|
+
'INDEX(DATASET_LOCKS DATASET_LOCKS_RSE_ID_IDX)',
|
|
135
|
+
'oracle'
|
|
136
|
+
).where(
|
|
137
|
+
models.DatasetLock.rse_id == rse_id
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
for rse_id, scope, name, rule_id, account, state, length, bytes_, accessed_at in session.execute(stmt).yield_per(500):
|
|
125
141
|
yield {'rse_id': rse_id,
|
|
126
142
|
'rse': get_rse_name(rse_id, session=session),
|
|
127
143
|
'scope': scope,
|
|
@@ -135,7 +151,7 @@ def get_dataset_locks_by_rse_id(rse_id, *, session: "Session"):
|
|
|
135
151
|
|
|
136
152
|
|
|
137
153
|
@read_session
|
|
138
|
-
def get_replica_locks(scope, name, nowait=False, restrict_rses=None, *, session: "Session"):
|
|
154
|
+
def get_replica_locks(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional[Iterable[str]] = None, *, session: "Session") -> list[models.ReplicaLock]:
|
|
139
155
|
"""
|
|
140
156
|
Get the active replica locks for a file
|
|
141
157
|
|
|
@@ -148,19 +164,29 @@ def get_replica_locks(scope, name, nowait=False, restrict_rses=None, *, session:
|
|
|
148
164
|
:raises: NoResultFound
|
|
149
165
|
"""
|
|
150
166
|
|
|
151
|
-
|
|
167
|
+
stmt = select(
|
|
168
|
+
models.ReplicaLock
|
|
169
|
+
).where(
|
|
170
|
+
and_(models.ReplicaLock.scope == scope,
|
|
171
|
+
models.ReplicaLock.name == name)
|
|
172
|
+
)
|
|
152
173
|
if restrict_rses is not None:
|
|
153
174
|
rse_clause = []
|
|
154
175
|
for rse_id in restrict_rses:
|
|
155
176
|
rse_clause.append(models.ReplicaLock.rse_id == rse_id)
|
|
156
177
|
if rse_clause:
|
|
157
|
-
|
|
178
|
+
stmt = stmt.where(
|
|
179
|
+
or_(*rse_clause)
|
|
180
|
+
)
|
|
158
181
|
|
|
159
|
-
|
|
182
|
+
stmt = stmt.with_for_update(
|
|
183
|
+
nowait=nowait
|
|
184
|
+
)
|
|
185
|
+
return list(session.execute(stmt).scalars().all())
|
|
160
186
|
|
|
161
187
|
|
|
162
188
|
@read_session
|
|
163
|
-
def get_replica_locks_for_rule_id(rule_id, *, session: "Session"):
|
|
189
|
+
def get_replica_locks_for_rule_id(rule_id: str, *, session: "Session") -> list[dict[str, Any]]:
|
|
164
190
|
"""
|
|
165
191
|
Get the active replica locks for a rule_id.
|
|
166
192
|
|
|
@@ -172,9 +198,13 @@ def get_replica_locks_for_rule_id(rule_id, *, session: "Session"):
|
|
|
172
198
|
|
|
173
199
|
locks = []
|
|
174
200
|
|
|
175
|
-
|
|
201
|
+
stmt = select(
|
|
202
|
+
models.ReplicaLock
|
|
203
|
+
).where(
|
|
204
|
+
models.ReplicaLock.rule_id == rule_id
|
|
205
|
+
)
|
|
176
206
|
|
|
177
|
-
for row in
|
|
207
|
+
for row in session.execute(stmt).scalars().all():
|
|
178
208
|
locks.append({'scope': row.scope,
|
|
179
209
|
'name': row.name,
|
|
180
210
|
'rse_id': row.rse_id,
|
|
@@ -186,7 +216,7 @@ def get_replica_locks_for_rule_id(rule_id, *, session: "Session"):
|
|
|
186
216
|
|
|
187
217
|
|
|
188
218
|
@read_session
|
|
189
|
-
def get_replica_locks_for_rule_id_per_rse(rule_id, *, session: "Session"):
|
|
219
|
+
def get_replica_locks_for_rule_id_per_rse(rule_id: str, *, session: "Session") -> list[dict[str, str]]:
|
|
190
220
|
"""
|
|
191
221
|
Get the active replica locks for a rule_id per rse.
|
|
192
222
|
|
|
@@ -198,24 +228,30 @@ def get_replica_locks_for_rule_id_per_rse(rule_id, *, session: "Session"):
|
|
|
198
228
|
|
|
199
229
|
locks = []
|
|
200
230
|
|
|
201
|
-
|
|
231
|
+
stmt = select(
|
|
232
|
+
models.ReplicaLock.rse_id
|
|
233
|
+
).where(
|
|
234
|
+
models.ReplicaLock.rule_id == rule_id
|
|
235
|
+
).group_by(
|
|
236
|
+
models.ReplicaLock.rse_id
|
|
237
|
+
)
|
|
202
238
|
|
|
203
|
-
for
|
|
204
|
-
locks.append({'rse_id':
|
|
205
|
-
'rse': get_rse_name(rse_id=
|
|
239
|
+
for rse_id in session.execute(stmt).scalars().all():
|
|
240
|
+
locks.append({'rse_id': rse_id,
|
|
241
|
+
'rse': get_rse_name(rse_id=rse_id, session=session)})
|
|
206
242
|
|
|
207
243
|
return locks
|
|
208
244
|
|
|
209
245
|
|
|
210
246
|
@read_session
|
|
211
|
-
def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_rses=None, only_stuck=False,
|
|
212
|
-
total_threads=None, thread_id=None,
|
|
213
|
-
*, session: "Session"):
|
|
247
|
+
def get_files_and_replica_locks_of_dataset(scope: InternalScope, name: str, nowait: bool = False, restrict_rses: Optional[Iterable[str]] = None, only_stuck: bool = False,
|
|
248
|
+
total_threads: Optional[int] = None, thread_id: Optional[int] = None,
|
|
249
|
+
*, session: "Session") -> dict[tuple[InternalScope, str], Union[models.ReplicaLock, list[models.ReplicaLock]]]:
|
|
214
250
|
"""
|
|
215
251
|
Get all the files of a dataset and, if existing, all locks of the file.
|
|
216
252
|
|
|
217
253
|
:param scope: Scope of the dataset
|
|
218
|
-
:param name: Name of the
|
|
254
|
+
:param name: Name of the dataset
|
|
219
255
|
:param nowait: Nowait parameter for the FOR UPDATE statement
|
|
220
256
|
:param restrict_rses: Possible RSE_ids to filter on.
|
|
221
257
|
:param only_stuck: If true, only get STUCK locks.
|
|
@@ -227,83 +263,60 @@ def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_r
|
|
|
227
263
|
:raises: NoResultFound
|
|
228
264
|
"""
|
|
229
265
|
locks = {}
|
|
230
|
-
if session.bind.dialect.name == 'postgresql':
|
|
231
|
-
content_query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
232
|
-
models.DataIdentifierAssociation.child_name).\
|
|
233
|
-
with_hint(models.DataIdentifierAssociation,
|
|
234
|
-
"INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
|
|
235
|
-
'oracle').\
|
|
236
|
-
filter(models.DataIdentifierAssociation.scope == scope,
|
|
237
|
-
models.DataIdentifierAssociation.name == name)
|
|
238
266
|
|
|
267
|
+
rse_clause = [true()]
|
|
268
|
+
if restrict_rses is not None:
|
|
269
|
+
rse_clause = [models.ReplicaLock.rse_id == rse_id for rse_id in restrict_rses]
|
|
270
|
+
|
|
271
|
+
base_stmt = select(
|
|
272
|
+
models.DataIdentifierAssociation.child_scope,
|
|
273
|
+
models.DataIdentifierAssociation.child_name
|
|
274
|
+
).where(
|
|
275
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
276
|
+
models.DataIdentifierAssociation.name == name)
|
|
277
|
+
)
|
|
278
|
+
stmt = base_stmt.add_columns(
|
|
279
|
+
models.ReplicaLock
|
|
280
|
+
).with_for_update(
|
|
281
|
+
nowait=nowait,
|
|
282
|
+
of=models.ReplicaLock.state
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
if session.bind.dialect.name == 'postgresql':
|
|
239
286
|
if total_threads and total_threads > 1:
|
|
240
|
-
|
|
241
|
-
|
|
287
|
+
base_stmt = filter_thread_work(session=session, query=base_stmt, total_threads=total_threads,
|
|
288
|
+
thread_id=thread_id, hash_variable='child_name')
|
|
242
289
|
|
|
243
|
-
for child_scope, child_name in
|
|
290
|
+
for child_scope, child_name in session.execute(base_stmt).yield_per(1000):
|
|
244
291
|
locks[(child_scope, child_name)] = []
|
|
245
292
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
'oracle').\
|
|
252
|
-
filter(and_(models.DataIdentifierAssociation.child_scope == models.ReplicaLock.scope,
|
|
253
|
-
models.DataIdentifierAssociation.child_name == models.ReplicaLock.name))\
|
|
254
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
255
|
-
models.DataIdentifierAssociation.name == name)
|
|
256
|
-
|
|
257
|
-
if restrict_rses is not None:
|
|
258
|
-
rse_clause = []
|
|
259
|
-
for rse_id in restrict_rses:
|
|
260
|
-
rse_clause.append(models.ReplicaLock.rse_id == rse_id)
|
|
261
|
-
if rse_clause:
|
|
262
|
-
query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
263
|
-
models.DataIdentifierAssociation.child_name,
|
|
264
|
-
models.ReplicaLock).\
|
|
265
|
-
with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle').\
|
|
266
|
-
filter(and_(models.DataIdentifierAssociation.child_scope == models.ReplicaLock.scope,
|
|
267
|
-
models.DataIdentifierAssociation.child_name == models.ReplicaLock.name,
|
|
268
|
-
or_(*rse_clause)))\
|
|
269
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
270
|
-
models.DataIdentifierAssociation.name == name)
|
|
293
|
+
stmt = stmt.where(
|
|
294
|
+
and_(models.DataIdentifierAssociation.child_scope == models.ReplicaLock.scope,
|
|
295
|
+
models.DataIdentifierAssociation.child_name == models.ReplicaLock.name,
|
|
296
|
+
or_(*rse_clause))
|
|
297
|
+
)
|
|
271
298
|
else:
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
rse_clause = []
|
|
283
|
-
for rse_id in restrict_rses:
|
|
284
|
-
rse_clause.append(models.ReplicaLock.rse_id == rse_id)
|
|
285
|
-
if rse_clause:
|
|
286
|
-
query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
287
|
-
models.DataIdentifierAssociation.child_name,
|
|
288
|
-
models.ReplicaLock).\
|
|
289
|
-
with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle').\
|
|
290
|
-
outerjoin(models.ReplicaLock,
|
|
291
|
-
and_(models.DataIdentifierAssociation.child_scope == models.ReplicaLock.scope,
|
|
292
|
-
models.DataIdentifierAssociation.child_name == models.ReplicaLock.name,
|
|
293
|
-
or_(*rse_clause)))\
|
|
294
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
295
|
-
models.DataIdentifierAssociation.name == name)
|
|
299
|
+
stmt = stmt.with_hint(
|
|
300
|
+
models.DataIdentifierAssociation,
|
|
301
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
302
|
+
'oracle'
|
|
303
|
+
).outerjoin(
|
|
304
|
+
models.ReplicaLock,
|
|
305
|
+
and_(models.DataIdentifierAssociation.child_scope == models.ReplicaLock.scope,
|
|
306
|
+
models.DataIdentifierAssociation.child_name == models.ReplicaLock.name,
|
|
307
|
+
or_(*rse_clause))
|
|
308
|
+
)
|
|
296
309
|
|
|
297
310
|
if only_stuck:
|
|
298
|
-
|
|
311
|
+
stmt = stmt.where(
|
|
312
|
+
models.ReplicaLock.state == LockState.STUCK
|
|
313
|
+
)
|
|
299
314
|
|
|
300
315
|
if total_threads and total_threads > 1:
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
query = query.with_for_update(nowait=nowait, of=models.ReplicaLock.state)
|
|
316
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_threads,
|
|
317
|
+
thread_id=thread_id, hash_variable='child_name')
|
|
305
318
|
|
|
306
|
-
for child_scope, child_name, lock in
|
|
319
|
+
for child_scope, child_name, lock in session.execute(stmt).all():
|
|
307
320
|
if (child_scope, child_name) not in locks:
|
|
308
321
|
if lock is None:
|
|
309
322
|
locks[(child_scope, child_name)] = []
|
|
@@ -316,7 +329,7 @@ def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_r
|
|
|
316
329
|
|
|
317
330
|
|
|
318
331
|
@transactional_session
|
|
319
|
-
def successful_transfer(scope, name, rse_id, nowait, *, session: "Session", logger=logging.log):
|
|
332
|
+
def successful_transfer(scope: InternalScope, name: str, rse_id: str, nowait: bool, *, session: "Session", logger: LoggerFunction = logging.log) -> None:
|
|
320
333
|
"""
|
|
321
334
|
Update the state of all replica locks because of an successful transfer
|
|
322
335
|
|
|
@@ -327,13 +340,28 @@ def successful_transfer(scope, name, rse_id, nowait, *, session: "Session", logg
|
|
|
327
340
|
:param session: DB Session.
|
|
328
341
|
"""
|
|
329
342
|
|
|
330
|
-
|
|
331
|
-
|
|
343
|
+
stmt = select(
|
|
344
|
+
models.ReplicaLock
|
|
345
|
+
).where(
|
|
346
|
+
and_(models.ReplicaLock.scope == scope,
|
|
347
|
+
models.ReplicaLock.name == name,
|
|
348
|
+
models.ReplicaLock.rse_id == rse_id)
|
|
349
|
+
).with_for_update(
|
|
350
|
+
nowait=nowait
|
|
351
|
+
)
|
|
352
|
+
for lock in session.execute(stmt).scalars().all():
|
|
332
353
|
if lock.state == LockState.OK:
|
|
333
354
|
continue
|
|
334
355
|
logger(logging.DEBUG, 'Marking lock %s:%s for rule %s on rse %s as OK' % (lock.scope, lock.name, str(lock.rule_id), get_rse_name(rse_id=lock.rse_id, session=session)))
|
|
335
356
|
# Update the rule counters
|
|
336
|
-
|
|
357
|
+
stmt = select(
|
|
358
|
+
models.ReplicationRule
|
|
359
|
+
).where(
|
|
360
|
+
models.ReplicationRule.id == lock.rule_id
|
|
361
|
+
).with_for_update(
|
|
362
|
+
nowait=nowait
|
|
363
|
+
)
|
|
364
|
+
rule = session.execute(stmt).scalar_one()
|
|
337
365
|
logger(logging.DEBUG, 'Updating rule counters for rule %s [%d/%d/%d]' % (str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt))
|
|
338
366
|
|
|
339
367
|
if lock.state == LockState.REPLICATING:
|
|
@@ -367,8 +395,14 @@ def successful_transfer(scope, name, rse_id, nowait, *, session: "Session", logg
|
|
|
367
395
|
rule.state = RuleState.OK
|
|
368
396
|
# Try to update the DatasetLocks
|
|
369
397
|
if rule.grouping != RuleGrouping.NONE:
|
|
370
|
-
|
|
371
|
-
|
|
398
|
+
stmt = select(
|
|
399
|
+
models.DatasetLock
|
|
400
|
+
).where(
|
|
401
|
+
models.DatasetLock.rule_id == rule.id
|
|
402
|
+
).with_for_update(
|
|
403
|
+
nowait=nowait
|
|
404
|
+
)
|
|
405
|
+
for ds_lock in session.execute(stmt).scalars().all():
|
|
372
406
|
ds_lock.state = LockState.OK
|
|
373
407
|
session.flush()
|
|
374
408
|
rucio.core.rule.generate_rule_notifications(rule=rule, replicating_locks_before=rule.locks_replicating_cnt + 1, session=session)
|
|
@@ -385,7 +419,8 @@ def successful_transfer(scope, name, rse_id, nowait, *, session: "Session", logg
|
|
|
385
419
|
|
|
386
420
|
|
|
387
421
|
@transactional_session
|
|
388
|
-
def failed_transfer(scope, name, rse_id, error_message=None, broken_rule_id
|
|
422
|
+
def failed_transfer(scope: InternalScope, name: str, rse_id: str, error_message: Optional[str] = None, broken_rule_id: Optional[str] = None,
|
|
423
|
+
broken_message: Optional[str] = None, nowait: bool = True, *, session: "Session", logger: LoggerFunction = logging.log) -> None:
|
|
389
424
|
"""
|
|
390
425
|
Update the state of all replica locks because of a failed transfer.
|
|
391
426
|
If a transfer is permanently broken for a rule, the broken_rule_id should be filled which puts this rule into the SUSPENDED state.
|
|
@@ -400,20 +435,44 @@ def failed_transfer(scope, name, rse_id, error_message=None, broken_rule_id=None
|
|
|
400
435
|
:param session: The database session in use.
|
|
401
436
|
"""
|
|
402
437
|
|
|
403
|
-
staging_required = get_rse_attribute(rse_id,
|
|
438
|
+
staging_required = get_rse_attribute(rse_id, RseAttr.STAGING_REQUIRED, session=session)
|
|
404
439
|
if staging_required:
|
|
405
440
|
rse_name = get_rse_name(rse_id=rse_id, session=session)
|
|
406
441
|
logger(logging.DEBUG, f'Destination RSE {rse_name} is type staging_required so do not update other OK replica locks.')
|
|
407
|
-
|
|
442
|
+
stmt = select(
|
|
443
|
+
models.ReplicaLock
|
|
444
|
+
).where(
|
|
445
|
+
and_(models.ReplicaLock.scope == scope,
|
|
446
|
+
models.ReplicaLock.name == name,
|
|
447
|
+
models.ReplicaLock.rse_id == rse_id,
|
|
448
|
+
models.ReplicaLock.state == LockState.REPLICATING)
|
|
449
|
+
).with_for_update(
|
|
450
|
+
nowait=nowait
|
|
451
|
+
)
|
|
408
452
|
else:
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
453
|
+
stmt = select(
|
|
454
|
+
models.ReplicaLock
|
|
455
|
+
).where(
|
|
456
|
+
and_(models.ReplicaLock.scope == scope,
|
|
457
|
+
models.ReplicaLock.name == name,
|
|
458
|
+
models.ReplicaLock.rse_id == rse_id)
|
|
459
|
+
).with_for_update(
|
|
460
|
+
nowait=nowait
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
for lock in session.execute(stmt).scalars().all():
|
|
412
464
|
if lock.state == LockState.STUCK:
|
|
413
465
|
continue
|
|
414
466
|
logger(logging.DEBUG, 'Marking lock %s:%s for rule %s on rse %s as STUCK' % (lock.scope, lock.name, str(lock.rule_id), get_rse_name(rse_id=lock.rse_id, session=session)))
|
|
415
467
|
# Update the rule counters
|
|
416
|
-
|
|
468
|
+
stmt = select(
|
|
469
|
+
models.ReplicationRule
|
|
470
|
+
).where(
|
|
471
|
+
models.ReplicationRule.id == lock.rule_id
|
|
472
|
+
).with_for_update(
|
|
473
|
+
nowait=nowait
|
|
474
|
+
)
|
|
475
|
+
rule = session.execute(stmt).scalar_one()
|
|
417
476
|
logger(logging.DEBUG, 'Updating rule counters for rule %s [%d/%d/%d]' % (str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt))
|
|
418
477
|
if lock.state == LockState.REPLICATING:
|
|
419
478
|
rule.locks_replicating_cnt -= 1
|
|
@@ -428,29 +487,47 @@ def failed_transfer(scope, name, rse_id, error_message=None, broken_rule_id=None
|
|
|
428
487
|
pass
|
|
429
488
|
elif lock.rule_id == broken_rule_id:
|
|
430
489
|
rule.state = RuleState.SUSPENDED
|
|
431
|
-
|
|
490
|
+
if broken_message is not None and len(broken_message) > 245:
|
|
491
|
+
rule.error = (broken_message[:245] + '...')
|
|
492
|
+
else:
|
|
493
|
+
rule.error = broken_message
|
|
432
494
|
# Try to update the DatasetLocks
|
|
433
495
|
if rule.grouping != RuleGrouping.NONE:
|
|
434
|
-
|
|
435
|
-
|
|
496
|
+
stmt = select(
|
|
497
|
+
models.DatasetLock
|
|
498
|
+
).where(
|
|
499
|
+
models.DatasetLock.rule_id == rule.id
|
|
500
|
+
).with_for_update(
|
|
501
|
+
nowait=nowait
|
|
502
|
+
)
|
|
503
|
+
for ds_lock in session.execute(stmt).scalars().all():
|
|
436
504
|
ds_lock.state = LockState.STUCK
|
|
437
505
|
elif rule.locks_stuck_cnt > 0:
|
|
438
506
|
if rule.state != RuleState.STUCK:
|
|
439
507
|
rule.state = RuleState.STUCK
|
|
440
508
|
# Try to update the DatasetLocks
|
|
441
509
|
if rule.grouping != RuleGrouping.NONE:
|
|
442
|
-
|
|
443
|
-
|
|
510
|
+
stmt = select(
|
|
511
|
+
models.DatasetLock
|
|
512
|
+
).where(
|
|
513
|
+
models.DatasetLock.rule_id == rule.id
|
|
514
|
+
).with_for_update(
|
|
515
|
+
nowait=nowait
|
|
516
|
+
)
|
|
517
|
+
for ds_lock in session.execute(stmt).scalars().all():
|
|
444
518
|
ds_lock.state = LockState.STUCK
|
|
445
519
|
if rule.error != error_message:
|
|
446
|
-
|
|
520
|
+
if error_message is not None and len(error_message) > 245:
|
|
521
|
+
rule.error = (error_message[:245] + '...')
|
|
522
|
+
else:
|
|
523
|
+
rule.error = error_message
|
|
447
524
|
|
|
448
525
|
# Insert rule history
|
|
449
526
|
rucio.core.rule.insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
|
|
450
527
|
|
|
451
528
|
|
|
452
529
|
@transactional_session
|
|
453
|
-
def touch_dataset_locks(dataset_locks, *, session: "Session"):
|
|
530
|
+
def touch_dataset_locks(dataset_locks: Iterable[dict[str, Any]], *, session: "Session") -> bool:
|
|
454
531
|
"""
|
|
455
532
|
Update the accessed_at timestamp of the given dataset locks + eol_at.
|
|
456
533
|
|
|
@@ -464,10 +541,35 @@ def touch_dataset_locks(dataset_locks, *, session: "Session"):
|
|
|
464
541
|
for dataset_lock in dataset_locks:
|
|
465
542
|
eol_at = define_eol(dataset_lock['scope'], dataset_lock['name'], rses=[{'id': dataset_lock['rse_id']}], session=session)
|
|
466
543
|
try:
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
544
|
+
stmt = update(
|
|
545
|
+
models.DatasetLock
|
|
546
|
+
).where(
|
|
547
|
+
and_(models.DatasetLock.scope == dataset_lock['scope'],
|
|
548
|
+
models.DatasetLock.name == dataset_lock['name'],
|
|
549
|
+
models.DatasetLock.rse_id == dataset_lock['rse_id'])
|
|
550
|
+
).values({
|
|
551
|
+
models.DatasetLock.accessed_at: dataset_lock.get('accessed_at') or now
|
|
552
|
+
}).execution_options(
|
|
553
|
+
synchronize_session=False
|
|
554
|
+
)
|
|
555
|
+
subq = select(
|
|
556
|
+
models.DatasetLock.rule_id
|
|
557
|
+
).where(
|
|
558
|
+
and_(models.DatasetLock.scope == dataset_lock['scope'],
|
|
559
|
+
models.DatasetLock.name == dataset_lock['name'],
|
|
560
|
+
models.DatasetLock.rse_id == dataset_lock['rse_id'])
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
stmt = update(
|
|
564
|
+
models.ReplicationRule
|
|
565
|
+
).where(
|
|
566
|
+
models.ReplicationRule.id.in_(subq)
|
|
567
|
+
).values({
|
|
568
|
+
models.ReplicationRule.eol_at: eol_at
|
|
569
|
+
}).execution_options(
|
|
570
|
+
synchronize_session=False
|
|
571
|
+
)
|
|
572
|
+
session.execute(stmt)
|
|
471
573
|
except DatabaseError:
|
|
472
574
|
return False
|
|
473
575
|
|