rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/core/replica.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -12,9 +11,11 @@
|
|
|
12
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
12
|
# See the License for the specific language governing permissions and
|
|
14
13
|
# limitations under the License.
|
|
14
|
+
|
|
15
15
|
import copy
|
|
16
16
|
import heapq
|
|
17
17
|
import logging
|
|
18
|
+
import math
|
|
18
19
|
import random
|
|
19
20
|
from collections import defaultdict, namedtuple
|
|
20
21
|
from curses.ascii import isprint
|
|
@@ -27,44 +28,43 @@ from struct import unpack
|
|
|
27
28
|
from traceback import format_exc
|
|
28
29
|
from typing import TYPE_CHECKING
|
|
29
30
|
|
|
30
|
-
import math
|
|
31
31
|
import requests
|
|
32
32
|
from dogpile.cache.api import NO_VALUE
|
|
33
|
-
from sqlalchemy import
|
|
33
|
+
from sqlalchemy import and_, delete, exists, func, insert, not_, or_, union, update
|
|
34
34
|
from sqlalchemy.exc import DatabaseError, IntegrityError
|
|
35
35
|
from sqlalchemy.orm import aliased
|
|
36
36
|
from sqlalchemy.orm.exc import FlushError, NoResultFound
|
|
37
|
-
from sqlalchemy.sql import
|
|
38
|
-
from sqlalchemy.sql.expression import case, select, text, false, true, null, literal, literal_column
|
|
37
|
+
from sqlalchemy.sql.expression import case, false, literal, literal_column, null, select, text, true
|
|
39
38
|
|
|
40
39
|
import rucio.core.did
|
|
41
40
|
import rucio.core.lock
|
|
42
41
|
from rucio.common import exception
|
|
43
42
|
from rucio.common.cache import make_region_memcached
|
|
44
43
|
from rucio.common.config import config_get, config_get_bool
|
|
45
|
-
from rucio.common.constants import SuspiciousAvailability
|
|
44
|
+
from rucio.common.constants import RseAttr, SuspiciousAvailability
|
|
46
45
|
from rucio.common.types import InternalScope
|
|
47
|
-
from rucio.common.utils import chunks,
|
|
46
|
+
from rucio.common.utils import add_url_query, chunks, clean_pfns, str_to_date
|
|
48
47
|
from rucio.core.credential import get_signed_url
|
|
49
48
|
from rucio.core.message import add_messages
|
|
50
49
|
from rucio.core.monitor import MetricManager
|
|
51
|
-
from rucio.core.rse import get_rse,
|
|
50
|
+
from rucio.core.rse import get_rse, get_rse_attribute, get_rse_name, get_rse_vo, list_rses
|
|
52
51
|
from rucio.core.rse_counter import decrease, increase
|
|
53
52
|
from rucio.core.rse_expression_parser import parse_expression
|
|
54
|
-
from rucio.db.sqla import
|
|
55
|
-
from rucio.db.sqla.constants import
|
|
56
|
-
|
|
57
|
-
from rucio.db.sqla.session import (read_session, stream_session, transactional_session,
|
|
58
|
-
DEFAULT_SCHEMA_NAME, BASE)
|
|
53
|
+
from rucio.db.sqla import filter_thread_work, models
|
|
54
|
+
from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, BadPFNStatus, DIDAvailability, DIDType, ReplicaState, RuleState
|
|
55
|
+
from rucio.db.sqla.session import BASE, DEFAULT_SCHEMA_NAME, read_session, stream_session, transactional_session
|
|
59
56
|
from rucio.db.sqla.util import temp_table_mngr
|
|
60
57
|
from rucio.rse import rsemanager as rsemgr
|
|
61
58
|
|
|
62
59
|
if TYPE_CHECKING:
|
|
63
|
-
from collections.abc import Sequence
|
|
64
|
-
from rucio.rse.protocols.protocol import RSEProtocol
|
|
60
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
65
61
|
from typing import Any, Optional
|
|
62
|
+
|
|
66
63
|
from sqlalchemy.orm import Session
|
|
67
64
|
|
|
65
|
+
from rucio.common.types import LoggerFunction
|
|
66
|
+
from rucio.rse.protocols.protocol import RSEProtocol
|
|
67
|
+
|
|
68
68
|
REGION = make_region_memcached(expiration_time=60)
|
|
69
69
|
METRICS = MetricManager(module=__name__)
|
|
70
70
|
|
|
@@ -88,33 +88,42 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
|
|
|
88
88
|
rse_clause = []
|
|
89
89
|
if rse_expression:
|
|
90
90
|
for rse in parse_expression(expression=rse_expression, filter_=filter_, session=session):
|
|
91
|
-
rse_clause.append(models.
|
|
91
|
+
rse_clause.append(models.BadReplica.rse_id == rse['id'])
|
|
92
92
|
elif filter_:
|
|
93
93
|
# Ensure we limit results to current VO even if we don't specify an RSE expression
|
|
94
94
|
for rse in list_rses(filters=filter_, session=session):
|
|
95
|
-
rse_clause.append(models.
|
|
95
|
+
rse_clause.append(models.BadReplica.rse_id == rse['id'])
|
|
96
96
|
|
|
97
97
|
if session.bind.dialect.name == 'oracle':
|
|
98
|
-
to_days = func.trunc(models.
|
|
98
|
+
to_days = func.trunc(models.BadReplica.created_at, 'DD')
|
|
99
99
|
elif session.bind.dialect.name == 'mysql':
|
|
100
|
-
to_days = func.date(models.
|
|
100
|
+
to_days = func.date(models.BadReplica.created_at)
|
|
101
101
|
elif session.bind.dialect.name == 'postgresql':
|
|
102
|
-
to_days = func.date_trunc('day', models.
|
|
102
|
+
to_days = func.date_trunc('day', models.BadReplica.created_at)
|
|
103
103
|
else:
|
|
104
|
-
to_days = func.strftime(models.
|
|
105
|
-
|
|
104
|
+
to_days = func.strftime(models.BadReplica.created_at, '%Y-%m-%d')
|
|
105
|
+
|
|
106
|
+
stmt = select(
|
|
107
|
+
func.count(),
|
|
108
|
+
to_days,
|
|
109
|
+
models.BadReplica.rse_id,
|
|
110
|
+
models.BadReplica.state,
|
|
111
|
+
models.BadReplica.reason
|
|
112
|
+
).select_from(
|
|
113
|
+
models.BadReplica
|
|
114
|
+
)
|
|
106
115
|
# To be added : HINTS
|
|
107
116
|
if rse_clause != []:
|
|
108
|
-
|
|
117
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
109
118
|
if from_date:
|
|
110
|
-
|
|
119
|
+
stmt = stmt.where(models.BadReplica.created_at > from_date)
|
|
111
120
|
if to_date:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
for
|
|
115
|
-
if (
|
|
116
|
-
incidents[(
|
|
117
|
-
incidents[(
|
|
121
|
+
stmt = stmt.where(models.BadReplica.created_at < to_date)
|
|
122
|
+
stmt = stmt.group_by(to_days, models.BadReplica.rse_id, models.BadReplica.reason, models.BadReplica.state)
|
|
123
|
+
for count, to_days, rse_id, state, reason in session.execute(stmt):
|
|
124
|
+
if (rse_id, to_days, reason) not in incidents:
|
|
125
|
+
incidents[(rse_id, to_days, reason)] = {}
|
|
126
|
+
incidents[(rse_id, to_days, reason)][str(state.name)] = count
|
|
118
127
|
|
|
119
128
|
for incident in incidents:
|
|
120
129
|
res = incidents[incident]
|
|
@@ -160,27 +169,38 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
|
|
|
160
169
|
for clause in [path_clause, did_clause]:
|
|
161
170
|
if clause:
|
|
162
171
|
for chunk in chunks(clause, 10):
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
172
|
+
stmt = select(
|
|
173
|
+
models.RSEFileAssociation.path,
|
|
174
|
+
models.RSEFileAssociation.scope,
|
|
175
|
+
models.RSEFileAssociation.name,
|
|
176
|
+
models.RSEFileAssociation.rse_id,
|
|
177
|
+
models.RSEFileAssociation.bytes,
|
|
178
|
+
func.max(
|
|
179
|
+
case(
|
|
180
|
+
(models.BadReplica.state == BadFilesStatus.SUSPICIOUS, 0),
|
|
181
|
+
(models.BadReplica.state == BadFilesStatus.BAD, 1),
|
|
182
|
+
else_=0))
|
|
183
|
+
).with_hint(
|
|
184
|
+
models.RSEFileAssociation,
|
|
185
|
+
'INDEX(REPLICAS REPLICAS_PATH_IDX',
|
|
186
|
+
'oracle'
|
|
187
|
+
).outerjoin(
|
|
188
|
+
models.BadReplica,
|
|
189
|
+
and_(models.RSEFileAssociation.scope == models.BadReplica.scope,
|
|
190
|
+
models.RSEFileAssociation.name == models.BadReplica.name,
|
|
191
|
+
models.RSEFileAssociation.rse_id == models.BadReplica.rse_id)
|
|
192
|
+
).where(
|
|
193
|
+
and_(models.RSEFileAssociation.rse_id == rse_id,
|
|
194
|
+
or_(*chunk))
|
|
195
|
+
).group_by(
|
|
196
|
+
models.RSEFileAssociation.path,
|
|
197
|
+
models.RSEFileAssociation.scope,
|
|
198
|
+
models.RSEFileAssociation.name,
|
|
199
|
+
models.RSEFileAssociation.rse_id,
|
|
200
|
+
models.RSEFileAssociation.bytes
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
for path, scope, name, rse_id, size, state in session.execute(stmt).all():
|
|
184
204
|
if (scope, name, path) in replicas:
|
|
185
205
|
replicas.remove((scope, name, path))
|
|
186
206
|
if (None, None, path) in replicas:
|
|
@@ -211,19 +231,26 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
|
|
|
211
231
|
:param session: The database session in use.
|
|
212
232
|
"""
|
|
213
233
|
result = []
|
|
214
|
-
|
|
234
|
+
stmt = select(
|
|
235
|
+
models.BadReplica.scope,
|
|
236
|
+
models.BadReplica.name,
|
|
237
|
+
models.BadReplica.rse_id,
|
|
238
|
+
models.BadReplica.state,
|
|
239
|
+
models.BadReplica.created_at,
|
|
240
|
+
models.BadReplica.updated_at
|
|
241
|
+
)
|
|
215
242
|
if state:
|
|
216
|
-
|
|
243
|
+
stmt = stmt.where(models.BadReplica.state == state)
|
|
217
244
|
if rse_id:
|
|
218
|
-
|
|
245
|
+
stmt = stmt.where(models.BadReplica.rse_id == rse_id)
|
|
219
246
|
if younger_than:
|
|
220
|
-
|
|
247
|
+
stmt = stmt.where(models.BadReplica.created_at >= younger_than)
|
|
221
248
|
if older_than:
|
|
222
|
-
|
|
249
|
+
stmt = stmt.where(models.BadReplica.created_at <= older_than)
|
|
223
250
|
if limit:
|
|
224
|
-
|
|
251
|
+
stmt = stmt.limit(limit)
|
|
225
252
|
|
|
226
|
-
for badfile in
|
|
253
|
+
for badfile in session.execute(stmt).yield_per(1000):
|
|
227
254
|
if badfile.scope.vo == vo:
|
|
228
255
|
if list_pfns:
|
|
229
256
|
result.append({'scope': badfile.scope, 'name': badfile.name, 'type': DIDType.FILE})
|
|
@@ -245,7 +272,7 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
|
|
|
245
272
|
|
|
246
273
|
|
|
247
274
|
@transactional_session
|
|
248
|
-
def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesStatus.BAD, scheme='srm', force=False, *, session: "Session"):
|
|
275
|
+
def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesStatus.BAD, scheme='srm', force=False, logger: "LoggerFunction" = logging.log, *, session: "Session"):
|
|
249
276
|
"""
|
|
250
277
|
Declare a list of bad replicas.
|
|
251
278
|
|
|
@@ -267,25 +294,17 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
267
294
|
rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
|
|
268
295
|
proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
|
|
269
296
|
if rse_info['deterministic']:
|
|
270
|
-
|
|
297
|
+
scope_proto = rsemgr.get_scope_protocol(vo=issuer.vo)
|
|
271
298
|
parsed_pfn = proto.parse_pfns(pfns=pfns)
|
|
272
299
|
for pfn in parsed_pfn:
|
|
273
|
-
#
|
|
274
|
-
|
|
275
|
-
if path.startswith('/user') or path.startswith('/group'):
|
|
276
|
-
scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
|
|
277
|
-
name = parsed_pfn[pfn]['name']
|
|
278
|
-
elif path.startswith('/'):
|
|
279
|
-
scope = path.split('/')[1]
|
|
280
|
-
name = parsed_pfn[pfn]['name']
|
|
281
|
-
else:
|
|
282
|
-
scope = path.split('/')[0]
|
|
283
|
-
name = parsed_pfn[pfn]['name']
|
|
300
|
+
# Translate into a scope and name
|
|
301
|
+
name, scope = scope_proto(parsed_pfn[pfn])
|
|
284
302
|
|
|
285
303
|
scope = InternalScope(scope, vo=issuer.vo)
|
|
286
304
|
replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': status})
|
|
287
305
|
path = '%s%s' % (parsed_pfn[pfn]['path'], parsed_pfn[pfn]['name'])
|
|
288
306
|
path_pfn_dict[path] = pfn
|
|
307
|
+
logger(logging.DEBUG, f"Declaring replica {scope}:{name} {status} at {rse_id} with path {path}")
|
|
289
308
|
|
|
290
309
|
else:
|
|
291
310
|
# For non-deterministic RSEs use the path + rse_id to extract the scope
|
|
@@ -295,10 +314,13 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
295
314
|
replicas.append({'scope': None, 'name': None, 'rse_id': rse_id, 'path': path, 'state': status})
|
|
296
315
|
path_pfn_dict[path] = pfn
|
|
297
316
|
|
|
317
|
+
logger(logging.DEBUG, f"Declaring replica with pfn: {pfn} {status} at {rse_id} with path {path}")
|
|
318
|
+
|
|
298
319
|
else:
|
|
299
320
|
# If pfns is a list of replicas, just use scope, name and rse_id
|
|
300
321
|
for pfn in pfns:
|
|
301
322
|
replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status})
|
|
323
|
+
logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path")
|
|
302
324
|
|
|
303
325
|
replicas_list = []
|
|
304
326
|
for replica in replicas:
|
|
@@ -308,7 +330,6 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
308
330
|
bad_replicas_to_update = []
|
|
309
331
|
|
|
310
332
|
for scope, name, path, __exists, already_declared, size in __exist_replicas(rse_id=rse_id, replicas=replicas_list, session=session):
|
|
311
|
-
|
|
312
333
|
declared = False
|
|
313
334
|
|
|
314
335
|
if __exists:
|
|
@@ -318,7 +339,7 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
318
339
|
declared = True
|
|
319
340
|
|
|
320
341
|
if status == BadFilesStatus.SUSPICIOUS or status == BadFilesStatus.BAD and not already_declared:
|
|
321
|
-
new_bad_replica = models.
|
|
342
|
+
new_bad_replica = models.BadReplica(scope=scope, name=name, rse_id=rse_id, reason=reason, state=status, account=issuer, bytes=size)
|
|
322
343
|
new_bad_replica.save(session=session, flush=False)
|
|
323
344
|
declared = True
|
|
324
345
|
|
|
@@ -333,7 +354,11 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
|
|
|
333
354
|
no_hidden_char = False
|
|
334
355
|
break
|
|
335
356
|
if no_hidden_char:
|
|
336
|
-
|
|
357
|
+
pfn = path_pfn_dict[path]
|
|
358
|
+
if f"{pfn} Unknown replica" not in unknown_replicas:
|
|
359
|
+
unknown_replicas.append('%s %s' % (pfn, 'Unknown replica'))
|
|
360
|
+
elif scope or name:
|
|
361
|
+
unknown_replicas.append(f"{(scope,name)} Unknown replica")
|
|
337
362
|
|
|
338
363
|
if status == BadFilesStatus.BAD:
|
|
339
364
|
# For BAD file, we modify the replica state, not for suspicious
|
|
@@ -379,11 +404,19 @@ def add_bad_dids(dids, rse_id, reason, issuer, state=BadFilesStatus.BAD, *, sess
|
|
|
379
404
|
for scope, name, _, __exists, already_declared, size in __exist_replicas(rse_id=rse_id, replicas=replicas_list, session=session):
|
|
380
405
|
if __exists and not already_declared:
|
|
381
406
|
replicas_for_update.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.BAD})
|
|
382
|
-
new_bad_replica = models.
|
|
383
|
-
|
|
407
|
+
new_bad_replica = models.BadReplica(scope=scope, name=name, rse_id=rse_id, reason=reason, state=state,
|
|
408
|
+
account=issuer, bytes=size)
|
|
384
409
|
new_bad_replica.save(session=session, flush=False)
|
|
385
|
-
|
|
386
|
-
|
|
410
|
+
stmt = delete(
|
|
411
|
+
models.Source
|
|
412
|
+
).where(
|
|
413
|
+
and_(models.Source.scope == scope,
|
|
414
|
+
models.Source.name == name,
|
|
415
|
+
models.Source.rse_id == rse_id)
|
|
416
|
+
).execution_options(
|
|
417
|
+
synchronize_session=False
|
|
418
|
+
)
|
|
419
|
+
session.execute(stmt)
|
|
387
420
|
else:
|
|
388
421
|
if already_declared:
|
|
389
422
|
unknown_replicas.append('%s:%s %s' % (did['scope'], name, 'Already declared'))
|
|
@@ -456,58 +489,65 @@ def get_pfn_to_rse(pfns, vo='def', *, session: "Session"):
|
|
|
456
489
|
storage_elements = []
|
|
457
490
|
se_condition = []
|
|
458
491
|
dict_rse = {}
|
|
459
|
-
|
|
460
|
-
scheme =
|
|
461
|
-
for
|
|
462
|
-
if
|
|
492
|
+
cleaned_pfns = clean_pfns(pfns)
|
|
493
|
+
scheme = cleaned_pfns[0].split(':')[0] if cleaned_pfns else None
|
|
494
|
+
for pfn in cleaned_pfns:
|
|
495
|
+
if pfn.split(':')[0] != scheme:
|
|
463
496
|
raise exception.InvalidType('The PFNs specified must have the same protocol')
|
|
464
497
|
|
|
465
|
-
split_se =
|
|
498
|
+
split_se = pfn.split('/')[2].split(':')
|
|
466
499
|
storage_element = split_se[0]
|
|
467
500
|
|
|
468
501
|
if storage_element not in storage_elements:
|
|
469
502
|
storage_elements.append(storage_element)
|
|
470
|
-
se_condition.append(models.
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
503
|
+
se_condition.append(models.RSEProtocol.hostname == storage_element)
|
|
504
|
+
stmt = select(
|
|
505
|
+
models.RSEProtocol.rse_id,
|
|
506
|
+
models.RSEProtocol.scheme,
|
|
507
|
+
models.RSEProtocol.hostname,
|
|
508
|
+
models.RSEProtocol.port,
|
|
509
|
+
models.RSEProtocol.prefix
|
|
510
|
+
).join(
|
|
511
|
+
models.RSE,
|
|
512
|
+
models.RSEProtocol.rse_id == models.RSE.id
|
|
513
|
+
).where(
|
|
514
|
+
and_(or_(*se_condition),
|
|
515
|
+
models.RSEProtocol.scheme == scheme,
|
|
516
|
+
models.RSE.deleted == false(),
|
|
517
|
+
models.RSE.staging_area == false())
|
|
518
|
+
)
|
|
519
|
+
|
|
480
520
|
protocols = {}
|
|
481
521
|
|
|
482
|
-
for rse_id, protocol, hostname, port, prefix in
|
|
522
|
+
for rse_id, protocol, hostname, port, prefix in session.execute(stmt).yield_per(10000):
|
|
483
523
|
if rse_id not in protocols:
|
|
484
524
|
protocols[rse_id] = []
|
|
485
525
|
protocols[rse_id].append('%s://%s:%s%s' % (protocol, hostname, port, prefix))
|
|
486
526
|
if '%s://%s%s' % (protocol, hostname, prefix) not in protocols[rse_id]:
|
|
487
527
|
protocols[rse_id].append('%s://%s%s' % (protocol, hostname, prefix))
|
|
488
528
|
hint = None
|
|
489
|
-
for
|
|
529
|
+
for pfn in cleaned_pfns:
|
|
490
530
|
if hint:
|
|
491
531
|
for pattern in protocols[hint]:
|
|
492
|
-
if
|
|
493
|
-
dict_rse[hint].append(
|
|
532
|
+
if pfn.find(pattern) > -1:
|
|
533
|
+
dict_rse[hint].append(pfn)
|
|
494
534
|
else:
|
|
495
535
|
mult_rse_match = 0
|
|
496
536
|
for rse_id in protocols:
|
|
497
537
|
for pattern in protocols[rse_id]:
|
|
498
|
-
if
|
|
538
|
+
if pfn.find(pattern) > -1 and get_rse_vo(rse_id=rse_id, session=session) == vo:
|
|
499
539
|
mult_rse_match += 1
|
|
500
540
|
if mult_rse_match > 1:
|
|
501
|
-
print('ERROR, multiple matches : %s at %s' % (
|
|
502
|
-
raise exception.RucioException('ERROR, multiple matches : %s at %s' % (
|
|
541
|
+
print('ERROR, multiple matches : %s at %s' % (pfn, rse_id))
|
|
542
|
+
raise exception.RucioException('ERROR, multiple matches : %s at %s' % (pfn, get_rse_name(rse_id=rse_id, session=session)))
|
|
503
543
|
hint = rse_id
|
|
504
544
|
if hint not in dict_rse:
|
|
505
545
|
dict_rse[hint] = []
|
|
506
|
-
dict_rse[hint].append(
|
|
546
|
+
dict_rse[hint].append(pfn)
|
|
507
547
|
if mult_rse_match == 0:
|
|
508
548
|
if 'unknown' not in unknown_replicas:
|
|
509
549
|
unknown_replicas['unknown'] = []
|
|
510
|
-
unknown_replicas['unknown'].append(
|
|
550
|
+
unknown_replicas['unknown'].append(pfn)
|
|
511
551
|
return scheme, dict_rse, unknown_replicas
|
|
512
552
|
|
|
513
553
|
|
|
@@ -520,18 +560,28 @@ def get_bad_replicas_backlog(*, session: "Session"):
|
|
|
520
560
|
|
|
521
561
|
:returns: a list of dictionary {'rse_id': cnt_bad_replicas}.
|
|
522
562
|
"""
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
563
|
+
stmt = select(
|
|
564
|
+
func.count(),
|
|
565
|
+
models.RSEFileAssociation.rse_id
|
|
566
|
+
).select_from(
|
|
567
|
+
models.RSEFileAssociation
|
|
568
|
+
).with_hint(
|
|
569
|
+
models.RSEFileAssociation,
|
|
570
|
+
'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))',
|
|
571
|
+
'oracle'
|
|
572
|
+
).join(
|
|
573
|
+
models.DataIdentifier,
|
|
574
|
+
and_(models.RSEFileAssociation.scope == models.DataIdentifier.scope,
|
|
575
|
+
models.RSEFileAssociation.name == models.DataIdentifier.name)
|
|
576
|
+
).where(
|
|
577
|
+
and_(models.DataIdentifier.availability != DIDAvailability.LOST,
|
|
578
|
+
models.RSEFileAssociation.state == ReplicaState.BAD)
|
|
579
|
+
).group_by(
|
|
580
|
+
models.RSEFileAssociation.rse_id
|
|
581
|
+
)
|
|
532
582
|
|
|
533
583
|
result = dict()
|
|
534
|
-
for cnt, rse_id in
|
|
584
|
+
for cnt, rse_id in session.execute(stmt).all():
|
|
535
585
|
result[rse_id] = cnt
|
|
536
586
|
return result
|
|
537
587
|
|
|
@@ -549,27 +599,36 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, *
|
|
|
549
599
|
:returns: a list of dictionary {'scope' scope, 'name': name, 'rse_id': rse_id, 'rse': rse}.
|
|
550
600
|
"""
|
|
551
601
|
schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
602
|
+
|
|
603
|
+
stmt = select(
|
|
604
|
+
models.RSEFileAssociation.scope,
|
|
605
|
+
models.RSEFileAssociation.name,
|
|
606
|
+
models.RSEFileAssociation.rse_id
|
|
607
|
+
).with_hint(
|
|
608
|
+
models.RSEFileAssociation,
|
|
609
|
+
'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))',
|
|
610
|
+
'oracle'
|
|
611
|
+
).where(
|
|
612
|
+
models.RSEFileAssociation.state == ReplicaState.BAD
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_threads, thread_id=thread, hash_variable='%sreplicas.name' % (schema_dot))
|
|
616
|
+
|
|
617
|
+
stmt = stmt.join(
|
|
618
|
+
models.DataIdentifier,
|
|
619
|
+
and_(models.RSEFileAssociation.scope == models.DataIdentifier.scope,
|
|
620
|
+
models.RSEFileAssociation.name == models.DataIdentifier.name)
|
|
621
|
+
).where(
|
|
622
|
+
models.DataIdentifier.availability != DIDAvailability.LOST
|
|
623
|
+
)
|
|
563
624
|
|
|
564
625
|
if rses:
|
|
565
|
-
rse_clause =
|
|
566
|
-
|
|
567
|
-
rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
|
|
568
|
-
query = query.filter(or_(*rse_clause))
|
|
626
|
+
rse_clause = [models.RSEFileAssociation.rse_id == rse['id'] for rse in rses]
|
|
627
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
569
628
|
|
|
570
|
-
|
|
629
|
+
stmt = stmt.limit(limit)
|
|
571
630
|
rows = []
|
|
572
|
-
for scope, name, rse_id in
|
|
631
|
+
for scope, name, rse_id in session.execute(stmt).yield_per(1000):
|
|
573
632
|
rows.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'rse': get_rse_name(rse_id=rse_id, session=session)})
|
|
574
633
|
return rows
|
|
575
634
|
|
|
@@ -599,20 +658,12 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
|
|
|
599
658
|
pfndict = {}
|
|
600
659
|
proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
|
|
601
660
|
if rse_info['deterministic']:
|
|
661
|
+
scope_proto = rsemgr.get_scope_protocol(vo=vo)
|
|
602
662
|
parsed_pfn = proto.parse_pfns(pfns=pfns)
|
|
603
663
|
|
|
604
|
-
# WARNING : this part is ATLAS specific and must be changed
|
|
605
664
|
for pfn in parsed_pfn:
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
|
|
609
|
-
name = parsed_pfn[pfn]['name']
|
|
610
|
-
elif path.startswith('/'):
|
|
611
|
-
scope = path.split('/')[1]
|
|
612
|
-
name = parsed_pfn[pfn]['name']
|
|
613
|
-
else:
|
|
614
|
-
scope = path.split('/')[0]
|
|
615
|
-
name = parsed_pfn[pfn]['name']
|
|
665
|
+
# Translate into a scope and name
|
|
666
|
+
name, scope = scope_proto(parsed_pfn[pfn])
|
|
616
667
|
scope = InternalScope(scope, vo)
|
|
617
668
|
yield {pfn: {'scope': scope, 'name': name}}
|
|
618
669
|
else:
|
|
@@ -621,8 +672,16 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
|
|
|
621
672
|
for pfn in parsed_pfn:
|
|
622
673
|
path = '%s%s' % (parsed_pfn[pfn]['path'], parsed_pfn[pfn]['name'])
|
|
623
674
|
pfndict[path] = pfn
|
|
624
|
-
condition.append(and_(models.RSEFileAssociation.path == path,
|
|
625
|
-
|
|
675
|
+
condition.append(and_(models.RSEFileAssociation.path == path,
|
|
676
|
+
models.RSEFileAssociation.rse_id == rse_id))
|
|
677
|
+
stmt = select(
|
|
678
|
+
models.RSEFileAssociation.scope,
|
|
679
|
+
models.RSEFileAssociation.name,
|
|
680
|
+
models.RSEFileAssociation.path
|
|
681
|
+
).where(
|
|
682
|
+
or_(*condition)
|
|
683
|
+
)
|
|
684
|
+
for scope, name, pfn in session.execute(stmt).all():
|
|
626
685
|
yield {pfndict[pfn]: {'scope': scope, 'name': name}}
|
|
627
686
|
|
|
628
687
|
|
|
@@ -651,7 +710,7 @@ def _pick_n_random(nrandom, generator):
|
|
|
651
710
|
element = next(iterator)
|
|
652
711
|
i += 1
|
|
653
712
|
|
|
654
|
-
index_to_substitute = random.randint(0, i)
|
|
713
|
+
index_to_substitute = random.randint(0, i) # noqa: S311
|
|
655
714
|
if index_to_substitute < nrandom:
|
|
656
715
|
selected[index_to_substitute] = element
|
|
657
716
|
except StopIteration:
|
|
@@ -667,15 +726,21 @@ def _list_files_wo_replicas(files_wo_replica, *, session: "Session"):
|
|
|
667
726
|
for file in sorted(files_wo_replica, key=lambda f: (f['scope'], f['name'])):
|
|
668
727
|
file_wo_clause.append(and_(models.DataIdentifier.scope == file['scope'],
|
|
669
728
|
models.DataIdentifier.name == file['name']))
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
729
|
+
stmt = select(
|
|
730
|
+
models.DataIdentifier.scope,
|
|
731
|
+
models.DataIdentifier.name,
|
|
732
|
+
models.DataIdentifier.bytes,
|
|
733
|
+
models.DataIdentifier.md5,
|
|
734
|
+
models.DataIdentifier.adler32
|
|
735
|
+
).with_hint(
|
|
736
|
+
models.DataIdentifier,
|
|
737
|
+
'INDEX(DIDS DIDS_PK)',
|
|
738
|
+
'oracle'
|
|
739
|
+
).where(
|
|
740
|
+
and_(models.DataIdentifier.did_type == DIDType.FILE,
|
|
741
|
+
or_(*file_wo_clause))
|
|
742
|
+
)
|
|
743
|
+
for scope, name, bytes_, md5, adler32 in session.execute(stmt):
|
|
679
744
|
yield scope, name, bytes_, md5, adler32
|
|
680
745
|
|
|
681
746
|
|
|
@@ -795,7 +860,7 @@ def _build_list_replicas_pfn(
|
|
|
795
860
|
"""
|
|
796
861
|
Generate the PFN for the given scope/name on the rse.
|
|
797
862
|
If needed, sign the PFN url
|
|
798
|
-
If relevant, add the server-side root proxy to
|
|
863
|
+
If relevant, add the server-side root proxy to the pfn url
|
|
799
864
|
"""
|
|
800
865
|
pfn: str = list(protocol.lfns2pfns(lfns={'scope': scope.external,
|
|
801
866
|
'name': name,
|
|
@@ -803,7 +868,7 @@ def _build_list_replicas_pfn(
|
|
|
803
868
|
|
|
804
869
|
# do we need to sign the URLs?
|
|
805
870
|
if sign_urls and protocol.attributes['scheme'] == 'https':
|
|
806
|
-
service = get_rse_attribute(rse_id,
|
|
871
|
+
service = get_rse_attribute(rse_id, RseAttr.SIGN_URL, session=session)
|
|
807
872
|
if service:
|
|
808
873
|
pfn = get_signed_url(rse_id=rse_id, service=service, operation='read', url=pfn, lifetime=signature_lifetime)
|
|
809
874
|
|
|
@@ -814,7 +879,7 @@ def _build_list_replicas_pfn(
|
|
|
814
879
|
if domain == 'wan' and protocol.attributes['scheme'] in ['root', 'http', 'https'] and client_location:
|
|
815
880
|
|
|
816
881
|
if 'site' in client_location and client_location['site']:
|
|
817
|
-
replica_site = get_rse_attribute(rse_id,
|
|
882
|
+
replica_site = get_rse_attribute(rse_id, RseAttr.SITE, session=session)
|
|
818
883
|
|
|
819
884
|
# does it match with the client? if not, it's an outgoing connection
|
|
820
885
|
# therefore the internal proxy must be prepended
|
|
@@ -843,7 +908,7 @@ def _build_list_replicas_pfn(
|
|
|
843
908
|
# don't forget to mangle gfal-style davs URL into generic https URL
|
|
844
909
|
pfn = f"root://{root_proxy_internal}//{pfn.replace('davs://', 'https://')}"
|
|
845
910
|
|
|
846
|
-
simulate_multirange = get_rse_attribute(rse_id,
|
|
911
|
+
simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE)
|
|
847
912
|
|
|
848
913
|
if simulate_multirange is not None:
|
|
849
914
|
try:
|
|
@@ -851,9 +916,9 @@ def _build_list_replicas_pfn(
|
|
|
851
916
|
simulate_multirange = int(simulate_multirange)
|
|
852
917
|
except ValueError:
|
|
853
918
|
simulate_multirange = 1
|
|
854
|
-
logger(logging.WARNING, 'Value encountered when retrieving RSE attribute "
|
|
919
|
+
logger(logging.WARNING, 'Value encountered when retrieving RSE attribute "%s" not compatible with "int", used default value "1".', RseAttr.SIMULATE_MULTIRANGE)
|
|
855
920
|
if simulate_multirange <= 0:
|
|
856
|
-
logger(logging.WARNING, f'Value {simulate_multirange} encountered when retrieving RSE attribute "
|
|
921
|
+
logger(logging.WARNING, f'Value {simulate_multirange} encountered when retrieving RSE attribute "{RseAttr.SIMULATE_MULTIRANGE}" is <= 0, used default value "1".')
|
|
857
922
|
simulate_multirange = 1
|
|
858
923
|
pfn += f'&#multirange=false&nconnections={simulate_multirange}'
|
|
859
924
|
|
|
@@ -936,7 +1001,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
|
|
|
936
1001
|
t_scope = scope
|
|
937
1002
|
t_name = name
|
|
938
1003
|
|
|
939
|
-
if 'determinism_type' in protocol.attributes: # PFN is
|
|
1004
|
+
if 'determinism_type' in protocol.attributes: # PFN is cacheable
|
|
940
1005
|
try:
|
|
941
1006
|
path = pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)]
|
|
942
1007
|
except KeyError: # No cache entry scope:name found for this protocol
|
|
@@ -1081,7 +1146,7 @@ def list_replicas(
|
|
|
1081
1146
|
models.RSE.volatile,
|
|
1082
1147
|
).join(
|
|
1083
1148
|
models.RSE,
|
|
1084
|
-
and_(models.
|
|
1149
|
+
and_(models.RSEFileAssociation.rse_id == models.RSE.id,
|
|
1085
1150
|
models.RSE.deleted == false())
|
|
1086
1151
|
)
|
|
1087
1152
|
|
|
@@ -1100,7 +1165,11 @@ def list_replicas(
|
|
|
1100
1165
|
stmt = stmt.where(models.RSE.id.in_([rse['id'] for rse in rses]))
|
|
1101
1166
|
else:
|
|
1102
1167
|
rses_temp_table = temp_table_mngr(session).create_id_table()
|
|
1103
|
-
|
|
1168
|
+
values = [{'id': rse['id']} for rse in rses]
|
|
1169
|
+
insert_stmt = insert(
|
|
1170
|
+
rses_temp_table
|
|
1171
|
+
)
|
|
1172
|
+
session.execute(insert_stmt, values)
|
|
1104
1173
|
stmt = stmt.join(rses_temp_table, models.RSE.id == rses_temp_table.id)
|
|
1105
1174
|
|
|
1106
1175
|
if not all_states:
|
|
@@ -1122,16 +1191,16 @@ def list_replicas(
|
|
|
1122
1191
|
created temporary table.
|
|
1123
1192
|
"""
|
|
1124
1193
|
resolved_files_temp_table = temp_table_mngr(session).create_scope_name_table()
|
|
1194
|
+
selectable = rucio.core.did.list_child_dids_stmt(temp_table, did_type=DIDType.FILE)
|
|
1125
1195
|
|
|
1126
1196
|
stmt = insert(
|
|
1127
1197
|
resolved_files_temp_table
|
|
1128
1198
|
).from_select(
|
|
1129
1199
|
['scope', 'name'],
|
|
1130
|
-
|
|
1200
|
+
selectable
|
|
1131
1201
|
)
|
|
1132
|
-
result = session.execute(stmt)
|
|
1133
1202
|
|
|
1134
|
-
return
|
|
1203
|
+
return session.execute(stmt).rowcount, resolved_files_temp_table
|
|
1135
1204
|
|
|
1136
1205
|
def _list_replicas_for_collection_files_stmt(temp_table, replicas_subquery):
|
|
1137
1206
|
"""
|
|
@@ -1264,7 +1333,11 @@ def list_replicas(
|
|
|
1264
1333
|
return
|
|
1265
1334
|
|
|
1266
1335
|
input_dids_temp_table = temp_table_mngr(session).create_scope_name_table()
|
|
1267
|
-
|
|
1336
|
+
values = [{'scope': scope, 'name': name} for scope, name in dids]
|
|
1337
|
+
stmt = insert(
|
|
1338
|
+
input_dids_temp_table
|
|
1339
|
+
)
|
|
1340
|
+
session.execute(stmt, values)
|
|
1268
1341
|
|
|
1269
1342
|
num_files, num_collections, num_constituents = _inspect_dids(input_dids_temp_table, session=session)
|
|
1270
1343
|
|
|
@@ -1325,10 +1398,16 @@ def list_replicas(
|
|
|
1325
1398
|
# leave us with less than nrandom replicas.
|
|
1326
1399
|
nrandom * 4
|
|
1327
1400
|
)
|
|
1328
|
-
#
|
|
1401
|
+
# Reuse input temp table. We don't need its content anymore
|
|
1329
1402
|
random_dids_temp_table = input_dids_temp_table
|
|
1330
1403
|
session.execute(delete(random_dids_temp_table))
|
|
1331
|
-
|
|
1404
|
+
stmt = insert(
|
|
1405
|
+
random_dids_temp_table
|
|
1406
|
+
).from_select(
|
|
1407
|
+
['scope', 'name'],
|
|
1408
|
+
stmt
|
|
1409
|
+
)
|
|
1410
|
+
session.execute(stmt)
|
|
1332
1411
|
|
|
1333
1412
|
# Fetch all replicas for randomly selected dids and apply filters on python side
|
|
1334
1413
|
stmt = _list_replicas_for_collection_files_stmt(random_dids_temp_table, replicas_subquery)
|
|
@@ -1426,18 +1505,28 @@ def __bulk_add_file_dids(files, account, dataset_meta=None, *, session: "Session
|
|
|
1426
1505
|
:param dids: the list of files.
|
|
1427
1506
|
:param account: The account owner.
|
|
1428
1507
|
:param session: The database session in use.
|
|
1429
|
-
:returns:
|
|
1508
|
+
:returns: list of replicas.
|
|
1430
1509
|
"""
|
|
1431
1510
|
condition = []
|
|
1432
1511
|
for f in files:
|
|
1433
|
-
condition.append(and_(models.DataIdentifier.scope == f['scope'],
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1512
|
+
condition.append(and_(models.DataIdentifier.scope == f['scope'],
|
|
1513
|
+
models.DataIdentifier.name == f['name'],
|
|
1514
|
+
models.DataIdentifier.did_type == DIDType.FILE))
|
|
1515
|
+
|
|
1516
|
+
stmt = select(
|
|
1517
|
+
models.DataIdentifier.scope,
|
|
1518
|
+
models.DataIdentifier.name,
|
|
1519
|
+
models.DataIdentifier.bytes,
|
|
1520
|
+
models.DataIdentifier.md5,
|
|
1521
|
+
models.DataIdentifier.adler32,
|
|
1522
|
+
).with_hint(
|
|
1523
|
+
models.DataIdentifier,
|
|
1524
|
+
'INDEX(DIDS DIDS_PK)',
|
|
1525
|
+
'oracle'
|
|
1526
|
+
).where(
|
|
1527
|
+
or_(*condition)
|
|
1528
|
+
)
|
|
1529
|
+
available_files = [res._asdict() for res in session.execute(stmt).all()]
|
|
1441
1530
|
new_files = list()
|
|
1442
1531
|
for file in files:
|
|
1443
1532
|
found = False
|
|
@@ -1484,14 +1573,25 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
|
|
|
1484
1573
|
# Check for the replicas already available
|
|
1485
1574
|
condition = []
|
|
1486
1575
|
for f in files:
|
|
1487
|
-
condition.append(and_(models.RSEFileAssociation.scope == f['scope'],
|
|
1576
|
+
condition.append(and_(models.RSEFileAssociation.scope == f['scope'],
|
|
1577
|
+
models.RSEFileAssociation.name == f['name'],
|
|
1578
|
+
models.RSEFileAssociation.rse_id == rse_id))
|
|
1488
1579
|
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1580
|
+
stmt = select(
|
|
1581
|
+
models.RSEFileAssociation.scope,
|
|
1582
|
+
models.RSEFileAssociation.name,
|
|
1583
|
+
models.RSEFileAssociation.rse_id,
|
|
1584
|
+
).with_hint(
|
|
1585
|
+
models.RSEFileAssociation,
|
|
1586
|
+
'INDEX(REPLICAS REPLICAS_PK)',
|
|
1587
|
+
'oracle'
|
|
1588
|
+
).where(
|
|
1589
|
+
or_(*condition)
|
|
1590
|
+
)
|
|
1493
1591
|
|
|
1494
|
-
|
|
1592
|
+
available_replicas = [res._asdict() for res in session.execute(stmt).all()]
|
|
1593
|
+
|
|
1594
|
+
default_tombstone_delay = get_rse_attribute(rse_id, RseAttr.TOMBSTONE_DELAY, session=session)
|
|
1495
1595
|
default_tombstone = tombstone_from_delay(default_tombstone_delay)
|
|
1496
1596
|
|
|
1497
1597
|
new_replicas = []
|
|
@@ -1512,7 +1612,10 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
|
|
|
1512
1612
|
'lock_cnt': file.get('lock_cnt', 0),
|
|
1513
1613
|
'tombstone': file.get('tombstone') or default_tombstone})
|
|
1514
1614
|
try:
|
|
1515
|
-
|
|
1615
|
+
stmt = insert(
|
|
1616
|
+
models.RSEFileAssociation
|
|
1617
|
+
)
|
|
1618
|
+
new_replicas and session.execute(stmt, new_replicas)
|
|
1516
1619
|
session.flush()
|
|
1517
1620
|
return nbfiles, bytes_
|
|
1518
1621
|
except IntegrityError as error:
|
|
@@ -1538,13 +1641,13 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1538
1641
|
:param ignore_availability: Ignore the RSE blocklisting.
|
|
1539
1642
|
:param session: The database session in use.
|
|
1540
1643
|
|
|
1541
|
-
:returns:
|
|
1644
|
+
:returns: list of replicas.
|
|
1542
1645
|
"""
|
|
1543
1646
|
|
|
1544
1647
|
def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=None):
|
|
1545
1648
|
p = rsemgr.create_protocol(rse_settings=rse_settings, operation='write', scheme=scheme, domain=domain, protocol_attr=protocol_attr)
|
|
1546
1649
|
expected_pfns = p.lfns2pfns(lfns)
|
|
1547
|
-
return
|
|
1650
|
+
return clean_pfns(expected_pfns.values())
|
|
1548
1651
|
|
|
1549
1652
|
replica_rse = get_rse(rse_id=rse_id, session=session)
|
|
1550
1653
|
|
|
@@ -1559,9 +1662,9 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1559
1662
|
if not replica_rse['deterministic']:
|
|
1560
1663
|
raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %s ' % (replica_rse['rse']))
|
|
1561
1664
|
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1665
|
+
__bulk_add_file_dids(files=files, account=account,
|
|
1666
|
+
dataset_meta=dataset_meta,
|
|
1667
|
+
session=session)
|
|
1565
1668
|
|
|
1566
1669
|
pfns = {} # dict[str, list[str]], {scheme: [pfns], scheme: [pfns]}
|
|
1567
1670
|
for file in files:
|
|
@@ -1582,7 +1685,7 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1582
1685
|
else:
|
|
1583
1686
|
# Check that the pfns match to the expected pfns
|
|
1584
1687
|
lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
|
|
1585
|
-
pfns[scheme] =
|
|
1688
|
+
pfns[scheme] = clean_pfns(pfns[scheme])
|
|
1586
1689
|
|
|
1587
1690
|
for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan'):
|
|
1588
1691
|
pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)))
|
|
@@ -1597,11 +1700,25 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
|
|
|
1597
1700
|
|
|
1598
1701
|
nbfiles, bytes_ = __bulk_add_replicas(rse_id=rse_id, files=files, account=account, session=session)
|
|
1599
1702
|
increase(rse_id=rse_id, files=nbfiles, bytes_=bytes_, session=session)
|
|
1600
|
-
return replicas
|
|
1601
1703
|
|
|
1602
1704
|
|
|
1603
1705
|
@transactional_session
|
|
1604
|
-
def add_replica(
|
|
1706
|
+
def add_replica(
|
|
1707
|
+
rse_id: str,
|
|
1708
|
+
scope: InternalScope,
|
|
1709
|
+
name: str,
|
|
1710
|
+
bytes_: int,
|
|
1711
|
+
account: models.InternalAccount,
|
|
1712
|
+
adler32: "Optional[str]" = None,
|
|
1713
|
+
md5: "Optional[str]" = None,
|
|
1714
|
+
dsn: "Optional[str]" = None,
|
|
1715
|
+
pfn: "Optional[str]" = None,
|
|
1716
|
+
meta: "Optional[dict[str, Any]]" = None,
|
|
1717
|
+
rules: "Optional[list[dict[str, Any]]]" = None,
|
|
1718
|
+
tombstone: "Optional[datetime]" = None,
|
|
1719
|
+
*,
|
|
1720
|
+
session: "Session"
|
|
1721
|
+
) -> "list[dict[str, Any]]":
|
|
1605
1722
|
"""
|
|
1606
1723
|
Add File replica.
|
|
1607
1724
|
|
|
@@ -1618,10 +1735,10 @@ def add_replica(rse_id, scope, name, bytes_, account, adler32=None, md5=None, ds
|
|
|
1618
1735
|
:param tombstone: If True, create replica with a tombstone.
|
|
1619
1736
|
:param session: The database session in use.
|
|
1620
1737
|
|
|
1621
|
-
:returns:
|
|
1738
|
+
:returns: list of replicas.
|
|
1622
1739
|
"""
|
|
1623
|
-
|
|
1624
|
-
|
|
1740
|
+
meta = meta or {}
|
|
1741
|
+
rules = rules or []
|
|
1625
1742
|
|
|
1626
1743
|
file = {'scope': scope, 'name': name, 'bytes': bytes_, 'adler32': adler32, 'md5': md5, 'meta': meta, 'rules': rules, 'tombstone': tombstone}
|
|
1627
1744
|
if pfn:
|
|
@@ -1653,7 +1770,11 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
|
|
|
1653
1770
|
scope_name_temp_table2 = tt_mngr.create_scope_name_table()
|
|
1654
1771
|
association_temp_table = tt_mngr.create_association_table()
|
|
1655
1772
|
|
|
1656
|
-
|
|
1773
|
+
values = [{'scope': file['scope'], 'name': file['name']} for file in files]
|
|
1774
|
+
stmt = insert(
|
|
1775
|
+
scope_name_temp_table
|
|
1776
|
+
)
|
|
1777
|
+
session.execute(stmt, values)
|
|
1657
1778
|
|
|
1658
1779
|
# WARNING : This should not be necessary since that would mean the replica is used as a source.
|
|
1659
1780
|
stmt = delete(
|
|
@@ -1685,9 +1806,10 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
|
|
|
1685
1806
|
models.RSEFileAssociation,
|
|
1686
1807
|
).where(
|
|
1687
1808
|
exists(select(1)
|
|
1688
|
-
.where(
|
|
1689
|
-
|
|
1690
|
-
|
|
1809
|
+
.where(
|
|
1810
|
+
and_(models.RSEFileAssociation.scope == scope_name_temp_table.scope,
|
|
1811
|
+
models.RSEFileAssociation.name == scope_name_temp_table.name,
|
|
1812
|
+
models.RSEFileAssociation.rse_id == rse_id)))
|
|
1691
1813
|
).execution_options(
|
|
1692
1814
|
synchronize_session=False
|
|
1693
1815
|
)
|
|
@@ -1697,20 +1819,22 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
|
|
|
1697
1819
|
|
|
1698
1820
|
# Update bad replicas
|
|
1699
1821
|
stmt = update(
|
|
1700
|
-
models.
|
|
1822
|
+
models.BadReplica,
|
|
1701
1823
|
).where(
|
|
1702
1824
|
exists(select(1)
|
|
1703
|
-
.where(
|
|
1704
|
-
|
|
1705
|
-
|
|
1825
|
+
.where(
|
|
1826
|
+
and_(models.BadReplica.scope == scope_name_temp_table.scope,
|
|
1827
|
+
models.BadReplica.name == scope_name_temp_table.name,
|
|
1828
|
+
models.BadReplica.rse_id == rse_id)))
|
|
1706
1829
|
).where(
|
|
1707
|
-
models.
|
|
1708
|
-
).
|
|
1830
|
+
models.BadReplica.state == BadFilesStatus.BAD
|
|
1831
|
+
).values({
|
|
1832
|
+
models.BadReplica.state: BadFilesStatus.DELETED,
|
|
1833
|
+
models.BadReplica.updated_at: datetime.utcnow()
|
|
1834
|
+
}).execution_options(
|
|
1709
1835
|
synchronize_session=False
|
|
1710
|
-
).values(
|
|
1711
|
-
state=BadFilesStatus.DELETED,
|
|
1712
|
-
updated_at=datetime.utcnow()
|
|
1713
1836
|
)
|
|
1837
|
+
|
|
1714
1838
|
res = session.execute(stmt)
|
|
1715
1839
|
|
|
1716
1840
|
__cleanup_after_replica_deletion(scope_name_temp_table=scope_name_temp_table,
|
|
@@ -1762,8 +1886,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1762
1886
|
|
|
1763
1887
|
if clt_to_update:
|
|
1764
1888
|
# Get all collection_replicas at RSE, insert them into UpdatedCollectionReplica
|
|
1765
|
-
|
|
1766
|
-
session.execute(
|
|
1889
|
+
stmt = delete(scope_name_temp_table)
|
|
1890
|
+
session.execute(stmt)
|
|
1891
|
+
values = [sn._asdict() for sn in clt_to_update]
|
|
1892
|
+
stmt = insert(scope_name_temp_table)
|
|
1893
|
+
session.execute(stmt, values)
|
|
1767
1894
|
stmt = select(
|
|
1768
1895
|
models.DataIdentifierAssociation.scope,
|
|
1769
1896
|
models.DataIdentifierAssociation.name,
|
|
@@ -1790,8 +1917,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1790
1917
|
while parents_to_analyze:
|
|
1791
1918
|
did_associations_to_remove = set()
|
|
1792
1919
|
|
|
1793
|
-
|
|
1794
|
-
session.execute(
|
|
1920
|
+
stmt = delete(scope_name_temp_table)
|
|
1921
|
+
session.execute(stmt)
|
|
1922
|
+
values = [sn._asdict() for sn in parents_to_analyze]
|
|
1923
|
+
stmt = insert(scope_name_temp_table)
|
|
1924
|
+
session.execute(stmt, values)
|
|
1795
1925
|
parents_to_analyze.clear()
|
|
1796
1926
|
|
|
1797
1927
|
stmt = select(
|
|
@@ -1862,7 +1992,7 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1862
1992
|
did_condition.append(
|
|
1863
1993
|
and_(models.DataIdentifier.scope == parent_scope,
|
|
1864
1994
|
models.DataIdentifier.name == parent_name,
|
|
1865
|
-
models.DataIdentifier.is_open ==
|
|
1995
|
+
models.DataIdentifier.is_open == false(),
|
|
1866
1996
|
~exists(1).where(
|
|
1867
1997
|
and_(models.DataIdentifierAssociation.child_scope == parent_scope,
|
|
1868
1998
|
models.DataIdentifierAssociation.child_name == parent_name)),
|
|
@@ -1871,8 +2001,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1871
2001
|
models.DataIdentifierAssociation.name == parent_name))))
|
|
1872
2002
|
|
|
1873
2003
|
if did_associations_to_remove:
|
|
1874
|
-
|
|
1875
|
-
session.execute(
|
|
2004
|
+
stmt = delete(association_temp_table)
|
|
2005
|
+
session.execute(stmt)
|
|
2006
|
+
values = [a._asdict() for a in did_associations_to_remove]
|
|
2007
|
+
stmt = insert(association_temp_table)
|
|
2008
|
+
session.execute(stmt, values)
|
|
1876
2009
|
|
|
1877
2010
|
# get the list of modified parent scope, name
|
|
1878
2011
|
stmt = select(
|
|
@@ -1887,7 +2020,7 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1887
2020
|
association_temp_table.name == models.DataIdentifier.name)
|
|
1888
2021
|
).where(
|
|
1889
2022
|
or_(models.DataIdentifier.complete == true(),
|
|
1890
|
-
models.DataIdentifier.complete
|
|
2023
|
+
models.DataIdentifier.complete.is_(None)),
|
|
1891
2024
|
)
|
|
1892
2025
|
for parent_scope, parent_name, parent_did_type in session.execute(stmt):
|
|
1893
2026
|
message = {'scope': parent_scope,
|
|
@@ -1917,9 +2050,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1917
2050
|
|
|
1918
2051
|
# Get collection replicas of collections which became empty
|
|
1919
2052
|
if clt_replicas_to_delete:
|
|
1920
|
-
|
|
1921
|
-
session.execute(
|
|
1922
|
-
|
|
2053
|
+
stmt = delete(scope_name_temp_table)
|
|
2054
|
+
session.execute(stmt)
|
|
2055
|
+
values = [sn._asdict() for sn in clt_replicas_to_delete]
|
|
2056
|
+
stmt = insert(scope_name_temp_table)
|
|
2057
|
+
session.execute(stmt, values)
|
|
2058
|
+
stmt = delete(scope_name_temp_table2)
|
|
2059
|
+
session.execute(stmt)
|
|
1923
2060
|
stmt = select(
|
|
1924
2061
|
models.CollectionReplica.scope,
|
|
1925
2062
|
models.CollectionReplica.name,
|
|
@@ -1940,9 +2077,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1940
2077
|
).where(
|
|
1941
2078
|
models.DataIdentifierAssociation.scope == null()
|
|
1942
2079
|
)
|
|
1943
|
-
|
|
1944
|
-
|
|
2080
|
+
stmt = insert(
|
|
2081
|
+
scope_name_temp_table2
|
|
2082
|
+
).from_select(
|
|
2083
|
+
['scope', 'name'],
|
|
2084
|
+
stmt
|
|
1945
2085
|
)
|
|
2086
|
+
session.execute(stmt)
|
|
1946
2087
|
# Delete the retrieved collection replicas of empty collections
|
|
1947
2088
|
stmt = delete(
|
|
1948
2089
|
models.CollectionReplica,
|
|
@@ -1958,8 +2099,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1958
2099
|
# Update incomplete state
|
|
1959
2100
|
messages, dids_to_delete = [], set()
|
|
1960
2101
|
if incomplete_dids:
|
|
1961
|
-
|
|
1962
|
-
session.execute(
|
|
2102
|
+
stmt = delete(scope_name_temp_table)
|
|
2103
|
+
session.execute(stmt)
|
|
2104
|
+
values = [sn._asdict() for sn in incomplete_dids]
|
|
2105
|
+
stmt = insert(scope_name_temp_table)
|
|
2106
|
+
session.execute(stmt, values)
|
|
1963
2107
|
stmt = update(
|
|
1964
2108
|
models.DataIdentifier
|
|
1965
2109
|
).where(
|
|
@@ -1968,22 +2112,29 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1968
2112
|
models.DataIdentifier.name == scope_name_temp_table.name)))
|
|
1969
2113
|
).where(
|
|
1970
2114
|
models.DataIdentifier.complete != false(),
|
|
1971
|
-
).
|
|
2115
|
+
).values({
|
|
2116
|
+
models.DataIdentifier.complete: False
|
|
2117
|
+
}).execution_options(
|
|
1972
2118
|
synchronize_session=False
|
|
1973
|
-
).values(
|
|
1974
|
-
complete=False
|
|
1975
2119
|
)
|
|
2120
|
+
|
|
1976
2121
|
session.execute(stmt)
|
|
1977
2122
|
|
|
1978
2123
|
# delete empty dids
|
|
1979
2124
|
if did_condition:
|
|
1980
2125
|
for chunk in chunks(did_condition, 10):
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
2126
|
+
stmt = select(
|
|
2127
|
+
models.DataIdentifier.scope,
|
|
2128
|
+
models.DataIdentifier.name,
|
|
2129
|
+
models.DataIdentifier.did_type,
|
|
2130
|
+
).with_hint(
|
|
2131
|
+
models.DataIdentifier,
|
|
2132
|
+
'INDEX(DIDS DIDS_PK)',
|
|
2133
|
+
'oracle'
|
|
2134
|
+
).where(
|
|
2135
|
+
or_(*chunk)
|
|
2136
|
+
)
|
|
2137
|
+
for scope, name, did_type in session.execute(stmt):
|
|
1987
2138
|
if did_type == DIDType.DATASET:
|
|
1988
2139
|
messages.append({'event_type': 'ERASE',
|
|
1989
2140
|
'payload': dumps({'scope': scope.external,
|
|
@@ -1994,8 +2145,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
1994
2145
|
# Remove Archive Constituents
|
|
1995
2146
|
constituent_associations_to_delete = set()
|
|
1996
2147
|
if affected_archives:
|
|
1997
|
-
|
|
1998
|
-
session.execute(
|
|
2148
|
+
stmt = delete(scope_name_temp_table)
|
|
2149
|
+
session.execute(stmt)
|
|
2150
|
+
values = [sn._asdict() for sn in affected_archives]
|
|
2151
|
+
stmt = insert(scope_name_temp_table)
|
|
2152
|
+
session.execute(stmt, values)
|
|
1999
2153
|
|
|
2000
2154
|
stmt = select(
|
|
2001
2155
|
models.ConstituentAssociation
|
|
@@ -2038,8 +2192,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2038
2192
|
).save(session=session, flush=False)
|
|
2039
2193
|
|
|
2040
2194
|
if constituent_associations_to_delete:
|
|
2041
|
-
|
|
2042
|
-
session.execute(
|
|
2195
|
+
stmt = delete(association_temp_table)
|
|
2196
|
+
session.execute(stmt)
|
|
2197
|
+
values = [a._asdict() for a in constituent_associations_to_delete]
|
|
2198
|
+
stmt = insert(association_temp_table)
|
|
2199
|
+
session.execute(stmt, values)
|
|
2043
2200
|
stmt = delete(
|
|
2044
2201
|
models.ConstituentAssociation
|
|
2045
2202
|
).where(
|
|
@@ -2061,8 +2218,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2061
2218
|
rse_id=rse_id, files=[sn._asdict() for sn in chunk], session=session)
|
|
2062
2219
|
|
|
2063
2220
|
if dids_to_delete:
|
|
2064
|
-
|
|
2065
|
-
session.execute(
|
|
2221
|
+
stmt = delete(scope_name_temp_table)
|
|
2222
|
+
session.execute(stmt)
|
|
2223
|
+
values = [sn._asdict() for sn in dids_to_delete]
|
|
2224
|
+
stmt = insert(scope_name_temp_table)
|
|
2225
|
+
session.execute(stmt, values)
|
|
2066
2226
|
|
|
2067
2227
|
# Remove rules in Waiting for approval or Suspended
|
|
2068
2228
|
stmt = delete(
|
|
@@ -2120,9 +2280,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2120
2280
|
to_update = clt_to_set_not_archive.pop(0)
|
|
2121
2281
|
if not to_update:
|
|
2122
2282
|
continue
|
|
2123
|
-
|
|
2124
|
-
session.execute(
|
|
2125
|
-
|
|
2283
|
+
stmt = delete(scope_name_temp_table)
|
|
2284
|
+
session.execute(stmt)
|
|
2285
|
+
values = [sn._asdict() for sn in to_update]
|
|
2286
|
+
stmt = insert(scope_name_temp_table)
|
|
2287
|
+
session.execute(stmt, values)
|
|
2288
|
+
stmt = delete(scope_name_temp_table2)
|
|
2289
|
+
session.execute(stmt)
|
|
2126
2290
|
|
|
2127
2291
|
data_identifier_alias = aliased(models.DataIdentifier, name='did_alias')
|
|
2128
2292
|
# Fetch rows to be updated
|
|
@@ -2149,7 +2313,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2149
2313
|
).where(
|
|
2150
2314
|
data_identifier_alias.scope == null()
|
|
2151
2315
|
)
|
|
2152
|
-
|
|
2316
|
+
stmt = insert(
|
|
2317
|
+
scope_name_temp_table2
|
|
2318
|
+
).from_select(
|
|
2319
|
+
['scope', 'name'],
|
|
2320
|
+
stmt
|
|
2321
|
+
)
|
|
2322
|
+
session.execute(stmt)
|
|
2153
2323
|
# update the fetched rows
|
|
2154
2324
|
stmt = update(
|
|
2155
2325
|
models.DataIdentifier,
|
|
@@ -2157,10 +2327,10 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
|
|
|
2157
2327
|
exists(select(1)
|
|
2158
2328
|
.where(and_(models.DataIdentifier.scope == scope_name_temp_table2.scope,
|
|
2159
2329
|
models.DataIdentifier.name == scope_name_temp_table2.name)))
|
|
2160
|
-
).
|
|
2330
|
+
).values({
|
|
2331
|
+
models.DataIdentifier.is_archive: False
|
|
2332
|
+
}).execution_options(
|
|
2161
2333
|
synchronize_session=False
|
|
2162
|
-
).values(
|
|
2163
|
-
is_archive=False,
|
|
2164
2334
|
)
|
|
2165
2335
|
session.execute(stmt)
|
|
2166
2336
|
|
|
@@ -2178,8 +2348,14 @@ def get_replica(rse_id, scope, name, *, session: "Session"):
|
|
|
2178
2348
|
:returns: A dictionary with the list of replica attributes.
|
|
2179
2349
|
"""
|
|
2180
2350
|
try:
|
|
2181
|
-
|
|
2182
|
-
|
|
2351
|
+
stmt = select(
|
|
2352
|
+
models.RSEFileAssociation
|
|
2353
|
+
).where(
|
|
2354
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2355
|
+
models.RSEFileAssociation.name == name,
|
|
2356
|
+
models.RSEFileAssociation.rse_id == rse_id)
|
|
2357
|
+
)
|
|
2358
|
+
return session.execute(stmt).scalar_one().to_dict()
|
|
2183
2359
|
except NoResultFound:
|
|
2184
2360
|
raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
|
|
2185
2361
|
|
|
@@ -2236,8 +2412,11 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2236
2412
|
)
|
|
2237
2413
|
|
|
2238
2414
|
for chunk in chunks(session.execute(stmt).yield_per(2 * limit), math.ceil(1.25 * limit)):
|
|
2239
|
-
|
|
2240
|
-
session.execute(
|
|
2415
|
+
stmt = delete(temp_table_cls)
|
|
2416
|
+
session.execute(stmt)
|
|
2417
|
+
values = [{'scope': scope, 'name': name} for scope, name in chunk]
|
|
2418
|
+
stmt = insert(temp_table_cls)
|
|
2419
|
+
session.execute(stmt, values)
|
|
2241
2420
|
|
|
2242
2421
|
stmt = select(
|
|
2243
2422
|
models.RSEFileAssociation.scope,
|
|
@@ -2254,7 +2433,9 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2254
2433
|
models.RSEFileAssociation.name == temp_table_cls.name,
|
|
2255
2434
|
models.RSEFileAssociation.rse_id == rse_id)
|
|
2256
2435
|
).with_hint(
|
|
2257
|
-
replicas_alias,
|
|
2436
|
+
replicas_alias,
|
|
2437
|
+
'INDEX(%(name)s REPLICAS_PK)',
|
|
2438
|
+
'oracle'
|
|
2258
2439
|
).outerjoin(
|
|
2259
2440
|
replicas_alias,
|
|
2260
2441
|
and_(models.RSEFileAssociation.scope == replicas_alias.scope,
|
|
@@ -2262,7 +2443,9 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2262
2443
|
models.RSEFileAssociation.rse_id != replicas_alias.rse_id,
|
|
2263
2444
|
replicas_alias.state == ReplicaState.AVAILABLE)
|
|
2264
2445
|
).with_hint(
|
|
2265
|
-
models.Request,
|
|
2446
|
+
models.Request,
|
|
2447
|
+
'INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX)',
|
|
2448
|
+
'oracle'
|
|
2266
2449
|
).outerjoin(
|
|
2267
2450
|
models.Request,
|
|
2268
2451
|
and_(models.RSEFileAssociation.scope == models.Request.scope,
|
|
@@ -2304,8 +2487,11 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2304
2487
|
break
|
|
2305
2488
|
|
|
2306
2489
|
if rows:
|
|
2307
|
-
|
|
2308
|
-
session.execute(
|
|
2490
|
+
stmt = delete(temp_table_cls)
|
|
2491
|
+
session.execute(stmt)
|
|
2492
|
+
values = [{'scope': row['scope'], 'name': row['name']} for row in rows]
|
|
2493
|
+
stmt = insert(temp_table_cls)
|
|
2494
|
+
session.execute(stmt, values)
|
|
2309
2495
|
stmt = update(
|
|
2310
2496
|
models.RSEFileAssociation
|
|
2311
2497
|
).where(
|
|
@@ -2313,13 +2499,14 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
|
|
|
2313
2499
|
.where(and_(models.RSEFileAssociation.scope == temp_table_cls.scope,
|
|
2314
2500
|
models.RSEFileAssociation.name == temp_table_cls.name,
|
|
2315
2501
|
models.RSEFileAssociation.rse_id == rse_id)))
|
|
2316
|
-
).
|
|
2502
|
+
).values({
|
|
2503
|
+
models.RSEFileAssociation.updated_at: datetime.utcnow(),
|
|
2504
|
+
models.RSEFileAssociation.state: ReplicaState.BEING_DELETED,
|
|
2505
|
+
models.RSEFileAssociation.tombstone: OBSOLETE
|
|
2506
|
+
}).execution_options(
|
|
2317
2507
|
synchronize_session=False
|
|
2318
|
-
).values(
|
|
2319
|
-
updated_at=datetime.utcnow(),
|
|
2320
|
-
state=ReplicaState.BEING_DELETED,
|
|
2321
|
-
tombstone=OBSOLETE,
|
|
2322
2508
|
)
|
|
2509
|
+
|
|
2323
2510
|
session.execute(stmt)
|
|
2324
2511
|
|
|
2325
2512
|
return rows
|
|
@@ -2336,11 +2523,17 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
|
|
|
2336
2523
|
"""
|
|
2337
2524
|
|
|
2338
2525
|
for replica in replicas:
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2526
|
+
stmt = select(
|
|
2527
|
+
models.RSEFileAssociation
|
|
2528
|
+
).where(
|
|
2529
|
+
models.RSEFileAssociation.rse_id == replica['rse_id'],
|
|
2530
|
+
models.RSEFileAssociation.scope == replica['scope'],
|
|
2531
|
+
models.RSEFileAssociation.name == replica['name']
|
|
2532
|
+
).with_for_update(
|
|
2533
|
+
nowait=nowait
|
|
2534
|
+
)
|
|
2535
|
+
|
|
2536
|
+
if session.execute(stmt).scalar_one_or_none() is None:
|
|
2344
2537
|
# remember scope, name and rse
|
|
2345
2538
|
raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (replica['scope'], replica['name'], get_rse_name(replica['rse_id'], session=session)))
|
|
2346
2539
|
|
|
@@ -2349,18 +2542,43 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
|
|
|
2349
2542
|
|
|
2350
2543
|
values = {'state': replica['state']}
|
|
2351
2544
|
if replica['state'] == ReplicaState.BEING_DELETED:
|
|
2352
|
-
query = query.filter_by(lock_cnt=0)
|
|
2353
2545
|
# Exclude replicas use as sources
|
|
2354
|
-
stmt =
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2546
|
+
stmt = stmt.where(
|
|
2547
|
+
and_(models.RSEFileAssociation.lock_cnt == 0,
|
|
2548
|
+
not_(exists(select(1)
|
|
2549
|
+
.where(and_(models.RSEFileAssociation.scope == models.Source.scope,
|
|
2550
|
+
models.RSEFileAssociation.name == models.Source.name,
|
|
2551
|
+
models.RSEFileAssociation.rse_id == models.Source.rse_id)))))
|
|
2552
|
+
)
|
|
2358
2553
|
values['tombstone'] = OBSOLETE
|
|
2359
2554
|
elif replica['state'] == ReplicaState.AVAILABLE:
|
|
2360
2555
|
rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session)
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2556
|
+
stmt_bad_replicas = select(
|
|
2557
|
+
func.count()
|
|
2558
|
+
).select_from(
|
|
2559
|
+
models.BadReplica
|
|
2560
|
+
).where(
|
|
2561
|
+
and_(models.BadReplica.state == BadFilesStatus.BAD,
|
|
2562
|
+
models.BadReplica.rse_id == replica['rse_id'],
|
|
2563
|
+
models.BadReplica.scope == replica['scope'],
|
|
2564
|
+
models.BadReplica.name == replica['name'])
|
|
2565
|
+
)
|
|
2566
|
+
|
|
2567
|
+
if session.execute(stmt_bad_replicas).scalar():
|
|
2568
|
+
update_stmt = update(
|
|
2569
|
+
models.BadReplica
|
|
2570
|
+
).where(
|
|
2571
|
+
and_(models.BadReplica.state == BadFilesStatus.BAD,
|
|
2572
|
+
models.BadReplica.rse_id == replica['rse_id'],
|
|
2573
|
+
models.BadReplica.scope == replica['scope'],
|
|
2574
|
+
models.BadReplica.name == replica['name'])
|
|
2575
|
+
).values({
|
|
2576
|
+
models.BadReplica.state: BadFilesStatus.RECOVERED,
|
|
2577
|
+
models.BadReplica.updated_at: datetime.utcnow()
|
|
2578
|
+
}).execution_options(
|
|
2579
|
+
synchronize_session=False
|
|
2580
|
+
)
|
|
2581
|
+
session.execute(update_stmt)
|
|
2364
2582
|
elif replica['state'] == ReplicaState.UNAVAILABLE:
|
|
2365
2583
|
rucio.core.lock.failed_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'],
|
|
2366
2584
|
error_message=replica.get('error_message', None),
|
|
@@ -2368,12 +2586,27 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
|
|
|
2368
2586
|
broken_message=replica.get('broken_message', None),
|
|
2369
2587
|
nowait=nowait, session=session)
|
|
2370
2588
|
elif replica['state'] == ReplicaState.TEMPORARY_UNAVAILABLE:
|
|
2371
|
-
|
|
2589
|
+
stmt = stmt.where(
|
|
2590
|
+
models.RSEFileAssociation.state.in_([ReplicaState.AVAILABLE,
|
|
2591
|
+
ReplicaState.TEMPORARY_UNAVAILABLE])
|
|
2592
|
+
)
|
|
2372
2593
|
|
|
2373
2594
|
if 'path' in replica and replica['path']:
|
|
2374
2595
|
values['path'] = replica['path']
|
|
2375
2596
|
|
|
2376
|
-
|
|
2597
|
+
update_stmt = update(
|
|
2598
|
+
models.RSEFileAssociation
|
|
2599
|
+
).where(
|
|
2600
|
+
and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
|
|
2601
|
+
models.RSEFileAssociation.scope == replica['scope'],
|
|
2602
|
+
models.RSEFileAssociation.name == replica['name'])
|
|
2603
|
+
).values(
|
|
2604
|
+
values
|
|
2605
|
+
).execution_options(
|
|
2606
|
+
synchronize_session=False
|
|
2607
|
+
)
|
|
2608
|
+
|
|
2609
|
+
if not session.execute(update_stmt).rowcount:
|
|
2377
2610
|
if 'rse' not in replica:
|
|
2378
2611
|
replica['rse'] = get_rse_name(rse_id=replica['rse_id'], session=session)
|
|
2379
2612
|
raise exception.UnsupportedOperation('State %(state)s for replica %(scope)s:%(name)s on %(rse)s cannot be updated' % replica)
|
|
@@ -2393,32 +2626,68 @@ def touch_replica(replica, *, session: "Session"):
|
|
|
2393
2626
|
try:
|
|
2394
2627
|
accessed_at, none_value = replica.get('accessed_at') or datetime.utcnow(), None
|
|
2395
2628
|
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2629
|
+
stmt = select(
|
|
2630
|
+
models.RSEFileAssociation
|
|
2631
|
+
).with_hint(
|
|
2632
|
+
models.RSEFileAssociation,
|
|
2633
|
+
'INDEX(REPLICAS REPLICAS_PK)',
|
|
2634
|
+
'oracle'
|
|
2635
|
+
).where(
|
|
2636
|
+
and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
|
|
2637
|
+
models.RSEFileAssociation.scope == replica['scope'],
|
|
2638
|
+
models.RSEFileAssociation.name == replica['name'])
|
|
2639
|
+
).with_for_update(
|
|
2640
|
+
nowait=True
|
|
2641
|
+
)
|
|
2642
|
+
session.execute(stmt).one()
|
|
2643
|
+
|
|
2644
|
+
stmt = update(
|
|
2645
|
+
models.RSEFileAssociation
|
|
2646
|
+
).where(
|
|
2647
|
+
and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
|
|
2648
|
+
models.RSEFileAssociation.scope == replica['scope'],
|
|
2649
|
+
models.RSEFileAssociation.name == replica['name'])
|
|
2650
|
+
).prefix_with(
|
|
2651
|
+
'/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'
|
|
2652
|
+
).values({
|
|
2653
|
+
models.RSEFileAssociation.accessed_at: accessed_at,
|
|
2654
|
+
models.RSEFileAssociation.tombstone: case(
|
|
2655
|
+
(models.RSEFileAssociation.tombstone.not_in([OBSOLETE, none_value]),
|
|
2656
|
+
accessed_at),
|
|
2657
|
+
else_=models.RSEFileAssociation.tombstone)
|
|
2658
|
+
}).execution_options(
|
|
2659
|
+
synchronize_session=False
|
|
2660
|
+
)
|
|
2410
2661
|
session.execute(stmt)
|
|
2411
2662
|
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2663
|
+
stmt = select(
|
|
2664
|
+
models.DataIdentifier
|
|
2665
|
+
).with_hint(
|
|
2666
|
+
models.DataIdentifier,
|
|
2667
|
+
'INDEX(DIDS DIDS_PK)',
|
|
2668
|
+
'oracle'
|
|
2669
|
+
).where(
|
|
2670
|
+
and_(models.DataIdentifier.scope == replica['scope'],
|
|
2671
|
+
models.DataIdentifier.name == replica['name'],
|
|
2672
|
+
models.DataIdentifier.did_type == DIDType.FILE)
|
|
2673
|
+
).with_for_update(
|
|
2674
|
+
nowait=True
|
|
2675
|
+
)
|
|
2676
|
+
session.execute(stmt).one()
|
|
2416
2677
|
|
|
2417
|
-
stmt = update(
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2678
|
+
stmt = update(
|
|
2679
|
+
models.DataIdentifier
|
|
2680
|
+
).where(
|
|
2681
|
+
and_(models.DataIdentifier.scope == replica['scope'],
|
|
2682
|
+
models.DataIdentifier.name == replica['name'],
|
|
2683
|
+
models.DataIdentifier.did_type == DIDType.FILE)
|
|
2684
|
+
).prefix_with(
|
|
2685
|
+
'/*+ INDEX(DIDS DIDS_PK) */', dialect='oracle'
|
|
2686
|
+
).values({
|
|
2687
|
+
models.DataIdentifier.accessed_at: accessed_at
|
|
2688
|
+
}).execution_options(
|
|
2689
|
+
synchronize_session=False
|
|
2690
|
+
)
|
|
2422
2691
|
session.execute(stmt)
|
|
2423
2692
|
|
|
2424
2693
|
except DatabaseError:
|
|
@@ -2456,21 +2725,27 @@ def get_and_lock_file_replicas(scope, name, nowait=False, restrict_rses=None, *,
|
|
|
2456
2725
|
:returns: List of SQLAlchemy Replica Objects
|
|
2457
2726
|
"""
|
|
2458
2727
|
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2728
|
+
stmt = select(
|
|
2729
|
+
models.RSEFileAssociation
|
|
2730
|
+
).where(
|
|
2731
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2732
|
+
models.RSEFileAssociation.name == name,
|
|
2733
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
|
|
2734
|
+
).with_for_update(
|
|
2735
|
+
nowait=nowait
|
|
2736
|
+
)
|
|
2737
|
+
if restrict_rses is not None and len(restrict_rses) < 10:
|
|
2738
|
+
rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
|
|
2739
|
+
if rse_clause:
|
|
2740
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
2741
|
+
|
|
2742
|
+
return session.execute(stmt).scalars().all()
|
|
2468
2743
|
|
|
2469
2744
|
|
|
2470
2745
|
@transactional_session
|
|
2471
2746
|
def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
|
|
2472
2747
|
"""
|
|
2473
|
-
Get
|
|
2748
|
+
Get source replicas for a specific scope:name.
|
|
2474
2749
|
|
|
2475
2750
|
:param scope: The scope of the did.
|
|
2476
2751
|
:param name: The name of the did.
|
|
@@ -2479,15 +2754,21 @@ def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
|
|
|
2479
2754
|
:returns: List of SQLAlchemy Replica Objects
|
|
2480
2755
|
"""
|
|
2481
2756
|
|
|
2482
|
-
|
|
2757
|
+
stmt = select(
|
|
2758
|
+
models.RSEFileAssociation.rse_id
|
|
2759
|
+
).where(
|
|
2760
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
2761
|
+
models.RSEFileAssociation.name == name,
|
|
2762
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
|
|
2763
|
+
)
|
|
2483
2764
|
if source_rses:
|
|
2484
2765
|
if len(source_rses) < 10:
|
|
2485
2766
|
rse_clause = []
|
|
2486
2767
|
for rse_id in source_rses:
|
|
2487
2768
|
rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
|
|
2488
2769
|
if rse_clause:
|
|
2489
|
-
|
|
2490
|
-
return
|
|
2770
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
2771
|
+
return session.execute(stmt).scalars().all()
|
|
2491
2772
|
|
|
2492
2773
|
|
|
2493
2774
|
@transactional_session
|
|
@@ -2507,24 +2788,39 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
|
|
|
2507
2788
|
:returns: (files in dataset, replicas in dataset)
|
|
2508
2789
|
"""
|
|
2509
2790
|
files, replicas = {}, {}
|
|
2510
|
-
if session.bind.dialect.name == 'postgresql':
|
|
2511
|
-
# Get content
|
|
2512
|
-
content_query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
2513
|
-
models.DataIdentifierAssociation.child_name,
|
|
2514
|
-
models.DataIdentifierAssociation.bytes,
|
|
2515
|
-
models.DataIdentifierAssociation.md5,
|
|
2516
|
-
models.DataIdentifierAssociation.adler32).\
|
|
2517
|
-
with_hint(models.DataIdentifierAssociation,
|
|
2518
|
-
"INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
|
|
2519
|
-
'oracle').\
|
|
2520
|
-
filter(models.DataIdentifierAssociation.scope == scope,
|
|
2521
|
-
models.DataIdentifierAssociation.name == name)
|
|
2522
2791
|
|
|
2792
|
+
base_stmt = select(
|
|
2793
|
+
models.DataIdentifierAssociation.child_scope,
|
|
2794
|
+
models.DataIdentifierAssociation.child_name,
|
|
2795
|
+
models.DataIdentifierAssociation.bytes,
|
|
2796
|
+
models.DataIdentifierAssociation.md5,
|
|
2797
|
+
models.DataIdentifierAssociation.adler32,
|
|
2798
|
+
).where(
|
|
2799
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
2800
|
+
models.DataIdentifierAssociation.name == name)
|
|
2801
|
+
)
|
|
2802
|
+
|
|
2803
|
+
stmt = base_stmt.add_columns(
|
|
2804
|
+
models.RSEFileAssociation
|
|
2805
|
+
).where(
|
|
2806
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2807
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2808
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
|
|
2809
|
+
)
|
|
2810
|
+
|
|
2811
|
+
rse_clause = [true()]
|
|
2812
|
+
if restrict_rses is not None and len(restrict_rses) < 10:
|
|
2813
|
+
rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
|
|
2814
|
+
|
|
2815
|
+
if session.bind.dialect.name == 'postgresql':
|
|
2523
2816
|
if total_threads and total_threads > 1:
|
|
2524
|
-
|
|
2525
|
-
|
|
2817
|
+
base_stmt = filter_thread_work(session=session,
|
|
2818
|
+
query=base_stmt,
|
|
2819
|
+
total_threads=total_threads,
|
|
2820
|
+
thread_id=thread_id,
|
|
2821
|
+
hash_variable='child_name')
|
|
2526
2822
|
|
|
2527
|
-
for child_scope, child_name, bytes_, md5, adler32 in
|
|
2823
|
+
for child_scope, child_name, bytes_, md5, adler32 in session.execute(base_stmt).yield_per(1000):
|
|
2528
2824
|
files[(child_scope, child_name)] = {'scope': child_scope,
|
|
2529
2825
|
'name': child_name,
|
|
2530
2826
|
'bytes': bytes_,
|
|
@@ -2532,92 +2828,35 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
|
|
|
2532
2828
|
'adler32': adler32}
|
|
2533
2829
|
replicas[(child_scope, child_name)] = []
|
|
2534
2830
|
|
|
2535
|
-
|
|
2536
|
-
query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
2537
|
-
models.DataIdentifierAssociation.child_name,
|
|
2538
|
-
models.DataIdentifierAssociation.bytes,
|
|
2539
|
-
models.DataIdentifierAssociation.md5,
|
|
2540
|
-
models.DataIdentifierAssociation.adler32,
|
|
2541
|
-
models.RSEFileAssociation)\
|
|
2542
|
-
.with_hint(models.DataIdentifierAssociation,
|
|
2543
|
-
"INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
|
|
2544
|
-
'oracle')\
|
|
2545
|
-
.filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2546
|
-
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2547
|
-
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED))\
|
|
2548
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
2549
|
-
models.DataIdentifierAssociation.name == name)
|
|
2550
|
-
|
|
2551
|
-
if restrict_rses is not None:
|
|
2552
|
-
if len(restrict_rses) < 10:
|
|
2553
|
-
rse_clause = []
|
|
2554
|
-
for rse_id in restrict_rses:
|
|
2555
|
-
rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
|
|
2556
|
-
if rse_clause:
|
|
2557
|
-
query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
2558
|
-
models.DataIdentifierAssociation.child_name,
|
|
2559
|
-
models.DataIdentifierAssociation.bytes,
|
|
2560
|
-
models.DataIdentifierAssociation.md5,
|
|
2561
|
-
models.DataIdentifierAssociation.adler32,
|
|
2562
|
-
models.RSEFileAssociation)\
|
|
2563
|
-
.with_hint(models.DataIdentifierAssociation,
|
|
2564
|
-
"INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
|
|
2565
|
-
'oracle')\
|
|
2566
|
-
.filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2567
|
-
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2568
|
-
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
|
|
2569
|
-
or_(*rse_clause)))\
|
|
2570
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
2571
|
-
models.DataIdentifierAssociation.name == name)
|
|
2572
|
-
|
|
2831
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
2573
2832
|
else:
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)).\
|
|
2588
|
-
filter(models.DataIdentifierAssociation.scope == scope,
|
|
2589
|
-
models.DataIdentifierAssociation.name == name)
|
|
2590
|
-
|
|
2591
|
-
if restrict_rses is not None:
|
|
2592
|
-
if len(restrict_rses) < 10:
|
|
2593
|
-
rse_clause = []
|
|
2594
|
-
for rse_id in restrict_rses:
|
|
2595
|
-
rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
|
|
2596
|
-
if rse_clause:
|
|
2597
|
-
query = session.query(models.DataIdentifierAssociation.child_scope,
|
|
2598
|
-
models.DataIdentifierAssociation.child_name,
|
|
2599
|
-
models.DataIdentifierAssociation.bytes,
|
|
2600
|
-
models.DataIdentifierAssociation.md5,
|
|
2601
|
-
models.DataIdentifierAssociation.adler32,
|
|
2602
|
-
models.RSEFileAssociation)\
|
|
2603
|
-
.with_hint(models.DataIdentifierAssociation,
|
|
2604
|
-
"INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
|
|
2605
|
-
'oracle')\
|
|
2606
|
-
.outerjoin(models.RSEFileAssociation,
|
|
2607
|
-
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2608
|
-
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2609
|
-
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
|
|
2610
|
-
or_(*rse_clause)))\
|
|
2611
|
-
.filter(models.DataIdentifierAssociation.scope == scope,
|
|
2612
|
-
models.DataIdentifierAssociation.name == name)
|
|
2833
|
+
stmt = base_stmt.add_columns(
|
|
2834
|
+
models.RSEFileAssociation
|
|
2835
|
+
).with_hint(
|
|
2836
|
+
models.DataIdentifierAssociation,
|
|
2837
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
2838
|
+
'oracle'
|
|
2839
|
+
).outerjoin(
|
|
2840
|
+
models.RSEFileAssociation,
|
|
2841
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2842
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2843
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
|
|
2844
|
+
or_(*rse_clause))
|
|
2845
|
+
)
|
|
2613
2846
|
|
|
2614
2847
|
if total_threads and total_threads > 1:
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2848
|
+
stmt = filter_thread_work(session=session,
|
|
2849
|
+
query=stmt,
|
|
2850
|
+
total_threads=total_threads,
|
|
2851
|
+
thread_id=thread_id,
|
|
2852
|
+
hash_variable='child_name')
|
|
2853
|
+
|
|
2854
|
+
stmt = stmt.with_for_update(
|
|
2855
|
+
nowait=nowait,
|
|
2856
|
+
of=models.RSEFileAssociation.lock_cnt
|
|
2857
|
+
)
|
|
2619
2858
|
|
|
2620
|
-
for child_scope, child_name, bytes_, md5, adler32, replica in
|
|
2859
|
+
for child_scope, child_name, bytes_, md5, adler32, replica in session.execute(stmt).yield_per(1000):
|
|
2621
2860
|
if (child_scope, child_name) not in files:
|
|
2622
2861
|
files[(child_scope, child_name)] = {'scope': child_scope,
|
|
2623
2862
|
'name': child_name,
|
|
@@ -2651,15 +2890,23 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
|
|
|
2651
2890
|
:param session: The db session in use.
|
|
2652
2891
|
:returns: (files in dataset, replicas in dataset)
|
|
2653
2892
|
"""
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2893
|
+
stmt = select(
|
|
2894
|
+
models.DataIdentifierAssociation.child_scope,
|
|
2895
|
+
models.DataIdentifierAssociation.child_name,
|
|
2896
|
+
models.RSEFileAssociation.rse_id
|
|
2897
|
+
).with_hint(
|
|
2898
|
+
models.DataIdentifierAssociation,
|
|
2899
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
2900
|
+
'oracle'
|
|
2901
|
+
).outerjoin(
|
|
2902
|
+
models.RSEFileAssociation,
|
|
2903
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2904
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2905
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
|
|
2906
|
+
).where(
|
|
2907
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
2908
|
+
models.DataIdentifierAssociation.name == name)
|
|
2909
|
+
)
|
|
2663
2910
|
|
|
2664
2911
|
if source_rses:
|
|
2665
2912
|
if len(source_rses) < 10:
|
|
@@ -2667,25 +2914,34 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
|
|
|
2667
2914
|
for rse_id in source_rses:
|
|
2668
2915
|
rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
|
|
2669
2916
|
if rse_clause:
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2917
|
+
stmt = select(
|
|
2918
|
+
models.DataIdentifierAssociation.child_scope,
|
|
2919
|
+
models.DataIdentifierAssociation.child_name,
|
|
2920
|
+
models.RSEFileAssociation.rse_id
|
|
2921
|
+
).with_hint(
|
|
2922
|
+
models.DataIdentifierAssociation,
|
|
2923
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
2924
|
+
'oracle'
|
|
2925
|
+
).outerjoin(
|
|
2926
|
+
models.RSEFileAssociation,
|
|
2927
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
2928
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
2929
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
2930
|
+
or_(*rse_clause))
|
|
2931
|
+
).where(
|
|
2932
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
2933
|
+
models.DataIdentifierAssociation.name == name)
|
|
2934
|
+
)
|
|
2682
2935
|
if total_threads and total_threads > 1:
|
|
2683
|
-
|
|
2684
|
-
|
|
2936
|
+
stmt = filter_thread_work(session=session,
|
|
2937
|
+
query=stmt,
|
|
2938
|
+
total_threads=total_threads,
|
|
2939
|
+
thread_id=thread_id,
|
|
2940
|
+
hash_variable='child_name')
|
|
2685
2941
|
|
|
2686
2942
|
replicas = {}
|
|
2687
2943
|
|
|
2688
|
-
for child_scope, child_name, rse_id in
|
|
2944
|
+
for child_scope, child_name, rse_id in session.execute(stmt):
|
|
2689
2945
|
|
|
2690
2946
|
if (child_scope, child_name) in replicas:
|
|
2691
2947
|
if rse_id:
|
|
@@ -2707,8 +2963,18 @@ def get_replica_atime(replica, *, session: "Session"):
|
|
|
2707
2963
|
|
|
2708
2964
|
:returns: A datetime timestamp with the last access time.
|
|
2709
2965
|
"""
|
|
2710
|
-
|
|
2711
|
-
|
|
2966
|
+
stmt = select(
|
|
2967
|
+
models.RSEFileAssociation.accessed_at
|
|
2968
|
+
).with_hint(
|
|
2969
|
+
models.RSEFileAssociation,
|
|
2970
|
+
'INDEX(REPLICAS REPLICAS_PK)',
|
|
2971
|
+
'oracle'
|
|
2972
|
+
).where(
|
|
2973
|
+
and_(models.RSEFileAssociation.scope == replica['scope'],
|
|
2974
|
+
models.RSEFileAssociation.name == replica['name'],
|
|
2975
|
+
models.RSEFileAssociation.rse_id == replica['rse_id'])
|
|
2976
|
+
)
|
|
2977
|
+
return session.execute(stmt).scalar_one()
|
|
2712
2978
|
|
|
2713
2979
|
|
|
2714
2980
|
@transactional_session
|
|
@@ -2725,8 +2991,18 @@ def touch_collection_replicas(collection_replicas, *, session: "Session"):
|
|
|
2725
2991
|
now = datetime.utcnow()
|
|
2726
2992
|
for collection_replica in collection_replicas:
|
|
2727
2993
|
try:
|
|
2728
|
-
|
|
2729
|
-
|
|
2994
|
+
stmt = update(
|
|
2995
|
+
models.CollectionReplica
|
|
2996
|
+
).where(
|
|
2997
|
+
and_(models.CollectionReplica.scope == collection_replica['scope'],
|
|
2998
|
+
models.CollectionReplica.name == collection_replica['name'],
|
|
2999
|
+
models.CollectionReplica.rse_id == collection_replica['rse_id'])
|
|
3000
|
+
).values({
|
|
3001
|
+
models.CollectionReplica.accessed_at: collection_replica.get('accessed_at') or now
|
|
3002
|
+
}).execution_options(
|
|
3003
|
+
synchronize_session=False
|
|
3004
|
+
)
|
|
3005
|
+
session.execute(stmt)
|
|
2730
3006
|
except DatabaseError:
|
|
2731
3007
|
return False
|
|
2732
3008
|
|
|
@@ -2746,138 +3022,158 @@ def list_dataset_replicas(scope, name, deep=False, *, session: "Session"):
|
|
|
2746
3022
|
"""
|
|
2747
3023
|
|
|
2748
3024
|
if not deep:
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2761
|
-
.
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
3025
|
+
stmt = select(
|
|
3026
|
+
models.CollectionReplica.scope,
|
|
3027
|
+
models.CollectionReplica.name,
|
|
3028
|
+
models.RSE.rse,
|
|
3029
|
+
models.CollectionReplica.rse_id,
|
|
3030
|
+
models.CollectionReplica.bytes,
|
|
3031
|
+
models.CollectionReplica.length,
|
|
3032
|
+
models.CollectionReplica.available_bytes,
|
|
3033
|
+
models.CollectionReplica.available_replicas_cnt.label("available_length"),
|
|
3034
|
+
models.CollectionReplica.state,
|
|
3035
|
+
models.CollectionReplica.created_at,
|
|
3036
|
+
models.CollectionReplica.updated_at,
|
|
3037
|
+
models.CollectionReplica.accessed_at
|
|
3038
|
+
).where(
|
|
3039
|
+
and_(models.CollectionReplica.scope == scope,
|
|
3040
|
+
models.CollectionReplica.name == name,
|
|
3041
|
+
models.CollectionReplica.did_type == DIDType.DATASET,
|
|
3042
|
+
models.CollectionReplica.rse_id == models.RSE.id,
|
|
3043
|
+
models.RSE.deleted == false())
|
|
3044
|
+
)
|
|
3045
|
+
|
|
3046
|
+
for row in session.execute(stmt).all():
|
|
2766
3047
|
yield row._asdict()
|
|
2767
3048
|
|
|
2768
3049
|
else:
|
|
3050
|
+
# Find maximum values
|
|
3051
|
+
stmt = select(
|
|
3052
|
+
func.sum(models.DataIdentifierAssociation.bytes).label("bytes"),
|
|
3053
|
+
func.count().label("length")
|
|
3054
|
+
).select_from(
|
|
3055
|
+
models.DataIdentifierAssociation
|
|
3056
|
+
).with_hint(
|
|
3057
|
+
models.DataIdentifierAssociation,
|
|
3058
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
3059
|
+
'oracle'
|
|
3060
|
+
).where(
|
|
3061
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
3062
|
+
models.DataIdentifierAssociation.name == name)
|
|
3063
|
+
)
|
|
3064
|
+
|
|
3065
|
+
bytes_, length = session.execute(stmt).one()
|
|
3066
|
+
bytes_ = bytes_ or 0
|
|
3067
|
+
|
|
3068
|
+
# Find archives that contain files of the requested dataset
|
|
3069
|
+
sub_query_stmt = select(
|
|
3070
|
+
models.DataIdentifierAssociation.scope.label('dataset_scope'),
|
|
3071
|
+
models.DataIdentifierAssociation.name.label('dataset_name'),
|
|
3072
|
+
models.DataIdentifierAssociation.bytes.label('file_bytes'),
|
|
3073
|
+
models.ConstituentAssociation.child_scope.label('file_scope'),
|
|
3074
|
+
models.ConstituentAssociation.child_name.label('file_name'),
|
|
3075
|
+
models.RSEFileAssociation.scope.label('replica_scope'),
|
|
3076
|
+
models.RSEFileAssociation.name.label('replica_name'),
|
|
3077
|
+
models.RSE.rse,
|
|
3078
|
+
models.RSE.id.label('rse_id'),
|
|
3079
|
+
models.RSEFileAssociation.created_at,
|
|
3080
|
+
models.RSEFileAssociation.accessed_at,
|
|
3081
|
+
models.RSEFileAssociation.updated_at
|
|
3082
|
+
).where(
|
|
3083
|
+
and_(models.DataIdentifierAssociation.scope == scope,
|
|
3084
|
+
models.DataIdentifierAssociation.name == name,
|
|
3085
|
+
models.ConstituentAssociation.child_scope == models.DataIdentifierAssociation.child_scope,
|
|
3086
|
+
models.ConstituentAssociation.child_name == models.DataIdentifierAssociation.child_name,
|
|
3087
|
+
models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
|
|
3088
|
+
models.ConstituentAssociation.name == models.RSEFileAssociation.name,
|
|
3089
|
+
models.RSEFileAssociation.rse_id == models.RSE.id,
|
|
3090
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
3091
|
+
models.RSE.deleted == false())
|
|
3092
|
+
).subquery()
|
|
3093
|
+
|
|
3094
|
+
# Count the metrics
|
|
3095
|
+
group_query_stmt = select(
|
|
3096
|
+
sub_query_stmt.c.dataset_scope,
|
|
3097
|
+
sub_query_stmt.c.dataset_name,
|
|
3098
|
+
sub_query_stmt.c.file_scope,
|
|
3099
|
+
sub_query_stmt.c.file_name,
|
|
3100
|
+
sub_query_stmt.c.rse_id,
|
|
3101
|
+
sub_query_stmt.c.rse,
|
|
3102
|
+
func.sum(sub_query_stmt.c.file_bytes).label('file_bytes'),
|
|
3103
|
+
func.min(sub_query_stmt.c.created_at).label('created_at'),
|
|
3104
|
+
func.max(sub_query_stmt.c.updated_at).label('updated_at'),
|
|
3105
|
+
func.max(sub_query_stmt.c.accessed_at).label('accessed_at')
|
|
3106
|
+
).group_by(
|
|
3107
|
+
sub_query_stmt.c.dataset_scope,
|
|
3108
|
+
sub_query_stmt.c.dataset_name,
|
|
3109
|
+
sub_query_stmt.c.file_scope,
|
|
3110
|
+
sub_query_stmt.c.file_name,
|
|
3111
|
+
sub_query_stmt.c.rse_id,
|
|
3112
|
+
sub_query_stmt.c.rse
|
|
3113
|
+
).subquery()
|
|
3114
|
+
|
|
3115
|
+
# Bring it in the same column state as the non-archive query
|
|
3116
|
+
full_query_stmt = select(
|
|
3117
|
+
group_query_stmt.c.dataset_scope.label('scope'),
|
|
3118
|
+
group_query_stmt.c.dataset_name.label('name'),
|
|
3119
|
+
group_query_stmt.c.rse_id,
|
|
3120
|
+
group_query_stmt.c.rse,
|
|
3121
|
+
func.sum(group_query_stmt.c.file_bytes).label('available_bytes'),
|
|
3122
|
+
func.count().label('available_length'),
|
|
3123
|
+
func.min(group_query_stmt.c.created_at).label('created_at'),
|
|
3124
|
+
func.max(group_query_stmt.c.updated_at).label('updated_at'),
|
|
3125
|
+
func.max(group_query_stmt.c.accessed_at).label('accessed_at')
|
|
3126
|
+
).group_by(
|
|
3127
|
+
group_query_stmt.c.dataset_scope,
|
|
3128
|
+
group_query_stmt.c.dataset_name,
|
|
3129
|
+
group_query_stmt.c.rse_id,
|
|
3130
|
+
group_query_stmt.c.rse
|
|
3131
|
+
)
|
|
3132
|
+
|
|
3133
|
+
# Find the non-archive dataset replicas
|
|
3134
|
+
sub_query_stmt = select(
|
|
3135
|
+
models.DataIdentifierAssociation.scope,
|
|
3136
|
+
models.DataIdentifierAssociation.name,
|
|
3137
|
+
models.RSEFileAssociation.rse_id,
|
|
3138
|
+
func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
|
|
3139
|
+
func.count().label("available_length"),
|
|
3140
|
+
func.min(models.RSEFileAssociation.created_at).label("created_at"),
|
|
3141
|
+
func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
|
|
3142
|
+
func.max(models.RSEFileAssociation.accessed_at).label("accessed_at")
|
|
3143
|
+
).with_hint(
|
|
3144
|
+
models.DataIdentifierAssociation,
|
|
3145
|
+
'INDEX_RS_ASC(CONTENTS CONTENTS_PK) INDEX_RS_ASC(REPLICAS REPLICAS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
|
|
3146
|
+
'oracle'
|
|
3147
|
+
).where(
|
|
3148
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
3149
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
3150
|
+
models.DataIdentifierAssociation.scope == scope,
|
|
3151
|
+
models.DataIdentifierAssociation.name == name,
|
|
3152
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
|
|
3153
|
+
).group_by(
|
|
3154
|
+
models.DataIdentifierAssociation.scope,
|
|
3155
|
+
models.DataIdentifierAssociation.name,
|
|
3156
|
+
models.RSEFileAssociation.rse_id
|
|
3157
|
+
).subquery()
|
|
2769
3158
|
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
.
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
.
|
|
2776
|
-
.
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
models.ConstituentAssociation.child_name.label('file_name'),
|
|
2789
|
-
models.RSEFileAssociation.scope.label('replica_scope'),
|
|
2790
|
-
models.RSEFileAssociation.name.label('replica_name'),
|
|
2791
|
-
models.RSE.rse,
|
|
2792
|
-
models.RSE.id.label('rse_id'),
|
|
2793
|
-
models.RSEFileAssociation.created_at,
|
|
2794
|
-
models.RSEFileAssociation.accessed_at,
|
|
2795
|
-
models.RSEFileAssociation.updated_at)\
|
|
2796
|
-
.filter(models.DataIdentifierAssociation.scope == scope)\
|
|
2797
|
-
.filter(models.DataIdentifierAssociation.name == name)\
|
|
2798
|
-
.filter(models.ConstituentAssociation.child_scope == models.DataIdentifierAssociation.child_scope)\
|
|
2799
|
-
.filter(models.ConstituentAssociation.child_name == models.DataIdentifierAssociation.child_name)\
|
|
2800
|
-
.filter(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope)\
|
|
2801
|
-
.filter(models.ConstituentAssociation.name == models.RSEFileAssociation.name)\
|
|
2802
|
-
.filter(models.RSEFileAssociation.rse_id == models.RSE.id)\
|
|
2803
|
-
.filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
|
|
2804
|
-
.filter(models.RSE.deleted == false())\
|
|
2805
|
-
.subquery()
|
|
2806
|
-
|
|
2807
|
-
# count the metrics
|
|
2808
|
-
group_query_archives = session\
|
|
2809
|
-
.query(sub_query_archives.c.dataset_scope,
|
|
2810
|
-
sub_query_archives.c.dataset_name,
|
|
2811
|
-
sub_query_archives.c.file_scope,
|
|
2812
|
-
sub_query_archives.c.file_name,
|
|
2813
|
-
sub_query_archives.c.rse_id,
|
|
2814
|
-
sub_query_archives.c.rse,
|
|
2815
|
-
func.sum(sub_query_archives.c.file_bytes).label('file_bytes'),
|
|
2816
|
-
func.min(sub_query_archives.c.created_at).label('created_at'),
|
|
2817
|
-
func.max(sub_query_archives.c.updated_at).label('updated_at'),
|
|
2818
|
-
func.max(sub_query_archives.c.accessed_at).label('accessed_at'))\
|
|
2819
|
-
.group_by(sub_query_archives.c.dataset_scope,
|
|
2820
|
-
sub_query_archives.c.dataset_name,
|
|
2821
|
-
sub_query_archives.c.file_scope,
|
|
2822
|
-
sub_query_archives.c.file_name,
|
|
2823
|
-
sub_query_archives.c.rse_id,
|
|
2824
|
-
sub_query_archives.c.rse)\
|
|
2825
|
-
.subquery()
|
|
2826
|
-
|
|
2827
|
-
# bring it in the same column state as the non-archive query
|
|
2828
|
-
full_query_archives = session\
|
|
2829
|
-
.query(group_query_archives.c.dataset_scope.label('scope'),
|
|
2830
|
-
group_query_archives.c.dataset_name.label('name'),
|
|
2831
|
-
group_query_archives.c.rse_id,
|
|
2832
|
-
group_query_archives.c.rse,
|
|
2833
|
-
func.sum(group_query_archives.c.file_bytes).label('available_bytes'),
|
|
2834
|
-
func.count().label('available_length'),
|
|
2835
|
-
func.min(group_query_archives.c.created_at).label('created_at'),
|
|
2836
|
-
func.max(group_query_archives.c.updated_at).label('updated_at'),
|
|
2837
|
-
func.max(group_query_archives.c.accessed_at).label('accessed_at'))\
|
|
2838
|
-
.group_by(group_query_archives.c.dataset_scope,
|
|
2839
|
-
group_query_archives.c.dataset_name,
|
|
2840
|
-
group_query_archives.c.rse_id,
|
|
2841
|
-
group_query_archives.c.rse)
|
|
2842
|
-
|
|
2843
|
-
# find the non-archive dataset replicas
|
|
2844
|
-
sub_query = session\
|
|
2845
|
-
.query(models.DataIdentifierAssociation.scope,
|
|
2846
|
-
models.DataIdentifierAssociation.name,
|
|
2847
|
-
models.RSEFileAssociation.rse_id,
|
|
2848
|
-
func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
|
|
2849
|
-
func.count().label("available_length"),
|
|
2850
|
-
func.min(models.RSEFileAssociation.created_at).label("created_at"),
|
|
2851
|
-
func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
|
|
2852
|
-
func.max(models.RSEFileAssociation.accessed_at).label("accessed_at"))\
|
|
2853
|
-
.with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) INDEX_RS_ASC(REPLICAS REPLICAS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
|
|
2854
|
-
.filter(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope)\
|
|
2855
|
-
.filter(models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name)\
|
|
2856
|
-
.filter(models.DataIdentifierAssociation.scope == scope)\
|
|
2857
|
-
.filter(models.DataIdentifierAssociation.name == name)\
|
|
2858
|
-
.filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
|
|
2859
|
-
.group_by(models.DataIdentifierAssociation.scope,
|
|
2860
|
-
models.DataIdentifierAssociation.name,
|
|
2861
|
-
models.RSEFileAssociation.rse_id)\
|
|
2862
|
-
.subquery()
|
|
2863
|
-
|
|
2864
|
-
query = session\
|
|
2865
|
-
.query(sub_query.c.scope,
|
|
2866
|
-
sub_query.c.name,
|
|
2867
|
-
sub_query.c.rse_id,
|
|
2868
|
-
models.RSE.rse,
|
|
2869
|
-
sub_query.c.available_bytes,
|
|
2870
|
-
sub_query.c.available_length,
|
|
2871
|
-
sub_query.c.created_at,
|
|
2872
|
-
sub_query.c.updated_at,
|
|
2873
|
-
sub_query.c.accessed_at)\
|
|
2874
|
-
.filter(models.RSE.id == sub_query.c.rse_id)\
|
|
2875
|
-
.filter(models.RSE.deleted == false())
|
|
2876
|
-
|
|
2877
|
-
# join everything together
|
|
2878
|
-
final_query = query.union_all(full_query_archives)
|
|
2879
|
-
|
|
2880
|
-
for row in final_query.all():
|
|
3159
|
+
stmt = select(
|
|
3160
|
+
sub_query_stmt.c.scope,
|
|
3161
|
+
sub_query_stmt.c.name,
|
|
3162
|
+
sub_query_stmt.c.rse_id,
|
|
3163
|
+
models.RSE.rse,
|
|
3164
|
+
sub_query_stmt.c.available_bytes,
|
|
3165
|
+
sub_query_stmt.c.available_length,
|
|
3166
|
+
sub_query_stmt.c.created_at,
|
|
3167
|
+
sub_query_stmt.c.updated_at,
|
|
3168
|
+
sub_query_stmt.c.accessed_at
|
|
3169
|
+
).where(
|
|
3170
|
+
and_(sub_query_stmt.c.rse_id == models.RSE.id,
|
|
3171
|
+
models.RSE.deleted == false())
|
|
3172
|
+
)
|
|
3173
|
+
|
|
3174
|
+
# Join everything together
|
|
3175
|
+
final_stmt = stmt.union_all(full_query_stmt)
|
|
3176
|
+
for row in session.execute(final_stmt).all():
|
|
2881
3177
|
replica = row._asdict()
|
|
2882
3178
|
replica['length'], replica['bytes'] = length, bytes_
|
|
2883
3179
|
if replica['length'] == row.available_length:
|
|
@@ -2905,23 +3201,27 @@ def list_dataset_replicas_bulk(names_by_intscope, *, session: "Session"):
|
|
|
2905
3201
|
try:
|
|
2906
3202
|
# chunk size refers to the number of different scopes, see above
|
|
2907
3203
|
for chunk in chunks(condition, 10):
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
.
|
|
2923
|
-
|
|
2924
|
-
|
|
3204
|
+
stmt = select(
|
|
3205
|
+
models.CollectionReplica.scope,
|
|
3206
|
+
models.CollectionReplica.name,
|
|
3207
|
+
models.RSE.rse,
|
|
3208
|
+
models.CollectionReplica.rse_id,
|
|
3209
|
+
models.CollectionReplica.bytes,
|
|
3210
|
+
models.CollectionReplica.length,
|
|
3211
|
+
models.CollectionReplica.available_bytes,
|
|
3212
|
+
models.CollectionReplica.available_replicas_cnt.label("available_length"),
|
|
3213
|
+
models.CollectionReplica.state,
|
|
3214
|
+
models.CollectionReplica.created_at,
|
|
3215
|
+
models.CollectionReplica.updated_at,
|
|
3216
|
+
models.CollectionReplica.accessed_at
|
|
3217
|
+
).where(
|
|
3218
|
+
and_(models.CollectionReplica.did_type == DIDType.DATASET,
|
|
3219
|
+
models.CollectionReplica.rse_id == models.RSE.id,
|
|
3220
|
+
models.RSE.deleted == false(),
|
|
3221
|
+
or_(*chunk))
|
|
3222
|
+
)
|
|
3223
|
+
|
|
3224
|
+
for row in session.execute(stmt).all():
|
|
2925
3225
|
yield row._asdict()
|
|
2926
3226
|
except NoResultFound:
|
|
2927
3227
|
raise exception.DataIdentifierNotFound('No Data Identifiers found')
|
|
@@ -2995,50 +3295,74 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
|
|
|
2995
3295
|
|
|
2996
3296
|
:returns: A list of dict dataset replicas
|
|
2997
3297
|
"""
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
.
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3298
|
+
stmt = select(
|
|
3299
|
+
models.CollectionReplica.scope,
|
|
3300
|
+
models.CollectionReplica.name,
|
|
3301
|
+
models.RSE.id.label('rse_id'),
|
|
3302
|
+
models.RSE.rse,
|
|
3303
|
+
models.CollectionReplica.bytes,
|
|
3304
|
+
models.CollectionReplica.length,
|
|
3305
|
+
models.CollectionReplica.available_bytes,
|
|
3306
|
+
models.CollectionReplica.available_replicas_cnt.label("available_length"),
|
|
3307
|
+
models.CollectionReplica.state,
|
|
3308
|
+
models.CollectionReplica.created_at,
|
|
3309
|
+
models.CollectionReplica.updated_at,
|
|
3310
|
+
models.CollectionReplica.accessed_at
|
|
3311
|
+
).where(
|
|
3312
|
+
and_(models.CollectionReplica.did_type == DIDType.DATASET,
|
|
3313
|
+
models.CollectionReplica.rse_id == models.RSE.id,
|
|
3314
|
+
models.RSE.deleted == false(),
|
|
3315
|
+
models.RSE.id == rse_id)
|
|
3316
|
+
)
|
|
3014
3317
|
|
|
3015
3318
|
for (k, v) in filters and filters.items() or []:
|
|
3016
3319
|
if k == 'name' or k == 'scope':
|
|
3017
3320
|
v_str = v if k != 'scope' else v.internal
|
|
3018
3321
|
if '*' in v_str or '%' in v_str:
|
|
3019
3322
|
if session.bind.dialect.name == 'postgresql': # PostgreSQL escapes automatically
|
|
3020
|
-
|
|
3323
|
+
stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%')))
|
|
3021
3324
|
else:
|
|
3022
|
-
|
|
3325
|
+
stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%'), escape='\\'))
|
|
3023
3326
|
else:
|
|
3024
|
-
|
|
3327
|
+
stmt = stmt.where(getattr(models.CollectionReplica, k) == v)
|
|
3025
3328
|
# hints ?
|
|
3026
3329
|
elif k == 'created_before':
|
|
3027
3330
|
created_before = str_to_date(v)
|
|
3028
|
-
|
|
3331
|
+
stmt = stmt.where(models.CollectionReplica.created_at <= created_before)
|
|
3029
3332
|
elif k == 'created_after':
|
|
3030
3333
|
created_after = str_to_date(v)
|
|
3031
|
-
|
|
3334
|
+
stmt = stmt.where(models.CollectionReplica.created_at >= created_after)
|
|
3032
3335
|
else:
|
|
3033
|
-
|
|
3336
|
+
stmt = stmt.where(getattr(models.CollectionReplica, k) == v)
|
|
3034
3337
|
|
|
3035
3338
|
if limit:
|
|
3036
|
-
|
|
3339
|
+
stmt = stmt.limit(limit)
|
|
3037
3340
|
|
|
3038
|
-
for row in
|
|
3341
|
+
for row in session.execute(stmt).all():
|
|
3039
3342
|
yield row._asdict()
|
|
3040
3343
|
|
|
3041
3344
|
|
|
3345
|
+
@stream_session
|
|
3346
|
+
def list_replicas_per_rse(
|
|
3347
|
+
rse_id: str,
|
|
3348
|
+
limit: "Optional[int]" = None,
|
|
3349
|
+
*,
|
|
3350
|
+
session: "Session"
|
|
3351
|
+
) -> "Iterator[dict[str, Any]]":
|
|
3352
|
+
"""List all replicas at a given RSE."""
|
|
3353
|
+
list_stmt = select(
|
|
3354
|
+
models.RSEFileAssociation
|
|
3355
|
+
).where(
|
|
3356
|
+
models.RSEFileAssociation.rse_id == rse_id
|
|
3357
|
+
)
|
|
3358
|
+
|
|
3359
|
+
if limit:
|
|
3360
|
+
list_stmt = list_stmt.limit(limit)
|
|
3361
|
+
|
|
3362
|
+
for replica in session.execute(list_stmt).yield_per(100).scalars():
|
|
3363
|
+
yield replica.to_dict()
|
|
3364
|
+
|
|
3365
|
+
|
|
3042
3366
|
@transactional_session
|
|
3043
3367
|
def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=None, *, session: "Session"):
|
|
3044
3368
|
"""
|
|
@@ -3050,14 +3374,31 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
|
|
|
3050
3374
|
:returns: List of update requests for collection replicas.
|
|
3051
3375
|
"""
|
|
3052
3376
|
|
|
3377
|
+
stmt = delete(
|
|
3378
|
+
models.UpdatedCollectionReplica
|
|
3379
|
+
).where(
|
|
3380
|
+
and_(models.UpdatedCollectionReplica.rse_id.is_(None),
|
|
3381
|
+
~exists().where(
|
|
3382
|
+
and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,
|
|
3383
|
+
models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope)))
|
|
3384
|
+
).execution_options(
|
|
3385
|
+
synchronize_session=False
|
|
3386
|
+
)
|
|
3387
|
+
session.execute(stmt)
|
|
3388
|
+
|
|
3053
3389
|
# Delete update requests which do not have collection_replicas
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3390
|
+
stmt = delete(
|
|
3391
|
+
models.UpdatedCollectionReplica
|
|
3392
|
+
).where(
|
|
3393
|
+
and_(models.UpdatedCollectionReplica.rse_id.isnot(None),
|
|
3394
|
+
~exists().where(
|
|
3395
|
+
and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,
|
|
3396
|
+
models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
|
|
3397
|
+
models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id)))
|
|
3398
|
+
).execution_options(
|
|
3399
|
+
synchronize_session=False
|
|
3400
|
+
)
|
|
3401
|
+
session.execute(stmt)
|
|
3061
3402
|
|
|
3062
3403
|
# Delete duplicates
|
|
3063
3404
|
if session.bind.dialect.name == 'oracle':
|
|
@@ -3066,18 +3407,32 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
|
|
|
3066
3407
|
schema = BASE.metadata.schema + '.'
|
|
3067
3408
|
session.execute(text('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))) # NOQA: E501
|
|
3068
3409
|
elif session.bind.dialect.name == 'mysql':
|
|
3069
|
-
subquery1 =
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3410
|
+
subquery1 = select(
|
|
3411
|
+
func.max(models.UpdatedCollectionReplica.id).label('max_id')
|
|
3412
|
+
).group_by(
|
|
3413
|
+
models.UpdatedCollectionReplica.scope,
|
|
3414
|
+
models.UpdatedCollectionReplica.name,
|
|
3415
|
+
models.UpdatedCollectionReplica.rse_id
|
|
3416
|
+
).subquery()
|
|
3417
|
+
|
|
3418
|
+
subquery2 = select(
|
|
3419
|
+
subquery1.c.max_id
|
|
3420
|
+
)
|
|
3421
|
+
|
|
3422
|
+
stmt_del = delete(
|
|
3423
|
+
models.UpdatedCollectionReplica
|
|
3424
|
+
).where(
|
|
3425
|
+
models.UpdatedCollectionReplica.id.not_in(subquery2)
|
|
3426
|
+
).execution_options(
|
|
3427
|
+
synchronize_session=False
|
|
3428
|
+
)
|
|
3429
|
+
session.execute(stmt_del)
|
|
3075
3430
|
else:
|
|
3076
|
-
|
|
3431
|
+
stmt = select(models.UpdatedCollectionReplica)
|
|
3077
3432
|
update_requests_with_rse_id = []
|
|
3078
3433
|
update_requests_without_rse_id = []
|
|
3079
3434
|
duplicate_request_ids = []
|
|
3080
|
-
for update_request in
|
|
3435
|
+
for update_request in session.execute(stmt).scalars().all():
|
|
3081
3436
|
if update_request.rse_id is not None:
|
|
3082
3437
|
small_request = {'name': update_request.name, 'scope': update_request.scope, 'rse_id': update_request.rse_id}
|
|
3083
3438
|
if small_request not in update_requests_with_rse_id:
|
|
@@ -3093,12 +3448,19 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
|
|
|
3093
3448
|
duplicate_request_ids.append(update_request.id)
|
|
3094
3449
|
continue
|
|
3095
3450
|
for chunk in chunks(duplicate_request_ids, 100):
|
|
3096
|
-
|
|
3451
|
+
stmt = delete(
|
|
3452
|
+
models.UpdatedCollectionReplica
|
|
3453
|
+
).where(
|
|
3454
|
+
models.UpdatedCollectionReplica.id.in_(chunk)
|
|
3455
|
+
).execution_options(
|
|
3456
|
+
synchronize_session=False
|
|
3457
|
+
)
|
|
3458
|
+
session.execute(stmt)
|
|
3097
3459
|
|
|
3098
|
-
|
|
3460
|
+
stmt = select(models.UpdatedCollectionReplica)
|
|
3099
3461
|
if limit:
|
|
3100
|
-
|
|
3101
|
-
return [update_request.to_dict() for update_request in
|
|
3462
|
+
stmt = stmt.limit(limit)
|
|
3463
|
+
return [update_request.to_dict() for update_request in session.execute(stmt).scalars().all()]
|
|
3102
3464
|
|
|
3103
3465
|
|
|
3104
3466
|
@transactional_session
|
|
@@ -3117,11 +3479,14 @@ def update_collection_replica(update_request, *, session: "Session"):
|
|
|
3117
3479
|
available_replicas = 0
|
|
3118
3480
|
|
|
3119
3481
|
try:
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3482
|
+
stmt = select(
|
|
3483
|
+
models.CollectionReplica
|
|
3484
|
+
).where(
|
|
3485
|
+
and_(models.CollectionReplica.scope == update_request['scope'],
|
|
3486
|
+
models.CollectionReplica.name == update_request['name'],
|
|
3487
|
+
models.CollectionReplica.rse_id == update_request['rse_id'])
|
|
3488
|
+
)
|
|
3489
|
+
collection_replica = session.execute(stmt).scalar_one()
|
|
3125
3490
|
ds_length = collection_replica.length
|
|
3126
3491
|
old_available_replicas = collection_replica.available_replicas_cnt
|
|
3127
3492
|
ds_bytes = collection_replica.bytes
|
|
@@ -3129,16 +3494,21 @@ def update_collection_replica(update_request, *, session: "Session"):
|
|
|
3129
3494
|
pass
|
|
3130
3495
|
|
|
3131
3496
|
try:
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3497
|
+
stmt = select(
|
|
3498
|
+
func.sum(models.RSEFileAssociation.bytes).label('ds_available_bytes'),
|
|
3499
|
+
func.count().label('available_replicas')
|
|
3500
|
+
).select_from(
|
|
3501
|
+
models.RSEFileAssociation
|
|
3502
|
+
).where(
|
|
3503
|
+
and_(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
|
|
3504
|
+
models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
|
|
3505
|
+
models.RSEFileAssociation.rse_id == update_request['rse_id'],
|
|
3506
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
3507
|
+
models.DataIdentifierAssociation.name == update_request['name'],
|
|
3508
|
+
models.DataIdentifierAssociation.scope == update_request['scope'])
|
|
3509
|
+
)
|
|
3510
|
+
file_replica = session.execute(stmt).one()
|
|
3511
|
+
|
|
3142
3512
|
available_replicas = file_replica.available_replicas
|
|
3143
3513
|
ds_available_bytes = file_replica.ds_available_bytes
|
|
3144
3514
|
except NoResultFound:
|
|
@@ -3150,36 +3520,53 @@ def update_collection_replica(update_request, *, session: "Session"):
|
|
|
3150
3520
|
ds_replica_state = ReplicaState.UNAVAILABLE
|
|
3151
3521
|
|
|
3152
3522
|
if old_available_replicas is not None and old_available_replicas > 0 and available_replicas == 0:
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3523
|
+
stmt = delete(
|
|
3524
|
+
models.CollectionReplica
|
|
3525
|
+
).where(
|
|
3526
|
+
and_(models.CollectionReplica.scope == update_request['scope'],
|
|
3527
|
+
models.CollectionReplica.name == update_request['name'],
|
|
3528
|
+
models.CollectionReplica.rse_id == update_request['rse_id'])
|
|
3529
|
+
)
|
|
3530
|
+
session.execute(stmt)
|
|
3157
3531
|
else:
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3532
|
+
stmt = select(
|
|
3533
|
+
models.CollectionReplica
|
|
3534
|
+
).where(
|
|
3535
|
+
and_(models.CollectionReplica.scope == update_request['scope'],
|
|
3536
|
+
models.CollectionReplica.name == update_request['name'],
|
|
3537
|
+
models.CollectionReplica.rse_id == update_request['rse_id'])
|
|
3538
|
+
)
|
|
3539
|
+
updated_replica = session.execute(stmt).scalar_one()
|
|
3540
|
+
|
|
3162
3541
|
updated_replica.state = ds_replica_state
|
|
3163
3542
|
updated_replica.available_replicas_cnt = available_replicas
|
|
3164
3543
|
updated_replica.length = ds_length
|
|
3165
3544
|
updated_replica.bytes = ds_bytes
|
|
3166
3545
|
updated_replica.available_bytes = ds_available_bytes
|
|
3167
3546
|
else:
|
|
3547
|
+
stmt = select(
|
|
3548
|
+
func.count().label('ds_length'),
|
|
3549
|
+
func.sum(models.DataIdentifierAssociation.bytes).label('ds_bytes')
|
|
3550
|
+
).select_from(
|
|
3551
|
+
models.DataIdentifierAssociation
|
|
3552
|
+
).where(
|
|
3553
|
+
and_(models.DataIdentifierAssociation.scope == update_request['scope'],
|
|
3554
|
+
models.DataIdentifierAssociation.name == update_request['name'])
|
|
3555
|
+
)
|
|
3556
|
+
association = session.execute(stmt).one()
|
|
3557
|
+
|
|
3168
3558
|
# Check all dataset replicas
|
|
3169
|
-
association = session.query(models.DataIdentifierAssociation)\
|
|
3170
|
-
.filter_by(scope=update_request['scope'],
|
|
3171
|
-
name=update_request['name'])\
|
|
3172
|
-
.with_entities(label('ds_length', func.count()),
|
|
3173
|
-
label('ds_bytes', func.sum(models.DataIdentifierAssociation.bytes)))\
|
|
3174
|
-
.one()
|
|
3175
3559
|
ds_length = association.ds_length
|
|
3176
3560
|
ds_bytes = association.ds_bytes
|
|
3177
3561
|
ds_replica_state = None
|
|
3178
3562
|
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3563
|
+
stmt = select(
|
|
3564
|
+
models.CollectionReplica
|
|
3565
|
+
).where(
|
|
3566
|
+
and_(models.CollectionReplica.scope == update_request['scope'],
|
|
3567
|
+
models.CollectionReplica.name == update_request['name'])
|
|
3568
|
+
)
|
|
3569
|
+
for collection_replica in session.execute(stmt).scalars().all():
|
|
3183
3570
|
if ds_length:
|
|
3184
3571
|
collection_replica.length = ds_length
|
|
3185
3572
|
else:
|
|
@@ -3189,31 +3576,47 @@ def update_collection_replica(update_request, *, session: "Session"):
|
|
|
3189
3576
|
else:
|
|
3190
3577
|
collection_replica.bytes = 0
|
|
3191
3578
|
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3579
|
+
stmt = select(
|
|
3580
|
+
func.sum(models.RSEFileAssociation.bytes).label('ds_available_bytes'),
|
|
3581
|
+
func.count().label('available_replicas'),
|
|
3582
|
+
models.RSEFileAssociation.rse_id
|
|
3583
|
+
).select_from(
|
|
3584
|
+
models.RSEFileAssociation
|
|
3585
|
+
).where(
|
|
3586
|
+
and_(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
|
|
3587
|
+
models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
|
|
3588
|
+
models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
|
|
3589
|
+
models.DataIdentifierAssociation.name == update_request['name'],
|
|
3590
|
+
models.DataIdentifierAssociation.scope == update_request['scope'])
|
|
3591
|
+
).group_by(
|
|
3592
|
+
models.RSEFileAssociation.rse_id
|
|
3593
|
+
)
|
|
3594
|
+
|
|
3595
|
+
for file_replica in session.execute(stmt).all():
|
|
3204
3596
|
if file_replica.available_replicas >= ds_length:
|
|
3205
3597
|
ds_replica_state = ReplicaState.AVAILABLE
|
|
3206
3598
|
else:
|
|
3207
3599
|
ds_replica_state = ReplicaState.UNAVAILABLE
|
|
3208
3600
|
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3601
|
+
stmt = select(
|
|
3602
|
+
models.CollectionReplica
|
|
3603
|
+
).where(
|
|
3604
|
+
and_(models.CollectionReplica.scope == update_request['scope'],
|
|
3605
|
+
models.CollectionReplica.name == update_request['name'],
|
|
3606
|
+
models.CollectionReplica.rse_id == file_replica.rse_id)
|
|
3607
|
+
)
|
|
3608
|
+
collection_replica = session.execute(stmt).scalars().first()
|
|
3212
3609
|
if collection_replica:
|
|
3213
3610
|
collection_replica.state = ds_replica_state
|
|
3214
3611
|
collection_replica.available_replicas_cnt = file_replica.available_replicas
|
|
3215
3612
|
collection_replica.available_bytes = file_replica.ds_available_bytes
|
|
3216
|
-
|
|
3613
|
+
|
|
3614
|
+
stmt = delete(
|
|
3615
|
+
models.UpdatedCollectionReplica
|
|
3616
|
+
).where(
|
|
3617
|
+
models.UpdatedCollectionReplica.id == update_request['id']
|
|
3618
|
+
)
|
|
3619
|
+
session.execute(stmt)
|
|
3217
3620
|
|
|
3218
3621
|
|
|
3219
3622
|
@read_session
|
|
@@ -3229,12 +3632,23 @@ def get_bad_pfns(limit=10000, thread=None, total_threads=None, *, session: "Sess
|
|
|
3229
3632
|
returns: list of PFNs {'pfn': pfn, 'state': state, 'reason': reason, 'account': account, 'expires_at': expires_at}
|
|
3230
3633
|
"""
|
|
3231
3634
|
result = []
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3635
|
+
|
|
3636
|
+
stmt = select(
|
|
3637
|
+
models.BadPFN.path,
|
|
3638
|
+
models.BadPFN.state,
|
|
3639
|
+
models.BadPFN.reason,
|
|
3640
|
+
models.BadPFN.account,
|
|
3641
|
+
models.BadPFN.expires_at
|
|
3642
|
+
)
|
|
3643
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_threads, thread_id=thread, hash_variable='path')
|
|
3644
|
+
stmt = stmt.order_by(
|
|
3645
|
+
models.BadPFN.created_at
|
|
3646
|
+
).limit(
|
|
3647
|
+
limit
|
|
3648
|
+
)
|
|
3649
|
+
|
|
3650
|
+
for path, state, reason, account, expires_at in session.execute(stmt).yield_per(1000):
|
|
3651
|
+
result.append({'pfn': clean_pfns([str(path)])[0], 'state': state, 'reason': reason, 'account': account, 'expires_at': expires_at})
|
|
3238
3652
|
return result
|
|
3239
3653
|
|
|
3240
3654
|
|
|
@@ -3251,15 +3665,37 @@ def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAV
|
|
|
3251
3665
|
:returns: True is successful.
|
|
3252
3666
|
"""
|
|
3253
3667
|
for replica in replicas:
|
|
3668
|
+
scope_name_rse_state = and_(models.BadReplica.scope == replica['scope'],
|
|
3669
|
+
models.BadReplica.name == replica['name'],
|
|
3670
|
+
models.BadReplica.rse_id == replica['rse_id'],
|
|
3671
|
+
models.BadReplica.state == state)
|
|
3254
3672
|
insert_new_row = True
|
|
3255
3673
|
if state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3674
|
+
stmt = select(
|
|
3675
|
+
models.BadReplica
|
|
3676
|
+
).where(
|
|
3677
|
+
scope_name_rse_state
|
|
3678
|
+
)
|
|
3679
|
+
if session.execute(stmt).scalar_one_or_none():
|
|
3680
|
+
stmt = update(
|
|
3681
|
+
models.BadReplica
|
|
3682
|
+
).where(
|
|
3683
|
+
scope_name_rse_state
|
|
3684
|
+
).values({
|
|
3685
|
+
models.BadReplica.state: BadFilesStatus.TEMPORARY_UNAVAILABLE,
|
|
3686
|
+
models.BadReplica.updated_at: datetime.utcnow(),
|
|
3687
|
+
models.BadReplica.account: account,
|
|
3688
|
+
models.BadReplica.reason: reason,
|
|
3689
|
+
models.BadReplica.expires_at: expires_at
|
|
3690
|
+
}).execution_options(
|
|
3691
|
+
synchronize_session=False
|
|
3692
|
+
)
|
|
3693
|
+
session.execute(stmt)
|
|
3694
|
+
|
|
3259
3695
|
insert_new_row = False
|
|
3260
3696
|
if insert_new_row:
|
|
3261
|
-
new_bad_replica = models.
|
|
3262
|
-
|
|
3697
|
+
new_bad_replica = models.BadReplica(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], reason=reason,
|
|
3698
|
+
state=state, account=account, bytes=None, expires_at=expires_at)
|
|
3263
3699
|
new_bad_replica.save(session=session, flush=False)
|
|
3264
3700
|
try:
|
|
3265
3701
|
session.flush()
|
|
@@ -3286,11 +3722,17 @@ def bulk_delete_bad_pfns(pfns, *, session: "Session"):
|
|
|
3286
3722
|
"""
|
|
3287
3723
|
pfn_clause = []
|
|
3288
3724
|
for pfn in pfns:
|
|
3289
|
-
pfn_clause.append(models.
|
|
3725
|
+
pfn_clause.append(models.BadPFN.path == pfn)
|
|
3290
3726
|
|
|
3291
3727
|
for chunk in chunks(pfn_clause, 100):
|
|
3292
|
-
|
|
3293
|
-
|
|
3728
|
+
stmt = delete(
|
|
3729
|
+
models.BadPFN
|
|
3730
|
+
).where(
|
|
3731
|
+
or_(*chunk)
|
|
3732
|
+
).execution_options(
|
|
3733
|
+
synchronize_session=False
|
|
3734
|
+
)
|
|
3735
|
+
session.execute(stmt)
|
|
3294
3736
|
|
|
3295
3737
|
return True
|
|
3296
3738
|
|
|
@@ -3307,14 +3749,20 @@ def bulk_delete_bad_replicas(bad_replicas, *, session: "Session"):
|
|
|
3307
3749
|
"""
|
|
3308
3750
|
replica_clause = []
|
|
3309
3751
|
for replica in bad_replicas:
|
|
3310
|
-
replica_clause.append(and_(models.
|
|
3311
|
-
models.
|
|
3312
|
-
models.
|
|
3313
|
-
models.
|
|
3752
|
+
replica_clause.append(and_(models.BadReplica.scope == replica['scope'],
|
|
3753
|
+
models.BadReplica.name == replica['name'],
|
|
3754
|
+
models.BadReplica.rse_id == replica['rse_id'],
|
|
3755
|
+
models.BadReplica.state == replica['state']))
|
|
3314
3756
|
|
|
3315
3757
|
for chunk in chunks(replica_clause, 100):
|
|
3316
|
-
|
|
3317
|
-
|
|
3758
|
+
stmt = delete(
|
|
3759
|
+
models.BadReplica
|
|
3760
|
+
).where(
|
|
3761
|
+
or_(*chunk)
|
|
3762
|
+
).execution_options(
|
|
3763
|
+
synchronize_session=False
|
|
3764
|
+
)
|
|
3765
|
+
session.execute(stmt)
|
|
3318
3766
|
return True
|
|
3319
3767
|
|
|
3320
3768
|
|
|
@@ -3343,9 +3791,9 @@ def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, *, session:
|
|
|
3343
3791
|
elif rep_state == BadPFNStatus.BAD and expires_at is not None:
|
|
3344
3792
|
raise exception.InputValidationError("When adding a BAD pfn the expires_at value shouldn't be set.")
|
|
3345
3793
|
|
|
3346
|
-
pfns =
|
|
3794
|
+
pfns = clean_pfns(pfns)
|
|
3347
3795
|
for pfn in pfns:
|
|
3348
|
-
new_pfn = models.
|
|
3796
|
+
new_pfn = models.BadPFN(path=str(pfn), account=account, state=rep_state, reason=reason, expires_at=expires_at)
|
|
3349
3797
|
new_pfn = session.merge(new_pfn)
|
|
3350
3798
|
new_pfn.save(session=session, flush=False)
|
|
3351
3799
|
|
|
@@ -3373,15 +3821,25 @@ def list_expired_temporary_unavailable_replicas(total_workers, worker_number, li
|
|
|
3373
3821
|
:param session: The database session in use.
|
|
3374
3822
|
"""
|
|
3375
3823
|
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3824
|
+
stmt = select(
|
|
3825
|
+
models.BadReplica.scope,
|
|
3826
|
+
models.BadReplica.name,
|
|
3827
|
+
models.BadReplica.rse_id,
|
|
3828
|
+
).with_hint(
|
|
3829
|
+
models.ReplicationRule,
|
|
3830
|
+
'INDEX(bad_replicas BAD_REPLICAS_EXPIRES_AT_IDX)',
|
|
3831
|
+
'oracle'
|
|
3832
|
+
).where(
|
|
3833
|
+
and_(models.BadReplica.state == BadFilesStatus.TEMPORARY_UNAVAILABLE,
|
|
3834
|
+
models.BadReplica.expires_at < datetime.utcnow())
|
|
3835
|
+
).order_by(
|
|
3836
|
+
models.BadReplica.expires_at
|
|
3837
|
+
)
|
|
3838
|
+
|
|
3839
|
+
stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
|
|
3840
|
+
stmt = stmt.limit(limit)
|
|
3381
3841
|
|
|
3382
|
-
|
|
3383
|
-
query = query.limit(limit)
|
|
3384
|
-
return query.all()
|
|
3842
|
+
return session.execute(stmt).all()
|
|
3385
3843
|
|
|
3386
3844
|
|
|
3387
3845
|
@read_session
|
|
@@ -3395,9 +3853,15 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
|
|
|
3395
3853
|
:returns: A dictionary with the list of states as keys and the rse_ids as value
|
|
3396
3854
|
"""
|
|
3397
3855
|
|
|
3398
|
-
|
|
3856
|
+
stmt = select(
|
|
3857
|
+
models.RSEFileAssociation.rse_id,
|
|
3858
|
+
models.RSEFileAssociation.state
|
|
3859
|
+
).where(
|
|
3860
|
+
and_(models.RSEFileAssociation.scope == scope,
|
|
3861
|
+
models.RSEFileAssociation.name == name)
|
|
3862
|
+
)
|
|
3399
3863
|
states = {}
|
|
3400
|
-
for res in
|
|
3864
|
+
for res in session.execute(stmt).all():
|
|
3401
3865
|
rse_id, state = res
|
|
3402
3866
|
if state not in states:
|
|
3403
3867
|
states[state] = []
|
|
@@ -3406,7 +3870,19 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
|
|
|
3406
3870
|
|
|
3407
3871
|
|
|
3408
3872
|
@read_session
|
|
3409
|
-
def get_suspicious_files(
|
|
3873
|
+
def get_suspicious_files(
|
|
3874
|
+
rse_expression: str,
|
|
3875
|
+
available_elsewhere: int,
|
|
3876
|
+
filter_: "Optional[dict[str, Any]]" = None,
|
|
3877
|
+
logger: "LoggerFunction" = logging.log,
|
|
3878
|
+
younger_than: "Optional[datetime]" = None,
|
|
3879
|
+
nattempts: int = 0,
|
|
3880
|
+
nattempts_exact: bool = False,
|
|
3881
|
+
*,
|
|
3882
|
+
session: "Session",
|
|
3883
|
+
exclude_states: "Optional[Iterable[str]]" = None,
|
|
3884
|
+
is_suspicious: bool = False
|
|
3885
|
+
) -> "list[dict[str, Any]]":
|
|
3410
3886
|
"""
|
|
3411
3887
|
Gets a list of replicas from bad_replicas table which are: declared more than <nattempts> times since <younger_than> date,
|
|
3412
3888
|
present on the RSE specified by the <rse_expression> and do not have a state in <exclude_states> list.
|
|
@@ -3415,7 +3891,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
|
|
|
3415
3891
|
Keyword Arguments:
|
|
3416
3892
|
:param younger_than: Datetime object to select the replicas which were declared since younger_than date. Default value = 10 days ago.
|
|
3417
3893
|
:param nattempts: The minimum number of replica appearances in the bad_replica DB table from younger_than date. Default value = 0.
|
|
3418
|
-
:param nattempts_exact: If True, then only replicas with exactly 'nattempts'
|
|
3894
|
+
:param nattempts_exact: If True, then only replicas with exactly 'nattempts' appearances in the bad_replica DB table are retrieved. Replicas with more appearances are ignored.
|
|
3419
3895
|
:param rse_expression: The RSE expression where the replicas are located.
|
|
3420
3896
|
:param filter_: Dictionary of attributes by which the RSE results should be filtered. e.g.: {'availability_write': True}
|
|
3421
3897
|
:param exclude_states: List of states which eliminates replicas from search result if any of the states in the list
|
|
@@ -3432,6 +3908,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
|
|
|
3432
3908
|
[{'scope': scope, 'name': name, 'rse': rse, 'rse_id': rse_id, cnt': cnt, 'created_at': created_at}, ...]
|
|
3433
3909
|
"""
|
|
3434
3910
|
|
|
3911
|
+
exclude_states = exclude_states or ['B', 'R', 'D']
|
|
3435
3912
|
if available_elsewhere not in [SuspiciousAvailability["ALL"].value, SuspiciousAvailability["EXIST_COPIES"].value, SuspiciousAvailability["LAST_COPY"].value]:
|
|
3436
3913
|
logger(logging.WARNING, """ERROR, available_elsewhere must be set to one of the following:
|
|
3437
3914
|
SuspiciousAvailability["ALL"].value: (default) all suspicious replicas are returned
|
|
@@ -3454,7 +3931,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
|
|
|
3454
3931
|
exclude_states_clause.append(BadFilesStatus(state))
|
|
3455
3932
|
|
|
3456
3933
|
# making aliases for bad_replicas and replicas tables
|
|
3457
|
-
bad_replicas_alias = aliased(models.
|
|
3934
|
+
bad_replicas_alias = aliased(models.BadReplica, name='bad_replicas_alias')
|
|
3458
3935
|
replicas_alias = aliased(models.RSEFileAssociation, name='replicas_alias')
|
|
3459
3936
|
|
|
3460
3937
|
# assembling the selection rse_clause
|
|
@@ -3464,51 +3941,76 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
|
|
|
3464
3941
|
for rse in parsedexp:
|
|
3465
3942
|
rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
|
|
3466
3943
|
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3944
|
+
stmt = select(
|
|
3945
|
+
func.count(),
|
|
3946
|
+
bad_replicas_alias.scope,
|
|
3947
|
+
bad_replicas_alias.name,
|
|
3948
|
+
models.RSEFileAssociation.rse_id,
|
|
3949
|
+
func.min(models.RSEFileAssociation.created_at)
|
|
3950
|
+
).select_from(
|
|
3951
|
+
bad_replicas_alias
|
|
3952
|
+
).where(
|
|
3953
|
+
models.RSEFileAssociation.rse_id == bad_replicas_alias.rse_id,
|
|
3954
|
+
models.RSEFileAssociation.scope == bad_replicas_alias.scope,
|
|
3955
|
+
models.RSEFileAssociation.name == bad_replicas_alias.name,
|
|
3956
|
+
bad_replicas_alias.created_at >= younger_than
|
|
3957
|
+
)
|
|
3473
3958
|
if is_suspicious:
|
|
3474
|
-
|
|
3959
|
+
stmt = stmt.where(bad_replicas_alias.state == BadFilesStatus.SUSPICIOUS)
|
|
3475
3960
|
if rse_clause:
|
|
3476
|
-
|
|
3961
|
+
stmt = stmt.where(or_(*rse_clause))
|
|
3477
3962
|
|
|
3478
3963
|
# Only return replicas that have at least one copy on another RSE
|
|
3479
3964
|
if available_elsewhere == SuspiciousAvailability["EXIST_COPIES"].value:
|
|
3480
|
-
available_replica = exists(select(1)
|
|
3481
|
-
|
|
3482
|
-
|
|
3483
|
-
|
|
3484
|
-
|
|
3965
|
+
available_replica = exists(select(1)
|
|
3966
|
+
.where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
|
|
3967
|
+
replicas_alias.scope == bad_replicas_alias.scope,
|
|
3968
|
+
replicas_alias.name == bad_replicas_alias.name,
|
|
3969
|
+
replicas_alias.rse_id != bad_replicas_alias.rse_id)))
|
|
3970
|
+
stmt = stmt.where(available_replica)
|
|
3485
3971
|
|
|
3486
3972
|
# Only return replicas that are the last remaining copy
|
|
3487
3973
|
if available_elsewhere == SuspiciousAvailability["LAST_COPY"].value:
|
|
3488
|
-
last_replica = ~exists(select(1)
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3974
|
+
last_replica = ~exists(select(1)
|
|
3975
|
+
.where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
|
|
3976
|
+
replicas_alias.scope == bad_replicas_alias.scope,
|
|
3977
|
+
replicas_alias.name == bad_replicas_alias.name,
|
|
3978
|
+
replicas_alias.rse_id != bad_replicas_alias.rse_id)))
|
|
3979
|
+
stmt = stmt.where(last_replica)
|
|
3493
3980
|
|
|
3494
3981
|
# it is required that the selected replicas
|
|
3495
3982
|
# do not occur as BAD/DELETED/LOST/RECOVERED/...
|
|
3496
3983
|
# in the bad_replicas table during the same time window.
|
|
3497
|
-
other_states_present = exists(select(1)
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3984
|
+
other_states_present = exists(select(1)
|
|
3985
|
+
.where(and_(models.BadReplica.scope == bad_replicas_alias.scope,
|
|
3986
|
+
models.BadReplica.name == bad_replicas_alias.name,
|
|
3987
|
+
models.BadReplica.created_at >= younger_than,
|
|
3988
|
+
models.BadReplica.rse_id == bad_replicas_alias.rse_id,
|
|
3989
|
+
models.BadReplica.state.in_(exclude_states_clause))))
|
|
3990
|
+
stmt = stmt.where(not_(other_states_present))
|
|
3503
3991
|
|
|
3504
3992
|
# finally, the results are grouped by RSE, scope, name and required to have
|
|
3505
3993
|
# at least 'nattempts' occurrences in the result of the query per replica.
|
|
3506
3994
|
# If nattempts_exact, then only replicas are required to have exactly
|
|
3507
|
-
# 'nattempts'
|
|
3995
|
+
# 'nattempts' occurrences.
|
|
3508
3996
|
if nattempts_exact:
|
|
3509
|
-
|
|
3997
|
+
stmt = stmt.group_by(
|
|
3998
|
+
models.RSEFileAssociation.rse_id,
|
|
3999
|
+
bad_replicas_alias.scope,
|
|
4000
|
+
bad_replicas_alias.name
|
|
4001
|
+
).having(
|
|
4002
|
+
func.count() == nattempts
|
|
4003
|
+
)
|
|
4004
|
+
query_result = session.execute(stmt).all()
|
|
3510
4005
|
else:
|
|
3511
|
-
|
|
4006
|
+
stmt = stmt.group_by(
|
|
4007
|
+
models.RSEFileAssociation.rse_id,
|
|
4008
|
+
bad_replicas_alias.scope,
|
|
4009
|
+
bad_replicas_alias.name
|
|
4010
|
+
).having(
|
|
4011
|
+
func.count() > nattempts
|
|
4012
|
+
)
|
|
4013
|
+
query_result = session.execute(stmt).all()
|
|
3512
4014
|
|
|
3513
4015
|
# translating the rse_id to RSE name and assembling the return list of dictionaries
|
|
3514
4016
|
result = []
|
|
@@ -3533,20 +4035,44 @@ def get_suspicious_reason(rse_id, scope, name, nattempts=0, logger=logging.log,
|
|
|
3533
4035
|
:param session: The database session in use. Default value = None.
|
|
3534
4036
|
"""
|
|
3535
4037
|
# Alias for bad replicas
|
|
3536
|
-
bad_replicas_alias = aliased(models.
|
|
4038
|
+
bad_replicas_alias = aliased(models.BadReplica, name='bad_replicas_alias')
|
|
3537
4039
|
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3543
|
-
|
|
4040
|
+
stmt = select(
|
|
4041
|
+
bad_replicas_alias.scope,
|
|
4042
|
+
bad_replicas_alias.name,
|
|
4043
|
+
bad_replicas_alias.reason,
|
|
4044
|
+
bad_replicas_alias.rse_id
|
|
4045
|
+
).where(
|
|
4046
|
+
and_(bad_replicas_alias.rse_id == rse_id,
|
|
4047
|
+
bad_replicas_alias.scope == scope,
|
|
4048
|
+
bad_replicas_alias.state == 'S',
|
|
4049
|
+
bad_replicas_alias.name == name,
|
|
4050
|
+
~exists(select(1).where(
|
|
4051
|
+
and_(bad_replicas_alias.rse_id == rse_id,
|
|
4052
|
+
bad_replicas_alias.name == name,
|
|
4053
|
+
bad_replicas_alias.scope == scope,
|
|
4054
|
+
bad_replicas_alias.state != 'S'))))
|
|
4055
|
+
)
|
|
3544
4056
|
|
|
3545
|
-
|
|
4057
|
+
count_query = select(
|
|
4058
|
+
func.count()
|
|
4059
|
+
).select_from(
|
|
4060
|
+
stmt.subquery()
|
|
4061
|
+
)
|
|
4062
|
+
count = session.execute(count_query).scalar_one()
|
|
4063
|
+
|
|
4064
|
+
grouped_stmt = stmt.group_by(
|
|
4065
|
+
bad_replicas_alias.rse_id,
|
|
4066
|
+
bad_replicas_alias.scope,
|
|
4067
|
+
bad_replicas_alias.name,
|
|
4068
|
+
bad_replicas_alias.reason
|
|
4069
|
+
).having(
|
|
4070
|
+
func.count() > nattempts
|
|
4071
|
+
)
|
|
3546
4072
|
|
|
3547
4073
|
result = []
|
|
3548
4074
|
rses = {}
|
|
3549
|
-
for scope_, name_, reason, rse_id_ in
|
|
4075
|
+
for scope_, name_, reason, rse_id_ in session.execute(grouped_stmt).all():
|
|
3550
4076
|
if rse_id_ not in rses:
|
|
3551
4077
|
rse = get_rse_name(rse_id=rse_id_, session=session)
|
|
3552
4078
|
rses[rse_id_] = rse
|
|
@@ -3570,27 +4096,31 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, *, session: "Session"
|
|
|
3570
4096
|
:param session: database session in use.
|
|
3571
4097
|
"""
|
|
3572
4098
|
stmt = update(models.RSEFileAssociation).where(
|
|
3573
|
-
and_(
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
.execution_options(synchronize_session=False)\
|
|
3588
|
-
.values(tombstone=tombstone)
|
|
3589
|
-
result = session.execute(stmt)
|
|
4099
|
+
and_(models.RSEFileAssociation.rse_id == rse_id,
|
|
4100
|
+
models.RSEFileAssociation.name == name,
|
|
4101
|
+
models.RSEFileAssociation.scope == scope,
|
|
4102
|
+
~exists().where(
|
|
4103
|
+
and_(models.ReplicaLock.rse_id == rse_id,
|
|
4104
|
+
models.ReplicaLock.name == name,
|
|
4105
|
+
models.ReplicaLock.scope == scope)))
|
|
4106
|
+
).prefix_with(
|
|
4107
|
+
'/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'
|
|
4108
|
+
).values({
|
|
4109
|
+
models.RSEFileAssociation.tombstone: tombstone
|
|
4110
|
+
}).execution_options(
|
|
4111
|
+
synchronize_session=False
|
|
4112
|
+
)
|
|
3590
4113
|
|
|
3591
|
-
if
|
|
4114
|
+
if session.execute(stmt).rowcount == 0:
|
|
3592
4115
|
try:
|
|
3593
|
-
|
|
4116
|
+
stmt = select(
|
|
4117
|
+
models.RSEFileAssociation.tombstone
|
|
4118
|
+
).where(
|
|
4119
|
+
and_(models.RSEFileAssociation.rse_id == rse_id,
|
|
4120
|
+
models.RSEFileAssociation.name == name,
|
|
4121
|
+
models.RSEFileAssociation.scope == scope)
|
|
4122
|
+
)
|
|
4123
|
+
session.execute(stmt).scalar_one()
|
|
3594
4124
|
raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
|
|
3595
4125
|
except NoResultFound:
|
|
3596
4126
|
raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
|
|
@@ -3607,20 +4137,21 @@ def get_RSEcoverage_of_dataset(scope, name, *, session: "Session"):
|
|
|
3607
4137
|
:return: Dictionary { rse_id : <total bytes present at rse_id> }
|
|
3608
4138
|
"""
|
|
3609
4139
|
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
4140
|
+
stmt = select(
|
|
4141
|
+
models.RSEFileAssociation.rse_id,
|
|
4142
|
+
func.sum(models.DataIdentifierAssociation.bytes)
|
|
4143
|
+
).where(
|
|
4144
|
+
and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
|
|
4145
|
+
models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
|
|
4146
|
+
models.DataIdentifierAssociation.scope == scope,
|
|
4147
|
+
models.DataIdentifierAssociation.name == name,
|
|
4148
|
+
models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
|
|
4149
|
+
).group_by(
|
|
4150
|
+
models.RSEFileAssociation.rse_id
|
|
4151
|
+
)
|
|
3620
4152
|
|
|
3621
4153
|
result = {}
|
|
3622
|
-
|
|
3623
|
-
for rse_id, total in query:
|
|
4154
|
+
for rse_id, total in session.execute(stmt):
|
|
3624
4155
|
if total:
|
|
3625
4156
|
result[rse_id] = total
|
|
3626
4157
|
|