rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -21,7 +20,7 @@
|
|
|
21
20
|
import argparse
|
|
22
21
|
import json
|
|
23
22
|
import os
|
|
24
|
-
import
|
|
23
|
+
import secrets
|
|
25
24
|
import ssl
|
|
26
25
|
import sys
|
|
27
26
|
|
|
@@ -80,7 +79,7 @@ def cache_operation(args):
|
|
|
80
79
|
conn.set_ssl(key_file=args.ssl_key_file, cert_file=args.ssl_cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
|
|
81
80
|
|
|
82
81
|
conn.connect()
|
|
83
|
-
message = {'id':
|
|
82
|
+
message = {'id': secrets.randbelow(1001), 'payload': payload}
|
|
84
83
|
conn.send(destination=args.destination, body=json.dumps(message), id='rucio-cache-messaging', ack='auto', headers={'vo': 'atlas'})
|
|
85
84
|
conn.disconnect()
|
|
86
85
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -28,7 +27,7 @@ def get_parser():
|
|
|
28
27
|
"""
|
|
29
28
|
Returns the argparse parser.
|
|
30
29
|
"""
|
|
31
|
-
parser = argparse.ArgumentParser(description="Conveyor is a group of daemons to manage file transfers. The conveyor-finisher is the
|
|
30
|
+
parser = argparse.ArgumentParser(description="Conveyor is a group of daemons to manage file transfers. The conveyor-finisher is the responsible to update Rucio internal state after the transfer has finished.")
|
|
32
31
|
parser.add_argument("--run-once", action="store_true", default=False,
|
|
33
32
|
help='One iteration only')
|
|
34
33
|
parser.add_argument("--total-threads", action="store", default=1, type=int,
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -35,7 +34,7 @@ Upload a file and create a replication rule::
|
|
|
35
34
|
$ rucio add-rule mock:file 1 MOCK2
|
|
36
35
|
$ rucio-admin rse add-distance MOCK2 MOCK --distance 1
|
|
37
36
|
|
|
38
|
-
The rule should replicate the file from RSE MOCK to RSE MOCK2.
|
|
37
|
+
The rule should replicate the file from RSE MOCK to RSE MOCK2. Therefore a distance between these RSEs is needed.
|
|
39
38
|
|
|
40
39
|
Check transfer requests for the DID::
|
|
41
40
|
|
|
@@ -57,7 +56,7 @@ Check again the transfer requests for the DID::
|
|
|
57
56
|
session.get_session().query(models.Request).filter_by(scope='mock', name='file').first()
|
|
58
57
|
# {'request_type': TRANSFER, 'state': SUBMITTED', ...}
|
|
59
58
|
|
|
60
|
-
A
|
|
59
|
+
A transfer request got created by executing the transfer. Depending on the transfer submission, the request state can be different. In this example the transfer got submitted successfully.
|
|
61
60
|
|
|
62
61
|
When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::
|
|
63
62
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,11 +17,11 @@ import argparse
|
|
|
18
17
|
import datetime
|
|
19
18
|
import logging
|
|
20
19
|
import sys
|
|
20
|
+
|
|
21
21
|
import tabulate
|
|
22
22
|
|
|
23
23
|
import rucio.common.dumper.consistency as consistency
|
|
24
24
|
import rucio.common.dumper.data_models as data_models
|
|
25
|
-
|
|
26
25
|
from rucio.common.dumper import error
|
|
27
26
|
|
|
28
27
|
logger = logging.getLogger('rucio-dumper')
|
|
@@ -96,9 +95,10 @@ if __name__ == "__main__":
|
|
|
96
95
|
record_type = data_models.CompleteDataset
|
|
97
96
|
elif args.subcommand == 'dump-replicas':
|
|
98
97
|
record_type = data_models.Replica
|
|
99
|
-
|
|
100
|
-
assert args.subcommand in consistency.subcommands
|
|
98
|
+
elif args.subcommand in consistency.subcommands:
|
|
101
99
|
record_type = consistency.Consistency
|
|
100
|
+
else:
|
|
101
|
+
raise ValueError("Record type not set! Cannot dump files.")
|
|
102
102
|
|
|
103
103
|
if 'filter' in args and args.filter:
|
|
104
104
|
user_filter = data_models.Filter(args.filter, record_type).match
|
|
@@ -114,8 +114,8 @@ if __name__ == "__main__":
|
|
|
114
114
|
fields = set(record_type.get_fieldnames())
|
|
115
115
|
|
|
116
116
|
if 'group_by' in args and args.group_by:
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
if args.group_by not in fields:
|
|
118
|
+
raise ValueError("Group_by option %s not supported." % args.group_by)
|
|
119
119
|
fields = [args.group_by]
|
|
120
120
|
data_iter = data
|
|
121
121
|
data_dict = {}
|
|
@@ -126,11 +126,12 @@ if __name__ == "__main__":
|
|
|
126
126
|
data_dict[key].append(record)
|
|
127
127
|
|
|
128
128
|
if 'sum' in args and args.sum:
|
|
129
|
-
|
|
129
|
+
if args.sum not in set(record_type.get_fieldnames()):
|
|
130
|
+
raise ValueError("Sum option %s for record type %s not supported.\nChoice from %s" % (args.sum, args.record_type, set(record_type.get_fieldnames())))
|
|
130
131
|
fields.append(args.sum)
|
|
131
132
|
data = []
|
|
132
133
|
for key, value in data_dict.items():
|
|
133
|
-
total = sum(
|
|
134
|
+
total = sum(getattr(x, args.sum) for x in value)
|
|
134
135
|
# FIXME: Ugly hack to have some result
|
|
135
136
|
setattr(value[0], args.sum, total)
|
|
136
137
|
data.append(value[0])
|
|
@@ -139,12 +140,12 @@ if __name__ == "__main__":
|
|
|
139
140
|
|
|
140
141
|
if args.fields:
|
|
141
142
|
_show_fields = args.fields.split(',')
|
|
142
|
-
if not all(
|
|
143
|
+
if not all(f in fields for f in _show_fields):
|
|
143
144
|
error('Invalid field in --fields argument')
|
|
144
145
|
fields = _show_fields
|
|
145
146
|
elif args.hide:
|
|
146
147
|
_hide_fields = args.hide.split(',')
|
|
147
|
-
if not all(
|
|
148
|
+
if not all(f in fields for f in _hide_fields):
|
|
148
149
|
error('Invalid field in --hide argument')
|
|
149
150
|
fields = [f for f in fields if f not in _hide_fields]
|
|
150
151
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -114,11 +113,11 @@ Check the replicas for the DID mock:file::
|
|
|
114
113
|
from rucio.db.sqla import session, models
|
|
115
114
|
session.get_session().query(models.RSEFileAssociation).filter_by(name='file', scope='mock').first()
|
|
116
115
|
// [{'name': 'file','lock_cnt': 1, 'state': COPYING, 'scope': 'mock', 'rse_id': 'f81f366593754c01b0c340fa5ea0ab90'},
|
|
117
|
-
// {'scope': 'mock', 'rse_id': '1330d5daee37474c88ba888101d7b859', 'name': 'file', 'state':
|
|
116
|
+
// {'scope': 'mock', 'rse_id': '1330d5daee37474c88ba888101d7b859', 'name': 'file', 'state': AVAILABLE, 'lock_cnt': 1}]
|
|
118
117
|
|
|
119
118
|
The DID mock:file has now two replicas with one lock each.
|
|
120
119
|
As the file replica is attached to the dataset and the rule for the dataset specifies another RSE MOCK instead of the upload RSE, it has to be replicated to this RSE.
|
|
121
|
-
|
|
120
|
+
Therefore a second replica in state COPYING got created on RSE MOCK.
|
|
122
121
|
''', formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
123
122
|
parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
|
|
124
123
|
parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads for this process')
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -13,9 +12,8 @@
|
|
|
13
12
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
13
|
# See the License for the specific language governing permissions and
|
|
15
14
|
# limitations under the License.
|
|
16
|
-
|
|
17
15
|
"""
|
|
18
|
-
Kronos is a daemon
|
|
16
|
+
Kronos is a daemon that consumes tracer messages and updates the replica atime accordingly.
|
|
19
17
|
"""
|
|
20
18
|
|
|
21
19
|
import argparse
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -31,7 +30,7 @@ def get_parser():
|
|
|
31
30
|
parser = argparse.ArgumentParser(description="The Necromancer daemon is responsible for managing bad replicas. If a replica that got declared bad has other replicas, it will try to recover it by requesting a new transfer. If there are no replicas anymore, then the file gets marked as lost.", epilog='''
|
|
32
31
|
Lost replica:
|
|
33
32
|
In this example the file gets uploaded and will only have this replica as there are no replication rules. If it gets declared bad, there will be no replica to recover from.
|
|
34
|
-
|
|
33
|
+
Therefore the replica gets marked as lost.
|
|
35
34
|
|
|
36
35
|
Upload a file::
|
|
37
36
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -15,7 +14,7 @@
|
|
|
15
14
|
# limitations under the License.
|
|
16
15
|
|
|
17
16
|
"""
|
|
18
|
-
OAuth Manager is a daemon which is
|
|
17
|
+
OAuth Manager is a daemon which is responsible for:
|
|
19
18
|
- deletion of expired access tokens (in case there is a valid refresh token, expired access tokens will be kept until refresh_token expires as well.)
|
|
20
19
|
- deletion of expired OAuth session parameters
|
|
21
20
|
- refreshing access tokens via their refresh tokens.
|
|
@@ -33,7 +32,7 @@ def get_parser():
|
|
|
33
32
|
"""
|
|
34
33
|
parser = argparse.ArgumentParser(description='''
|
|
35
34
|
|
|
36
|
-
OAuth Manager is a daemon which is
|
|
35
|
+
OAuth Manager is a daemon which is responsible for:
|
|
37
36
|
- deletion of expired access tokens (in case there is a valid refresh token,
|
|
38
37
|
expired access tokens will be kept until refresh_token expires as well.)
|
|
39
38
|
- deletion of expired OAuth session parameters
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -68,7 +67,7 @@ $ id0=`uuidgen`
|
|
|
68
67
|
$ id1=`uuidgen`
|
|
69
68
|
$ id2=`uuidgen`
|
|
70
69
|
$ id3=`uuidgen`
|
|
71
|
-
$ echo "file available on MOCK_RECOVERY and
|
|
70
|
+
$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS (11 times)" > /tmp/file_available_suspicious'_'$id1
|
|
72
71
|
$ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS (11 times) and 1 time bad/deleted/lost on MOCK_SUSPICIOUS" > /tmp/file_available_suspicious_and_bad'_'$id2
|
|
73
72
|
$ echo "file declared as unavailable on MOCK_RECOVERY and declared as suspicious 11 times on MOCK_SUSPICIOUS" > /tmp/file_notavailable_suspicious'_'$id3
|
|
74
73
|
|
|
@@ -184,8 +183,8 @@ Terminal output:
|
|
|
184
183
|
|
|
185
184
|
2019-02-19 14:39:24,114 709 INFO replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
|
|
186
185
|
2019-02-19 14:39:24,124 709 INFO replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
|
|
187
|
-
2019-02-19 14:39:24,125 709 INFO replica_recoverer[0/0]: looking for replica
|
|
188
|
-
2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: found 1/1
|
|
186
|
+
2019-02-19 14:39:24,125 709 INFO replica_recoverer[0/0]: looking for replica pfns.
|
|
187
|
+
2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: found 1/1 pfns (took 0.035572052002 seconds) - declaring them as bad replicas now.
|
|
189
188
|
2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
|
|
190
189
|
2019-02-19 14:39:24,188 709 INFO replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
|
|
191
190
|
2019-02-19 14:39:24,192 709 INFO replica_recoverer[0/0]: graceful stop done
|
|
@@ -228,11 +227,11 @@ Note that attempting the use the ``--vos`` argument when in single-VO mode will
|
|
|
228
227
|
$ rucio-replica-recoverer --run-once --vos abc xyz
|
|
229
228
|
2020-07-28 15:21:33,349 5488 WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
|
|
230
229
|
''', formatter_class=argparse.RawDescriptionHelpFormatter) # NOQA: E501
|
|
231
|
-
parser.add_argument("--nattempts", action="store", default=
|
|
232
|
-
parser.add_argument("--younger-than", action="store", default=
|
|
230
|
+
parser.add_argument("--nattempts", action="store", default=5, type=int, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
|
|
231
|
+
parser.add_argument("--younger-than", action="store", default=5, type=int, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
|
|
233
232
|
parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
|
|
234
233
|
parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
|
|
235
|
-
parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
|
|
234
|
+
parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, type=int, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
|
|
236
235
|
parser.add_argument('--json-file-name', action="store", default="/opt/rucio/etc/suspicious_replica_recoverer.json", type=str, help='Name of the json file that that contains the policies which will be used by the suspicious replica recoverer.')
|
|
237
236
|
parser.add_argument('--sleep-time', action="store", default=3600, type=int, help='Concurrency control: Thread sleep time after each chunk of work.')
|
|
238
237
|
parser.add_argument('--active-mode', action="store_true", default=False, help='If NOT specified, the daemon will run without taking any actions on any files. In either case, the log file will be produced normally.')
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
#!python
|
|
2
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
"""
|
|
17
|
+
RSE-Decommissioner is a daemon that clears RSEs scheduled for decommissioning.
|
|
18
|
+
The actual operations (e.g. just delete all replicas, move the replicas to
|
|
19
|
+
specific RSEs, etc.) that are performed on the RSEs depend on the
|
|
20
|
+
"decommissioning profile", which must be specified for each RSE through RSE
|
|
21
|
+
attributes.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import signal
|
|
26
|
+
|
|
27
|
+
from rucio.daemons.rsedecommissioner.rse_decommissioner import run, stop
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_parser() -> argparse.ArgumentParser:
|
|
31
|
+
"""Construct and return the argparse parser."""
|
|
32
|
+
|
|
33
|
+
parser = argparse.ArgumentParser(
|
|
34
|
+
description=__doc__,
|
|
35
|
+
epilog="""\
|
|
36
|
+
To mark an RSE for decommissioning
|
|
37
|
+
----------------------------------
|
|
38
|
+
|
|
39
|
+
Clearing with the generic "delete all" profile:
|
|
40
|
+
|
|
41
|
+
$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
|
|
42
|
+
--value profile=generic_delete
|
|
43
|
+
|
|
44
|
+
Clearing with the generic "move all" profile:
|
|
45
|
+
|
|
46
|
+
$ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
|
|
47
|
+
--value profile=generic_move,destination=ANOTHER_RSE""",
|
|
48
|
+
formatter_class=argparse.RawDescriptionHelpFormatter
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
parser.add_argument('--run-once', dest='run_once', action='store_true',
|
|
52
|
+
help='One iteration only.')
|
|
53
|
+
parser.add_argument('--sleep-time', dest='sleep_time', action='store',
|
|
54
|
+
default=86400, type=int,
|
|
55
|
+
help='Concurrency control: thread sleep time after each'
|
|
56
|
+
' chunk of work')
|
|
57
|
+
return parser
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == '__main__':
|
|
61
|
+
signal.signal(signal.SIGTERM, stop)
|
|
62
|
+
args = get_parser().parse_args()
|
|
63
|
+
try:
|
|
64
|
+
run(once=args.run_once, sleep_time=args.sleep_time)
|
|
65
|
+
except KeyboardInterrupt:
|
|
66
|
+
stop()
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!python
|
|
2
|
-
# -*- coding: utf-8 -*-
|
|
3
2
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
4
3
|
#
|
|
5
4
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -28,7 +27,7 @@ def get_parser():
|
|
|
28
27
|
"""
|
|
29
28
|
Returns the argparse parser.
|
|
30
29
|
"""
|
|
31
|
-
parser = argparse.ArgumentParser(description="The Undertaker
|
|
30
|
+
parser = argparse.ArgumentParser(description="The Undertaker daemon is responsible for managing expired DIDs. It deletes DIDs, but not replicas by checking if there are DIDs where the 'expired_at' date property is older than the current timestamp.", epilog='''
|
|
32
31
|
Create a DID that is already expired by setting its lifetime to -1::
|
|
33
32
|
|
|
34
33
|
$ python
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rucio
|
|
3
|
+
Version: 35.8.0
|
|
4
|
+
Summary: Rucio Package
|
|
5
|
+
Home-page: https://rucio.cern.ch/
|
|
6
|
+
Author: Rucio
|
|
7
|
+
Author-email: rucio-dev@cern.ch
|
|
8
|
+
License: Apache License, Version 2.0
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Intended Audience :: Information Technology
|
|
12
|
+
Classifier: Intended Audience :: System Administrators
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Natural Language :: English
|
|
15
|
+
Classifier: Programming Language :: Python
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Environment :: No Input/Output (Daemon)
|
|
20
|
+
Requires-Python: >=3.9, <4
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
License-File: AUTHORS.rst
|
|
23
|
+
Requires-Dist: alembic==1.13.2
|
|
24
|
+
Requires-Dist: argcomplete==3.4.0
|
|
25
|
+
Requires-Dist: boto3==1.34.142
|
|
26
|
+
Requires-Dist: dogpile-cache==1.2.2
|
|
27
|
+
Requires-Dist: flask==3.0.3
|
|
28
|
+
Requires-Dist: geoip2==4.8.0
|
|
29
|
+
Requires-Dist: google-auth==2.32.0
|
|
30
|
+
Requires-Dist: jsonschema==4.23.0
|
|
31
|
+
Requires-Dist: oic==1.7.0
|
|
32
|
+
Requires-Dist: paramiko==3.4.1
|
|
33
|
+
Requires-Dist: prometheus-client==0.20.0
|
|
34
|
+
Requires-Dist: pymemcache==4.0.0
|
|
35
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
36
|
+
Requires-Dist: python-magic==0.4.27
|
|
37
|
+
Requires-Dist: redis==5.0.7
|
|
38
|
+
Requires-Dist: requests==2.32.3
|
|
39
|
+
Requires-Dist: sqlalchemy==2.0.31
|
|
40
|
+
Requires-Dist: statsd==4.0.1
|
|
41
|
+
Requires-Dist: stomp-py==8.1.2
|
|
42
|
+
Requires-Dist: tabulate==0.9.0
|
|
43
|
+
Requires-Dist: urllib3==1.26.19
|
|
44
|
+
Provides-Extra: oracle
|
|
45
|
+
Requires-Dist: cx-oracle==8.3.0; extra == "oracle"
|
|
46
|
+
Provides-Extra: mongo
|
|
47
|
+
Requires-Dist: pymongo==4.8.0; extra == "mongo"
|
|
48
|
+
Provides-Extra: postgresql
|
|
49
|
+
Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
|
|
50
|
+
Provides-Extra: mysql
|
|
51
|
+
Requires-Dist: pymysql==1.1.1; extra == "mysql"
|
|
52
|
+
Provides-Extra: kerberos
|
|
53
|
+
Requires-Dist: kerberos==1.3.1; extra == "kerberos"
|
|
54
|
+
Requires-Dist: pykerberos==1.2.4; extra == "kerberos"
|
|
55
|
+
Requires-Dist: requests-kerberos==0.15.0; extra == "kerberos"
|
|
56
|
+
Provides-Extra: globus
|
|
57
|
+
Requires-Dist: globus-sdk==3.41.0; extra == "globus"
|
|
58
|
+
Requires-Dist: pyyaml==6.0.1; extra == "globus"
|
|
59
|
+
Provides-Extra: saml
|
|
60
|
+
Requires-Dist: python3-saml==1.16.0; extra == "saml"
|
|
61
|
+
Provides-Extra: dev
|
|
62
|
+
Requires-Dist: pyyaml==6.0.1; extra == "dev"
|
|
63
|
+
Dynamic: author
|
|
64
|
+
Dynamic: author-email
|
|
65
|
+
Dynamic: classifier
|
|
66
|
+
Dynamic: home-page
|
|
67
|
+
Dynamic: license
|
|
68
|
+
Dynamic: license-file
|
|
69
|
+
Dynamic: provides-extra
|
|
70
|
+
Dynamic: requires-dist
|
|
71
|
+
Dynamic: requires-python
|
|
72
|
+
Dynamic: summary
|