rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +0 -1
- rucio/alembicrevision.py +1 -2
- rucio/client/__init__.py +0 -1
- rucio/client/accountclient.py +45 -25
- rucio/client/accountlimitclient.py +37 -9
- rucio/client/baseclient.py +199 -154
- rucio/client/client.py +2 -3
- rucio/client/configclient.py +19 -6
- rucio/client/credentialclient.py +9 -4
- rucio/client/didclient.py +238 -63
- rucio/client/diracclient.py +13 -5
- rucio/client/downloadclient.py +162 -51
- rucio/client/exportclient.py +4 -4
- rucio/client/fileclient.py +3 -4
- rucio/client/importclient.py +4 -4
- rucio/client/lifetimeclient.py +21 -5
- rucio/client/lockclient.py +18 -8
- rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
- rucio/client/pingclient.py +0 -1
- rucio/client/replicaclient.py +15 -5
- rucio/client/requestclient.py +35 -19
- rucio/client/rseclient.py +133 -51
- rucio/client/ruleclient.py +29 -22
- rucio/client/scopeclient.py +8 -6
- rucio/client/subscriptionclient.py +47 -35
- rucio/client/touchclient.py +8 -4
- rucio/client/uploadclient.py +166 -82
- rucio/common/__init__.py +0 -1
- rucio/common/cache.py +4 -4
- rucio/common/config.py +52 -47
- rucio/common/constants.py +69 -2
- rucio/common/constraints.py +0 -1
- rucio/common/didtype.py +24 -22
- rucio/common/dumper/__init__.py +70 -41
- rucio/common/dumper/consistency.py +26 -22
- rucio/common/dumper/data_models.py +16 -23
- rucio/common/dumper/path_parsing.py +0 -1
- rucio/common/exception.py +281 -222
- rucio/common/extra.py +0 -1
- rucio/common/logging.py +54 -38
- rucio/common/pcache.py +122 -101
- rucio/common/plugins.py +153 -0
- rucio/common/policy.py +4 -4
- rucio/common/schema/__init__.py +17 -10
- rucio/common/schema/atlas.py +7 -5
- rucio/common/schema/belleii.py +7 -5
- rucio/common/schema/domatpc.py +7 -5
- rucio/common/schema/escape.py +7 -5
- rucio/common/schema/generic.py +8 -6
- rucio/common/schema/generic_multi_vo.py +7 -5
- rucio/common/schema/icecube.py +7 -5
- rucio/common/stomp_utils.py +0 -1
- rucio/common/stopwatch.py +0 -1
- rucio/common/test_rucio_server.py +2 -2
- rucio/common/types.py +262 -17
- rucio/common/utils.py +743 -451
- rucio/core/__init__.py +0 -1
- rucio/core/account.py +99 -29
- rucio/core/account_counter.py +89 -24
- rucio/core/account_limit.py +90 -24
- rucio/core/authentication.py +86 -29
- rucio/core/config.py +108 -38
- rucio/core/credential.py +14 -7
- rucio/core/did.py +680 -782
- rucio/core/did_meta_plugins/__init__.py +8 -6
- rucio/core/did_meta_plugins/did_column_meta.py +17 -12
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
- rucio/core/did_meta_plugins/filter_engine.py +90 -50
- rucio/core/did_meta_plugins/json_meta.py +41 -16
- rucio/core/did_meta_plugins/mongo_meta.py +25 -8
- rucio/core/did_meta_plugins/postgres_meta.py +3 -4
- rucio/core/dirac.py +46 -17
- rucio/core/distance.py +66 -43
- rucio/core/exporter.py +5 -5
- rucio/core/heartbeat.py +181 -81
- rucio/core/identity.py +22 -12
- rucio/core/importer.py +23 -12
- rucio/core/lifetime_exception.py +32 -32
- rucio/core/lock.py +244 -142
- rucio/core/message.py +79 -38
- rucio/core/{meta.py → meta_conventions.py} +57 -44
- rucio/core/monitor.py +19 -13
- rucio/core/naming_convention.py +68 -27
- rucio/core/nongrid_trace.py +17 -5
- rucio/core/oidc.py +151 -29
- rucio/core/permission/__init__.py +18 -6
- rucio/core/permission/atlas.py +50 -35
- rucio/core/permission/belleii.py +6 -5
- rucio/core/permission/escape.py +8 -6
- rucio/core/permission/generic.py +82 -80
- rucio/core/permission/generic_multi_vo.py +9 -7
- rucio/core/quarantined_replica.py +91 -58
- rucio/core/replica.py +1303 -772
- rucio/core/replica_sorter.py +10 -12
- rucio/core/request.py +1133 -285
- rucio/core/rse.py +142 -102
- rucio/core/rse_counter.py +49 -18
- rucio/core/rse_expression_parser.py +6 -7
- rucio/core/rse_selector.py +41 -16
- rucio/core/rule.py +1538 -474
- rucio/core/rule_grouping.py +213 -68
- rucio/core/scope.py +50 -22
- rucio/core/subscription.py +92 -44
- rucio/core/topology.py +66 -24
- rucio/core/trace.py +42 -28
- rucio/core/transfer.py +543 -259
- rucio/core/vo.py +36 -18
- rucio/core/volatile_replica.py +59 -32
- rucio/daemons/__init__.py +0 -1
- rucio/daemons/abacus/__init__.py +0 -1
- rucio/daemons/abacus/account.py +29 -19
- rucio/daemons/abacus/collection_replica.py +21 -10
- rucio/daemons/abacus/rse.py +22 -12
- rucio/daemons/atropos/__init__.py +0 -1
- rucio/daemons/atropos/atropos.py +1 -2
- rucio/daemons/auditor/__init__.py +56 -28
- rucio/daemons/auditor/hdfs.py +17 -6
- rucio/daemons/auditor/srmdumps.py +116 -45
- rucio/daemons/automatix/__init__.py +0 -1
- rucio/daemons/automatix/automatix.py +30 -18
- rucio/daemons/badreplicas/__init__.py +0 -1
- rucio/daemons/badreplicas/minos.py +29 -18
- rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
- rucio/daemons/badreplicas/necromancer.py +9 -13
- rucio/daemons/bb8/__init__.py +0 -1
- rucio/daemons/bb8/bb8.py +10 -13
- rucio/daemons/bb8/common.py +151 -154
- rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
- rucio/daemons/bb8/t2_background_rebalance.py +15 -8
- rucio/daemons/c3po/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/__init__.py +0 -1
- rucio/daemons/c3po/algorithms/simple.py +8 -5
- rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
- rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
- rucio/daemons/c3po/c3po.py +81 -52
- rucio/daemons/c3po/collectors/__init__.py +0 -1
- rucio/daemons/c3po/collectors/agis.py +17 -17
- rucio/daemons/c3po/collectors/free_space.py +32 -13
- rucio/daemons/c3po/collectors/jedi_did.py +14 -5
- rucio/daemons/c3po/collectors/mock_did.py +11 -6
- rucio/daemons/c3po/collectors/network_metrics.py +12 -4
- rucio/daemons/c3po/collectors/workload.py +21 -19
- rucio/daemons/c3po/utils/__init__.py +0 -1
- rucio/daemons/c3po/utils/dataset_cache.py +15 -5
- rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
- rucio/daemons/c3po/utils/expiring_list.py +6 -7
- rucio/daemons/c3po/utils/popularity.py +5 -2
- rucio/daemons/c3po/utils/timeseries.py +25 -12
- rucio/daemons/cache/__init__.py +0 -1
- rucio/daemons/cache/consumer.py +21 -15
- rucio/daemons/common.py +42 -18
- rucio/daemons/conveyor/__init__.py +0 -1
- rucio/daemons/conveyor/common.py +69 -37
- rucio/daemons/conveyor/finisher.py +83 -46
- rucio/daemons/conveyor/poller.py +101 -69
- rucio/daemons/conveyor/preparer.py +35 -28
- rucio/daemons/conveyor/receiver.py +64 -21
- rucio/daemons/conveyor/stager.py +33 -28
- rucio/daemons/conveyor/submitter.py +71 -47
- rucio/daemons/conveyor/throttler.py +99 -35
- rucio/daemons/follower/__init__.py +0 -1
- rucio/daemons/follower/follower.py +12 -8
- rucio/daemons/hermes/__init__.py +0 -1
- rucio/daemons/hermes/hermes.py +57 -21
- rucio/daemons/judge/__init__.py +0 -1
- rucio/daemons/judge/cleaner.py +27 -17
- rucio/daemons/judge/evaluator.py +31 -18
- rucio/daemons/judge/injector.py +31 -23
- rucio/daemons/judge/repairer.py +28 -18
- rucio/daemons/oauthmanager/__init__.py +0 -1
- rucio/daemons/oauthmanager/oauthmanager.py +7 -8
- rucio/daemons/reaper/__init__.py +0 -1
- rucio/daemons/reaper/dark_reaper.py +15 -9
- rucio/daemons/reaper/reaper.py +109 -67
- rucio/daemons/replicarecoverer/__init__.py +0 -1
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
- rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +0 -1
- rucio/daemons/storage/consistency/__init__.py +0 -1
- rucio/daemons/storage/consistency/actions.py +152 -59
- rucio/daemons/tracer/__init__.py +0 -1
- rucio/daemons/tracer/kronos.py +47 -24
- rucio/daemons/transmogrifier/__init__.py +0 -1
- rucio/daemons/transmogrifier/transmogrifier.py +35 -26
- rucio/daemons/undertaker/__init__.py +0 -1
- rucio/daemons/undertaker/undertaker.py +10 -10
- rucio/db/__init__.py +0 -1
- rucio/db/sqla/__init__.py +16 -2
- rucio/db/sqla/constants.py +10 -1
- rucio/db/sqla/migrate_repo/__init__.py +0 -1
- rucio/db/sqla/migrate_repo/env.py +0 -1
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
- rucio/db/sqla/models.py +122 -216
- rucio/db/sqla/sautils.py +12 -5
- rucio/db/sqla/session.py +71 -43
- rucio/db/sqla/types.py +3 -4
- rucio/db/sqla/util.py +91 -69
- rucio/gateway/__init__.py +13 -0
- rucio/{api → gateway}/account.py +119 -46
- rucio/{api → gateway}/account_limit.py +12 -13
- rucio/{api → gateway}/authentication.py +106 -33
- rucio/{api → gateway}/config.py +12 -13
- rucio/{api → gateway}/credential.py +15 -4
- rucio/{api → gateway}/did.py +384 -140
- rucio/{api → gateway}/dirac.py +16 -6
- rucio/{api → gateway}/exporter.py +3 -4
- rucio/{api → gateway}/heartbeat.py +17 -5
- rucio/{api → gateway}/identity.py +63 -19
- rucio/{api → gateway}/importer.py +3 -4
- rucio/{api → gateway}/lifetime_exception.py +35 -10
- rucio/{api → gateway}/lock.py +34 -12
- rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
- rucio/{api → gateway}/permission.py +4 -5
- rucio/{api → gateway}/quarantined_replica.py +13 -4
- rucio/{api → gateway}/replica.py +12 -11
- rucio/{api → gateway}/request.py +129 -28
- rucio/{api → gateway}/rse.py +11 -12
- rucio/{api → gateway}/rule.py +117 -35
- rucio/{api → gateway}/scope.py +24 -14
- rucio/{api → gateway}/subscription.py +65 -43
- rucio/{api → gateway}/vo.py +17 -7
- rucio/rse/__init__.py +3 -4
- rucio/rse/protocols/__init__.py +0 -1
- rucio/rse/protocols/bittorrent.py +184 -0
- rucio/rse/protocols/cache.py +1 -2
- rucio/rse/protocols/dummy.py +1 -2
- rucio/rse/protocols/gfal.py +12 -10
- rucio/rse/protocols/globus.py +7 -7
- rucio/rse/protocols/gsiftp.py +2 -3
- rucio/rse/protocols/http_cache.py +1 -2
- rucio/rse/protocols/mock.py +1 -2
- rucio/rse/protocols/ngarc.py +1 -2
- rucio/rse/protocols/posix.py +12 -13
- rucio/rse/protocols/protocol.py +116 -52
- rucio/rse/protocols/rclone.py +6 -7
- rucio/rse/protocols/rfio.py +4 -5
- rucio/rse/protocols/srm.py +9 -10
- rucio/rse/protocols/ssh.py +8 -9
- rucio/rse/protocols/storm.py +2 -3
- rucio/rse/protocols/webdav.py +17 -14
- rucio/rse/protocols/xrootd.py +23 -17
- rucio/rse/rsemanager.py +19 -7
- rucio/tests/__init__.py +0 -1
- rucio/tests/common.py +43 -17
- rucio/tests/common_server.py +3 -3
- rucio/transfertool/__init__.py +0 -1
- rucio/transfertool/bittorrent.py +199 -0
- rucio/transfertool/bittorrent_driver.py +52 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
- rucio/transfertool/fts3.py +250 -138
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +9 -8
- rucio/transfertool/globus_library.py +1 -2
- rucio/transfertool/mock.py +21 -12
- rucio/transfertool/transfertool.py +33 -24
- rucio/vcsversion.py +4 -4
- rucio/version.py +5 -13
- rucio/web/__init__.py +0 -1
- rucio/web/rest/__init__.py +0 -1
- rucio/web/rest/flaskapi/__init__.py +0 -1
- rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
- rucio/web/rest/flaskapi/v1/__init__.py +0 -1
- rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
- rucio/web/rest/flaskapi/v1/accounts.py +49 -48
- rucio/web/rest/flaskapi/v1/archives.py +12 -10
- rucio/web/rest/flaskapi/v1/auth.py +146 -144
- rucio/web/rest/flaskapi/v1/common.py +82 -41
- rucio/web/rest/flaskapi/v1/config.py +5 -6
- rucio/web/rest/flaskapi/v1/credentials.py +7 -8
- rucio/web/rest/flaskapi/v1/dids.py +158 -28
- rucio/web/rest/flaskapi/v1/dirac.py +8 -8
- rucio/web/rest/flaskapi/v1/export.py +3 -5
- rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
- rucio/web/rest/flaskapi/v1/identities.py +3 -5
- rucio/web/rest/flaskapi/v1/import.py +3 -4
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
- rucio/web/rest/flaskapi/v1/locks.py +2 -4
- rucio/web/rest/flaskapi/v1/main.py +10 -2
- rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
- rucio/web/rest/flaskapi/v1/metrics.py +1 -2
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
- rucio/web/rest/flaskapi/v1/ping.py +6 -7
- rucio/web/rest/flaskapi/v1/redirect.py +8 -9
- rucio/web/rest/flaskapi/v1/replicas.py +43 -19
- rucio/web/rest/flaskapi/v1/requests.py +178 -21
- rucio/web/rest/flaskapi/v1/rses.py +61 -26
- rucio/web/rest/flaskapi/v1/rules.py +48 -18
- rucio/web/rest/flaskapi/v1/scopes.py +3 -5
- rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
- rucio/web/rest/flaskapi/v1/traces.py +4 -4
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +3 -5
- rucio/web/rest/main.py +0 -1
- rucio/web/rest/metrics.py +0 -1
- rucio/web/rest/ping.py +27 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
- rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
- rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
- {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
- rucio-35.8.0.dist-info/METADATA +72 -0
- rucio-35.8.0.dist-info/RECORD +493 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
- rucio/api/temporary_did.py +0 -49
- rucio/common/schema/cms.py +0 -478
- rucio/common/schema/lsst.py +0 -423
- rucio/core/permission/cms.py +0 -1166
- rucio/core/temporary_did.py +0 -188
- rucio/daemons/reaper/light_reaper.py +0 -255
- rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
- rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
- rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
- rucio-32.8.6.dist-info/METADATA +0 -83
- rucio-32.8.6.dist-info/RECORD +0 -481
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
- {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
- {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -18,27 +17,37 @@ import glob
|
|
|
18
17
|
import logging
|
|
19
18
|
import os
|
|
20
19
|
import queue as Queue
|
|
21
|
-
from datetime import datetime
|
|
22
|
-
from datetime import timedelta
|
|
23
|
-
|
|
24
20
|
import select
|
|
21
|
+
from datetime import datetime, timedelta
|
|
22
|
+
from typing import TYPE_CHECKING, Optional
|
|
25
23
|
|
|
26
24
|
from rucio.common import config
|
|
27
|
-
from rucio.common.dumper import LogPipeHandler
|
|
28
|
-
from rucio.common.dumper import mkdir
|
|
29
|
-
from rucio.common.dumper import temp_file
|
|
25
|
+
from rucio.common.dumper import LogPipeHandler, mkdir, temp_file
|
|
30
26
|
from rucio.common.dumper.consistency import Consistency
|
|
31
27
|
from rucio.common.types import InternalAccount, InternalScope
|
|
32
28
|
from rucio.common.utils import chunks
|
|
33
29
|
from rucio.core.quarantined_replica import add_quarantined_replicas
|
|
34
30
|
from rucio.core.replica import declare_bad_file_replicas, list_replicas
|
|
35
|
-
from rucio.core.rse import
|
|
31
|
+
from rucio.core.rse import get_rse_id, get_rse_usage
|
|
36
32
|
from rucio.daemons.auditor import srmdumps
|
|
37
33
|
from rucio.daemons.auditor.hdfs import ReplicaFromHDFS
|
|
38
34
|
from rucio.db.sqla.constants import BadFilesStatus
|
|
39
35
|
|
|
40
|
-
|
|
41
|
-
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from collections.abc import Iterable
|
|
38
|
+
from configparser import RawConfigParser
|
|
39
|
+
from multiprocessing import Queue as QueueType
|
|
40
|
+
from multiprocessing.connection import Connection
|
|
41
|
+
from multiprocessing.synchronize import Event
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def consistency(
|
|
45
|
+
rse: str,
|
|
46
|
+
delta: timedelta,
|
|
47
|
+
configuration: "RawConfigParser",
|
|
48
|
+
cache_dir: str,
|
|
49
|
+
results_dir: str
|
|
50
|
+
) -> Optional[str]:
|
|
42
51
|
logger = logging.getLogger('auditor-worker')
|
|
43
52
|
rsedump, rsedate = srmdumps.download_rse_dump(rse, configuration, destdir=cache_dir)
|
|
44
53
|
results_path = os.path.join(results_dir, '{0}_{1}'.format(rse, rsedate.strftime('%Y%m%d'))) # pylint: disable=no-member
|
|
@@ -66,11 +75,12 @@ def consistency(rse, delta, configuration, cache_dir, results_dir):
|
|
|
66
75
|
return results_path
|
|
67
76
|
|
|
68
77
|
|
|
69
|
-
def guess_replica_info(
|
|
78
|
+
def guess_replica_info(
|
|
79
|
+
path: str
|
|
80
|
+
) -> tuple[Optional[str], str]:
|
|
70
81
|
"""Try to extract the scope and name from a path.
|
|
71
82
|
|
|
72
|
-
``path
|
|
73
|
-
the RSE.
|
|
83
|
+
``path``: relative path to the file on the RSE.
|
|
74
84
|
|
|
75
85
|
Returns a ``tuple`` of which the first element is the scope of the
|
|
76
86
|
replica and the second element is its name.
|
|
@@ -84,7 +94,10 @@ def guess_replica_info(path):
|
|
|
84
94
|
return items[0], items[-1]
|
|
85
95
|
|
|
86
96
|
|
|
87
|
-
def bz2_compress_file(
|
|
97
|
+
def bz2_compress_file(
|
|
98
|
+
source: str,
|
|
99
|
+
chunk_size: int = 65000
|
|
100
|
+
) -> str:
|
|
88
101
|
"""Compress a file with bzip2.
|
|
89
102
|
|
|
90
103
|
The destination is the path passed through ``source`` extended with
|
|
@@ -93,13 +106,11 @@ def bz2_compress_file(source, chunk_size=65000):
|
|
|
93
106
|
Errors are deliberately not handled gracefully. Any exceptions
|
|
94
107
|
should be propagated to the caller.
|
|
95
108
|
|
|
96
|
-
``source
|
|
97
|
-
to compress.
|
|
109
|
+
``source``: absolute path to the file to compress.
|
|
98
110
|
|
|
99
|
-
``chunk_size
|
|
100
|
-
chunks by which to read the file.
|
|
111
|
+
``chunk_size``: size (in bytes) of the chunks by which to read the file.
|
|
101
112
|
|
|
102
|
-
Returns
|
|
113
|
+
Returns the destination path.
|
|
103
114
|
"""
|
|
104
115
|
destination = '{}.bz2'.format(source)
|
|
105
116
|
with open(source) as plain, bz2.BZ2File(destination, 'w') as compressed:
|
|
@@ -112,14 +123,18 @@ def bz2_compress_file(source, chunk_size=65000):
|
|
|
112
123
|
return destination
|
|
113
124
|
|
|
114
125
|
|
|
115
|
-
def process_output(
|
|
126
|
+
def process_output(
|
|
127
|
+
output: str,
|
|
128
|
+
sanity_check: bool = True,
|
|
129
|
+
compress: bool = True
|
|
130
|
+
) -> None:
|
|
116
131
|
"""Perform post-consistency-check actions.
|
|
117
132
|
|
|
118
133
|
DARK files are put in the quarantined-replica table so that they
|
|
119
134
|
may be deleted by the Dark Reaper. LOST files are reported as
|
|
120
135
|
suspicious so that they may be further checked by the cloud squads.
|
|
121
136
|
|
|
122
|
-
``output
|
|
137
|
+
``output``: absolute path to the file
|
|
123
138
|
produced by ``consistency()``. It must maintain its naming
|
|
124
139
|
convention.
|
|
125
140
|
|
|
@@ -155,7 +170,7 @@ def process_output(output, sanity_check=True, compress=True):
|
|
|
155
170
|
rse = os.path.basename(output[:output.rfind('_')])
|
|
156
171
|
rse_id = get_rse_id(rse=rse)
|
|
157
172
|
usage = get_rse_usage(rse_id=rse_id, source='rucio')[0]
|
|
158
|
-
threshold = config.config_get_float('auditor', 'threshold', False, 0.
|
|
173
|
+
threshold = config.config_get_float('auditor', 'threshold', False, 0.1)
|
|
159
174
|
|
|
160
175
|
# Perform a basic sanity check by comparing the number of entries
|
|
161
176
|
# with the total number of files on the RSE. If the percentage is
|
|
@@ -190,7 +205,16 @@ def process_output(output, sanity_check=True, compress=True):
|
|
|
190
205
|
logger.debug('Compressed "%s"', destination)
|
|
191
206
|
|
|
192
207
|
|
|
193
|
-
def check(
|
|
208
|
+
def check(
|
|
209
|
+
queue: "QueueType",
|
|
210
|
+
retry: "QueueType",
|
|
211
|
+
terminate: "Event",
|
|
212
|
+
logpipe: "Connection",
|
|
213
|
+
cache_dir: str,
|
|
214
|
+
results_dir: str,
|
|
215
|
+
keep_dumps: bool,
|
|
216
|
+
delta_in_days: int
|
|
217
|
+
) -> None:
|
|
194
218
|
logger = logging.getLogger('auditor-worker')
|
|
195
219
|
lib_logger = logging.getLogger('dumper')
|
|
196
220
|
|
|
@@ -213,7 +237,7 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
|
|
|
213
237
|
|
|
214
238
|
while not terminate.is_set():
|
|
215
239
|
try:
|
|
216
|
-
rse,
|
|
240
|
+
rse, attempts = queue.get(timeout=30)
|
|
217
241
|
except Queue.Empty:
|
|
218
242
|
continue
|
|
219
243
|
start = datetime.now()
|
|
@@ -225,7 +249,7 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
|
|
|
225
249
|
process_output(output)
|
|
226
250
|
except:
|
|
227
251
|
elapsed = (datetime.now() - start).total_seconds() / 60
|
|
228
|
-
logger.error('Check of "%s" failed in %d minutes, %d remaining
|
|
252
|
+
logger.error('Check of "%s" failed in %d minutes, %d remaining attempts', rse, elapsed, attempts, exc_info=True)
|
|
229
253
|
success = False
|
|
230
254
|
else:
|
|
231
255
|
elapsed = (datetime.now() - start).total_seconds() / 60
|
|
@@ -239,11 +263,15 @@ def check(queue, retry, terminate, logpipe, cache_dir, results_dir, keep_dumps,
|
|
|
239
263
|
for fil in remove:
|
|
240
264
|
os.remove(fil)
|
|
241
265
|
|
|
242
|
-
if not success and
|
|
243
|
-
retry.put((rse,
|
|
266
|
+
if not success and attempts > 0:
|
|
267
|
+
retry.put((rse, attempts - 1))
|
|
244
268
|
|
|
245
269
|
|
|
246
|
-
def activity_logger(
|
|
270
|
+
def activity_logger(
|
|
271
|
+
logpipes: "Iterable[Connection]",
|
|
272
|
+
logfilename: str,
|
|
273
|
+
terminate: "Event"
|
|
274
|
+
) -> None:
|
|
247
275
|
handler = logging.handlers.RotatingFileHandler(
|
|
248
276
|
logfilename,
|
|
249
277
|
maxBytes=20971520,
|
rucio/daemons/auditor/hdfs.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -20,13 +19,19 @@ import re
|
|
|
20
19
|
import shutil
|
|
21
20
|
import subprocess
|
|
22
21
|
import tempfile
|
|
22
|
+
from typing import TYPE_CHECKING
|
|
23
23
|
|
|
24
|
-
from rucio.common.dumper import DUMPS_CACHE_DIR
|
|
25
|
-
from rucio.common.dumper import temp_file
|
|
24
|
+
from rucio.common.dumper import DUMPS_CACHE_DIR, temp_file
|
|
26
25
|
from rucio.common.dumper.data_models import Replica
|
|
27
26
|
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from datetime import datetime
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
|
|
31
|
+
def _hdfs_get(
|
|
32
|
+
src_url: str,
|
|
33
|
+
dst_path: str
|
|
34
|
+
) -> None:
|
|
30
35
|
cmd = ['hadoop', 'fs', '-get', src_url, dst_path]
|
|
31
36
|
get = subprocess.Popen(
|
|
32
37
|
cmd,
|
|
@@ -34,7 +39,7 @@ def _hdfs_get(src_url, dst_path):
|
|
|
34
39
|
)
|
|
35
40
|
_, stderr = get.communicate()
|
|
36
41
|
if get.returncode != 0:
|
|
37
|
-
raise
|
|
42
|
+
raise OSError('_hdfs_get(): "{0}": {1}. Return code {2}'.format(
|
|
38
43
|
' '.join(cmd),
|
|
39
44
|
stderr,
|
|
40
45
|
get.returncode,
|
|
@@ -45,7 +50,13 @@ class ReplicaFromHDFS(Replica):
|
|
|
45
50
|
BASE_URL = '/user/rucio01/reports/{0}/replicas_per_rse/{1}/*'
|
|
46
51
|
|
|
47
52
|
@classmethod
|
|
48
|
-
def download(
|
|
53
|
+
def download(
|
|
54
|
+
cls,
|
|
55
|
+
rse: str,
|
|
56
|
+
date: "datetime",
|
|
57
|
+
cache_dir: str = DUMPS_CACHE_DIR,
|
|
58
|
+
buffer_size: int = 65536
|
|
59
|
+
) -> str:
|
|
49
60
|
logger = logging.getLogger('auditor.hdfs')
|
|
50
61
|
|
|
51
62
|
if not os.path.isdir(cache_dir):
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -22,32 +21,53 @@ import logging
|
|
|
22
21
|
import operator
|
|
23
22
|
import os
|
|
24
23
|
import re
|
|
24
|
+
from typing import IO, TYPE_CHECKING, Any, Optional
|
|
25
25
|
|
|
26
26
|
import gfal2
|
|
27
27
|
import requests
|
|
28
28
|
|
|
29
29
|
from rucio.common.config import get_config_dirs
|
|
30
|
-
from rucio.common.
|
|
31
|
-
from rucio.common.dumper import
|
|
30
|
+
from rucio.common.constants import RseAttr
|
|
31
|
+
from rucio.common.dumper import DUMPS_CACHE_DIR, HTTPDownloadFailed, ddmendpoint_url, gfal_download_to_file, http_download_to_file, temp_file
|
|
32
|
+
from rucio.core.credential import get_signed_url
|
|
33
|
+
from rucio.core.rse import get_rse_id, list_rse_attributes
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from collections.abc import Iterable
|
|
32
37
|
|
|
33
38
|
CHUNK_SIZE = 10485760
|
|
34
39
|
|
|
35
|
-
__DUMPERCONFIGDIRS = (
|
|
36
|
-
|
|
40
|
+
__DUMPERCONFIGDIRS = list(
|
|
41
|
+
filter(
|
|
42
|
+
os.path.exists,
|
|
43
|
+
(
|
|
44
|
+
os.path.join(confdir, 'auditor') for confdir in get_config_dirs()
|
|
45
|
+
)
|
|
46
|
+
)
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
OBJECTSTORE_NUM_TRIES = 30
|
|
37
50
|
|
|
38
51
|
|
|
39
|
-
class Parser(ConfigParser.RawConfigParser
|
|
52
|
+
class Parser(ConfigParser.RawConfigParser):
|
|
40
53
|
'''
|
|
41
54
|
RawConfigParser subclass that doesn't modify the the name of the options
|
|
42
|
-
and removes any quotes
|
|
55
|
+
and removes any quotes around the string values.
|
|
43
56
|
'''
|
|
44
57
|
remove_quotes_re = re.compile(r"^'(.+)'$")
|
|
45
58
|
remove_double_quotes_re = re.compile(r'^"(.+)"$')
|
|
46
59
|
|
|
47
|
-
def optionxform(
|
|
60
|
+
def optionxform(
|
|
61
|
+
self,
|
|
62
|
+
optionstr: str
|
|
63
|
+
) -> str:
|
|
48
64
|
return optionstr
|
|
49
65
|
|
|
50
|
-
def get(
|
|
66
|
+
def get(
|
|
67
|
+
self,
|
|
68
|
+
section: str,
|
|
69
|
+
option: str
|
|
70
|
+
) -> Any:
|
|
51
71
|
value = super(Parser, self).get(section, option)
|
|
52
72
|
if isinstance(value, str):
|
|
53
73
|
value = self.remove_quotes_re.sub(r'\1', value)
|
|
@@ -58,18 +78,23 @@ class Parser(ConfigParser.RawConfigParser, object):
|
|
|
58
78
|
return [(name, self.get(section, name)) for name in self.options(section)]
|
|
59
79
|
|
|
60
80
|
|
|
61
|
-
def mkdir(dir_):
|
|
81
|
+
def mkdir(dir_: str) -> None:
|
|
62
82
|
'''
|
|
63
83
|
This functions creates the `dir` directory if it doesn't exist. If `dir`
|
|
64
84
|
already exists this function does nothing.
|
|
65
85
|
'''
|
|
66
86
|
try:
|
|
67
87
|
os.mkdir(dir_)
|
|
68
|
-
except OSError as
|
|
69
|
-
|
|
88
|
+
except OSError as error:
|
|
89
|
+
if error.errno != 17:
|
|
90
|
+
raise error
|
|
70
91
|
|
|
71
92
|
|
|
72
|
-
def get_newest(
|
|
93
|
+
def get_newest(
|
|
94
|
+
base_url: str,
|
|
95
|
+
url_pattern: str,
|
|
96
|
+
links: "Iterable[str]"
|
|
97
|
+
) -> tuple[str, datetime.datetime]:
|
|
73
98
|
'''
|
|
74
99
|
Returns a tuple with the newest url in the `links` list matching the
|
|
75
100
|
pattern `url_pattern` and a datetime object representing the creation
|
|
@@ -104,7 +129,7 @@ def get_newest(base_url, url_pattern, links):
|
|
|
104
129
|
return max(times, key=operator.itemgetter(1))
|
|
105
130
|
|
|
106
131
|
|
|
107
|
-
def gfal_links(base_url):
|
|
132
|
+
def gfal_links(base_url: str) -> list[str]:
|
|
108
133
|
'''
|
|
109
134
|
Returns a list of the urls contained in `base_url`.
|
|
110
135
|
'''
|
|
@@ -112,19 +137,22 @@ def gfal_links(base_url):
|
|
|
112
137
|
return ['/'.join((base_url, f)) for f in ctxt.listdir(str(base_url))]
|
|
113
138
|
|
|
114
139
|
|
|
115
|
-
class _LinkCollector(HTMLParser.HTMLParser
|
|
140
|
+
class _LinkCollector(HTMLParser.HTMLParser):
|
|
116
141
|
def __init__(self):
|
|
117
142
|
super(_LinkCollector, self).__init__()
|
|
118
143
|
self.links = []
|
|
119
144
|
|
|
120
|
-
def handle_starttag(
|
|
145
|
+
def handle_starttag(
|
|
146
|
+
self, tag: str,
|
|
147
|
+
attrs: "Iterable[tuple[str, str]]"
|
|
148
|
+
) -> None:
|
|
121
149
|
if tag == 'a':
|
|
122
150
|
self.links.append(
|
|
123
151
|
next(value for key, value in attrs if key == 'href')
|
|
124
152
|
)
|
|
125
153
|
|
|
126
154
|
|
|
127
|
-
def http_links(base_url):
|
|
155
|
+
def http_links(base_url: str) -> list[str]:
|
|
128
156
|
'''
|
|
129
157
|
Returns a list of the urls contained in `base_url`.
|
|
130
158
|
'''
|
|
@@ -169,7 +197,7 @@ protocol_funcs = {
|
|
|
169
197
|
}
|
|
170
198
|
|
|
171
199
|
|
|
172
|
-
def protocol(url):
|
|
200
|
+
def protocol(url: str) -> str:
|
|
173
201
|
'''
|
|
174
202
|
Given the URL `url` returns a string with the protocol part.
|
|
175
203
|
'''
|
|
@@ -180,25 +208,26 @@ def protocol(url):
|
|
|
180
208
|
return proto
|
|
181
209
|
|
|
182
210
|
|
|
183
|
-
def get_links(base_url):
|
|
211
|
+
def get_links(base_url: str) -> list[str]:
|
|
184
212
|
'''
|
|
185
213
|
Given the URL `base_url` returns the URLs linked or contained in it.
|
|
186
214
|
'''
|
|
187
215
|
return protocol_funcs[protocol(base_url)]['links'](base_url)
|
|
188
216
|
|
|
189
217
|
|
|
190
|
-
def download(url, filename):
|
|
218
|
+
def download(url: str, filename: IO) -> None:
|
|
191
219
|
'''
|
|
192
220
|
Given the URL `url` downloads its contents on `filename`.
|
|
193
221
|
'''
|
|
194
222
|
return protocol_funcs[protocol(url)]['download'](url, filename)
|
|
195
223
|
|
|
196
224
|
|
|
197
|
-
def parse_configuration(conf_dirs=
|
|
225
|
+
def parse_configuration(conf_dirs: Optional[list[str]] = None) -> Parser:
|
|
198
226
|
'''
|
|
199
227
|
Parses the configuration for the endpoints contained in `conf_dir`.
|
|
200
228
|
Returns a ConfParser.RawConfParser subclass instance.
|
|
201
229
|
'''
|
|
230
|
+
conf_dirs = conf_dirs or __DUMPERCONFIGDIRS
|
|
202
231
|
logger = logging.getLogger('auditor.srmdumps')
|
|
203
232
|
if len(conf_dirs) == 0:
|
|
204
233
|
logger.error('No configuration directory given to load SRM dumps paths')
|
|
@@ -213,18 +242,23 @@ def parse_configuration(conf_dirs=__DUMPERCONFIGDIRS):
|
|
|
213
242
|
return configuration
|
|
214
243
|
|
|
215
244
|
|
|
216
|
-
def download_rse_dump(
|
|
245
|
+
def download_rse_dump(
|
|
246
|
+
rse: str,
|
|
247
|
+
configuration: ConfigParser.RawConfigParser,
|
|
248
|
+
date: Optional[datetime.datetime] = None,
|
|
249
|
+
destdir: str = DUMPS_CACHE_DIR
|
|
250
|
+
) -> tuple[str, datetime.datetime]:
|
|
217
251
|
'''
|
|
218
252
|
Downloads the dump for the given ddmendpoint. If this endpoint does not
|
|
219
|
-
follow the
|
|
253
|
+
follow the standardized method to publish the dumps it should have an
|
|
220
254
|
entry in the `configuration` object describing how to download the dump.
|
|
221
255
|
|
|
222
256
|
`rse` is the DDMEndpoint name.
|
|
223
257
|
|
|
224
258
|
`configuration` is a RawConfigParser subclass.
|
|
225
259
|
|
|
226
|
-
`date` is a datetime instance with the date of the desired dump or
|
|
227
|
-
to download the
|
|
260
|
+
`date` is a datetime instance with the date of the desired dump or None
|
|
261
|
+
to download the latest available dump.
|
|
228
262
|
|
|
229
263
|
`destdir` is the directory where the dump will be saved (the final component
|
|
230
264
|
in the path is created if it doesn't exist).
|
|
@@ -234,34 +268,71 @@ def download_rse_dump(rse, configuration, date='latest', destdir=DUMPS_CACHE_DIR
|
|
|
234
268
|
'''
|
|
235
269
|
logger = logging.getLogger('auditor.srmdumps')
|
|
236
270
|
base_url, url_pattern = generate_url(rse, configuration)
|
|
237
|
-
if date == 'latest':
|
|
238
|
-
logger.debug('Looking for site dumps in: "%s"', base_url)
|
|
239
|
-
links = get_links(base_url)
|
|
240
|
-
url, date = get_newest(base_url, url_pattern, links)
|
|
241
|
-
else:
|
|
242
|
-
url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
|
|
243
271
|
|
|
244
272
|
if not os.path.isdir(destdir):
|
|
245
273
|
os.mkdir(destdir)
|
|
246
274
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
275
|
+
# check for objectstores, which need to be handled differently
|
|
276
|
+
rse_id = get_rse_id(rse)
|
|
277
|
+
rse_attr = list_rse_attributes(rse_id)
|
|
278
|
+
if RseAttr.IS_OBJECT_STORE in rse_attr and rse_attr[RseAttr.IS_OBJECT_STORE] is not False:
|
|
279
|
+
tries = 1
|
|
280
|
+
if date is None:
|
|
281
|
+
# on objectstores, can't list dump files, so try the last N dates
|
|
282
|
+
date = datetime.datetime.now()
|
|
283
|
+
tries = OBJECTSTORE_NUM_TRIES
|
|
284
|
+
path = ''
|
|
285
|
+
while tries > 0:
|
|
286
|
+
url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
|
|
287
|
+
|
|
288
|
+
filename = '{0}_{1}_{2}_{3}'.format(
|
|
289
|
+
'ddmendpoint',
|
|
290
|
+
rse,
|
|
291
|
+
date.strftime('%d-%m-%Y'),
|
|
292
|
+
hashlib.sha1(url.encode()).hexdigest()
|
|
293
|
+
)
|
|
294
|
+
filename = re.sub(r'\W', '-', filename)
|
|
295
|
+
path = os.path.join(destdir, filename)
|
|
296
|
+
if not os.path.exists(path):
|
|
297
|
+
logger.debug('Trying to download: "%s"', url)
|
|
298
|
+
if RseAttr.SIGN_URL in rse_attr:
|
|
299
|
+
url = get_signed_url(rse_id, rse_attr[RseAttr.SIGN_URL], 'read', url)
|
|
300
|
+
try:
|
|
301
|
+
with temp_file(destdir, final_name=filename) as (f, _):
|
|
302
|
+
download(url, f)
|
|
303
|
+
tries = 0
|
|
304
|
+
except (HTTPDownloadFailed, gfal2.GError):
|
|
305
|
+
tries -= 1
|
|
306
|
+
date = date - datetime.timedelta(1)
|
|
307
|
+
else:
|
|
308
|
+
if date is None:
|
|
309
|
+
logger.debug('Looking for site dumps in: "%s"', base_url)
|
|
310
|
+
links = get_links(base_url)
|
|
311
|
+
url, date = get_newest(base_url, url_pattern, links)
|
|
312
|
+
else:
|
|
313
|
+
url = '{0}/{1}'.format(base_url, date.strftime(url_pattern))
|
|
314
|
+
|
|
315
|
+
filename = '{0}_{1}_{2}_{3}'.format(
|
|
316
|
+
'ddmendpoint',
|
|
317
|
+
rse,
|
|
318
|
+
date.strftime('%d-%m-%Y'),
|
|
319
|
+
hashlib.sha1(url.encode()).hexdigest()
|
|
320
|
+
)
|
|
321
|
+
filename = re.sub(r'\W', '-', filename)
|
|
322
|
+
path = os.path.join(destdir, filename)
|
|
323
|
+
|
|
324
|
+
if not os.path.exists(path):
|
|
325
|
+
logger.debug('Trying to download: "%s"', url)
|
|
326
|
+
with temp_file(destdir, final_name=filename) as (f, _):
|
|
327
|
+
download(url, f)
|
|
260
328
|
|
|
261
329
|
return (path, date)
|
|
262
330
|
|
|
263
331
|
|
|
264
|
-
def generate_url(
|
|
332
|
+
def generate_url(
|
|
333
|
+
rse: str,
|
|
334
|
+
config: ConfigParser.RawConfigParser
|
|
335
|
+
) -> tuple[str, str]:
|
|
265
336
|
'''
|
|
266
337
|
:param rse: Name of the endpoint.
|
|
267
338
|
:param config: RawConfigParser instance which may have configuration
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
1
|
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -16,40 +15,42 @@
|
|
|
16
15
|
import functools
|
|
17
16
|
import logging
|
|
18
17
|
import random
|
|
18
|
+
import secrets
|
|
19
19
|
import tempfile
|
|
20
20
|
import threading
|
|
21
21
|
from configparser import NoOptionError, NoSectionError
|
|
22
22
|
from datetime import datetime
|
|
23
23
|
from json import load
|
|
24
24
|
from os import remove, rmdir
|
|
25
|
-
from typing import TYPE_CHECKING
|
|
25
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
26
26
|
|
|
27
27
|
import rucio.db.sqla.util
|
|
28
28
|
from rucio.client import Client
|
|
29
29
|
from rucio.client.uploadclient import UploadClient
|
|
30
30
|
from rucio.common import exception
|
|
31
|
-
from rucio.common.config import config_get,
|
|
31
|
+
from rucio.common.config import config_get, config_get_bool, config_get_int
|
|
32
32
|
from rucio.common.logging import setup_logging
|
|
33
33
|
from rucio.common.stopwatch import Stopwatch
|
|
34
|
-
from rucio.common.types import InternalScope
|
|
34
|
+
from rucio.common.types import InternalScope, LoggerFunction
|
|
35
35
|
from rucio.common.utils import execute, generate_uuid
|
|
36
36
|
from rucio.core.monitor import MetricManager
|
|
37
37
|
from rucio.core.scope import list_scopes
|
|
38
38
|
from rucio.core.vo import map_vo
|
|
39
|
-
from rucio.daemons.common import run_daemon
|
|
39
|
+
from rucio.daemons.common import HeartbeatHandler, run_daemon
|
|
40
40
|
|
|
41
41
|
if TYPE_CHECKING:
|
|
42
|
+
from collections.abc import Mapping
|
|
42
43
|
from types import FrameType
|
|
43
|
-
from typing import Optional
|
|
44
44
|
|
|
45
|
-
from rucio.daemons.common import HeartbeatHandler
|
|
46
45
|
|
|
47
46
|
METRICS = MetricManager(module=__name__)
|
|
48
47
|
graceful_stop = threading.Event()
|
|
49
48
|
DAEMON_NAME = "automatix"
|
|
50
49
|
|
|
51
50
|
|
|
52
|
-
def get_data_distribution(
|
|
51
|
+
def get_data_distribution(
|
|
52
|
+
inputfile: str
|
|
53
|
+
) -> tuple[dict[str, int], dict[str, Any]]:
|
|
53
54
|
with open(inputfile) as data_file:
|
|
54
55
|
data = load(data_file)
|
|
55
56
|
probabilities = {}
|
|
@@ -62,8 +63,11 @@ def get_data_distribution(inputfile: str):
|
|
|
62
63
|
return probabilities, data
|
|
63
64
|
|
|
64
65
|
|
|
65
|
-
def choose_element(
|
|
66
|
-
|
|
66
|
+
def choose_element(
|
|
67
|
+
probabilities: "Mapping[str, int]",
|
|
68
|
+
data: "Mapping[str, Any]"
|
|
69
|
+
) -> dict[str, Any]:
|
|
70
|
+
rnd = random.uniform(0, 1) # noqa: S311
|
|
67
71
|
prob = 0
|
|
68
72
|
for key in probabilities:
|
|
69
73
|
prob = probabilities[key]
|
|
@@ -72,7 +76,11 @@ def choose_element(probabilities: dict, data: str) -> float:
|
|
|
72
76
|
return data[key]
|
|
73
77
|
|
|
74
78
|
|
|
75
|
-
def generate_file(
|
|
79
|
+
def generate_file(
|
|
80
|
+
fname: str,
|
|
81
|
+
size: int,
|
|
82
|
+
logger: LoggerFunction = logging.log
|
|
83
|
+
) -> int:
|
|
76
84
|
cmd = "/bin/dd if=/dev/urandom of=%s bs=%s count=1" % (fname, size)
|
|
77
85
|
exitcode, out, err = execute(cmd)
|
|
78
86
|
logger(logging.DEBUG, out)
|
|
@@ -80,7 +88,11 @@ def generate_file(fname: str, size: int, logger=logging.log) -> int:
|
|
|
80
88
|
return exitcode
|
|
81
89
|
|
|
82
90
|
|
|
83
|
-
def generate_didname(
|
|
91
|
+
def generate_didname(
|
|
92
|
+
metadata: "Mapping[str, str]",
|
|
93
|
+
dsn: Optional[str],
|
|
94
|
+
did_type: str
|
|
95
|
+
) -> str:
|
|
84
96
|
try:
|
|
85
97
|
did_prefix = config_get("automatix", "did_prefix")
|
|
86
98
|
except (NoOptionError, NoSectionError, RuntimeError):
|
|
@@ -102,11 +114,11 @@ def generate_didname(metadata: dict, dsn: str, did_type: str) -> str:
|
|
|
102
114
|
elif field == "uuid":
|
|
103
115
|
field_str = generate_uuid()
|
|
104
116
|
elif field == "randint":
|
|
105
|
-
field_str = str(
|
|
117
|
+
field_str = str(secrets.choice(range(0, 100001)))
|
|
106
118
|
else:
|
|
107
119
|
field_str = metadata.get(field, None)
|
|
108
120
|
if not field_str:
|
|
109
|
-
field_str = str(
|
|
121
|
+
field_str = str(secrets.choice(range(0, 100001)))
|
|
110
122
|
file_name = "%s%s%s" % (file_name, separator, field_str)
|
|
111
123
|
len_separator = len(separator)
|
|
112
124
|
return file_name[len_separator:]
|
|
@@ -133,7 +145,7 @@ def automatix(inputfile: str, sleep_time: int, once: bool = False) -> None:
|
|
|
133
145
|
)
|
|
134
146
|
|
|
135
147
|
|
|
136
|
-
def run_once(heartbeat_handler:
|
|
148
|
+
def run_once(heartbeat_handler: HeartbeatHandler, inputfile: str, **_kwargs) -> bool:
|
|
137
149
|
|
|
138
150
|
_, _, logger = heartbeat_handler.live()
|
|
139
151
|
try:
|
|
@@ -166,7 +178,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", inputfile: str, **_kwargs) -
|
|
|
166
178
|
account = config_get("automatix", "account", raise_exception=False, default="root")
|
|
167
179
|
scope = config_get("automatix", "scope", raise_exception=False, default="test")
|
|
168
180
|
client = Client(account=account)
|
|
169
|
-
vo = map_vo(client.vo)
|
|
181
|
+
vo = map_vo(client.vo) # type: ignore
|
|
170
182
|
filters = {"scope": InternalScope("*", vo=vo)}
|
|
171
183
|
scopes = list_scopes(filter_=filters)
|
|
172
184
|
if InternalScope(scope, vo=vo) not in scopes:
|
|
@@ -230,7 +242,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", inputfile: str, **_kwargs) -
|
|
|
230
242
|
upload_client = UploadClient(client)
|
|
231
243
|
ret = upload_client.upload(files)
|
|
232
244
|
if ret == 0:
|
|
233
|
-
logger(logging.INFO, "%s
|
|
245
|
+
logger(logging.INFO, "%s successfully registered" % dsn)
|
|
234
246
|
METRICS.counter(name="addnewdataset.done").inc()
|
|
235
247
|
METRICS.counter(name="addnewfile.done").inc(nbfiles)
|
|
236
248
|
METRICS.timer(name='datasetinjection').observe(stopwatch.elapsed)
|
|
@@ -274,7 +286,7 @@ def run(
|
|
|
274
286
|
[thread.join(timeout=3.14) for thread in threads]
|
|
275
287
|
|
|
276
288
|
|
|
277
|
-
def stop(signum:
|
|
289
|
+
def stop(signum: Optional[int] = None, frame: Optional["FrameType"] = None) -> None:
|
|
278
290
|
"""
|
|
279
291
|
Graceful exit.
|
|
280
292
|
"""
|