rucio 37.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rucio might be problematic. Click here for more details.
- rucio/__init__.py +17 -0
- rucio/alembicrevision.py +15 -0
- rucio/cli/__init__.py +14 -0
- rucio/cli/account.py +216 -0
- rucio/cli/bin_legacy/__init__.py +13 -0
- rucio/cli/bin_legacy/rucio.py +2825 -0
- rucio/cli/bin_legacy/rucio_admin.py +2500 -0
- rucio/cli/command.py +272 -0
- rucio/cli/config.py +72 -0
- rucio/cli/did.py +191 -0
- rucio/cli/download.py +128 -0
- rucio/cli/lifetime_exception.py +33 -0
- rucio/cli/replica.py +162 -0
- rucio/cli/rse.py +293 -0
- rucio/cli/rule.py +158 -0
- rucio/cli/scope.py +40 -0
- rucio/cli/subscription.py +73 -0
- rucio/cli/upload.py +60 -0
- rucio/cli/utils.py +226 -0
- rucio/client/__init__.py +15 -0
- rucio/client/accountclient.py +432 -0
- rucio/client/accountlimitclient.py +183 -0
- rucio/client/baseclient.py +983 -0
- rucio/client/client.py +120 -0
- rucio/client/configclient.py +126 -0
- rucio/client/credentialclient.py +59 -0
- rucio/client/didclient.py +868 -0
- rucio/client/diracclient.py +56 -0
- rucio/client/downloadclient.py +1783 -0
- rucio/client/exportclient.py +44 -0
- rucio/client/fileclient.py +50 -0
- rucio/client/importclient.py +42 -0
- rucio/client/lifetimeclient.py +90 -0
- rucio/client/lockclient.py +109 -0
- rucio/client/metaconventionsclient.py +140 -0
- rucio/client/pingclient.py +44 -0
- rucio/client/replicaclient.py +452 -0
- rucio/client/requestclient.py +125 -0
- rucio/client/richclient.py +317 -0
- rucio/client/rseclient.py +746 -0
- rucio/client/ruleclient.py +294 -0
- rucio/client/scopeclient.py +90 -0
- rucio/client/subscriptionclient.py +173 -0
- rucio/client/touchclient.py +82 -0
- rucio/client/uploadclient.py +969 -0
- rucio/common/__init__.py +13 -0
- rucio/common/bittorrent.py +234 -0
- rucio/common/cache.py +111 -0
- rucio/common/checksum.py +168 -0
- rucio/common/client.py +122 -0
- rucio/common/config.py +788 -0
- rucio/common/constants.py +217 -0
- rucio/common/constraints.py +17 -0
- rucio/common/didtype.py +237 -0
- rucio/common/dumper/__init__.py +342 -0
- rucio/common/dumper/consistency.py +497 -0
- rucio/common/dumper/data_models.py +362 -0
- rucio/common/dumper/path_parsing.py +75 -0
- rucio/common/exception.py +1208 -0
- rucio/common/extra.py +31 -0
- rucio/common/logging.py +420 -0
- rucio/common/pcache.py +1409 -0
- rucio/common/plugins.py +185 -0
- rucio/common/policy.py +93 -0
- rucio/common/schema/__init__.py +200 -0
- rucio/common/schema/generic.py +416 -0
- rucio/common/schema/generic_multi_vo.py +395 -0
- rucio/common/stomp_utils.py +423 -0
- rucio/common/stopwatch.py +55 -0
- rucio/common/test_rucio_server.py +154 -0
- rucio/common/types.py +483 -0
- rucio/common/utils.py +1688 -0
- rucio/core/__init__.py +13 -0
- rucio/core/account.py +496 -0
- rucio/core/account_counter.py +236 -0
- rucio/core/account_limit.py +425 -0
- rucio/core/authentication.py +620 -0
- rucio/core/config.py +437 -0
- rucio/core/credential.py +224 -0
- rucio/core/did.py +3004 -0
- rucio/core/did_meta_plugins/__init__.py +252 -0
- rucio/core/did_meta_plugins/did_column_meta.py +331 -0
- rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
- rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
- rucio/core/did_meta_plugins/filter_engine.py +672 -0
- rucio/core/did_meta_plugins/json_meta.py +240 -0
- rucio/core/did_meta_plugins/mongo_meta.py +229 -0
- rucio/core/did_meta_plugins/postgres_meta.py +352 -0
- rucio/core/dirac.py +237 -0
- rucio/core/distance.py +187 -0
- rucio/core/exporter.py +59 -0
- rucio/core/heartbeat.py +363 -0
- rucio/core/identity.py +301 -0
- rucio/core/importer.py +260 -0
- rucio/core/lifetime_exception.py +377 -0
- rucio/core/lock.py +577 -0
- rucio/core/message.py +288 -0
- rucio/core/meta_conventions.py +203 -0
- rucio/core/monitor.py +448 -0
- rucio/core/naming_convention.py +195 -0
- rucio/core/nongrid_trace.py +136 -0
- rucio/core/oidc.py +1463 -0
- rucio/core/permission/__init__.py +161 -0
- rucio/core/permission/generic.py +1124 -0
- rucio/core/permission/generic_multi_vo.py +1144 -0
- rucio/core/quarantined_replica.py +224 -0
- rucio/core/replica.py +4483 -0
- rucio/core/replica_sorter.py +362 -0
- rucio/core/request.py +3091 -0
- rucio/core/rse.py +2079 -0
- rucio/core/rse_counter.py +185 -0
- rucio/core/rse_expression_parser.py +459 -0
- rucio/core/rse_selector.py +304 -0
- rucio/core/rule.py +4484 -0
- rucio/core/rule_grouping.py +1620 -0
- rucio/core/scope.py +181 -0
- rucio/core/subscription.py +362 -0
- rucio/core/topology.py +490 -0
- rucio/core/trace.py +375 -0
- rucio/core/transfer.py +1531 -0
- rucio/core/vo.py +169 -0
- rucio/core/volatile_replica.py +151 -0
- rucio/daemons/__init__.py +13 -0
- rucio/daemons/abacus/__init__.py +13 -0
- rucio/daemons/abacus/account.py +116 -0
- rucio/daemons/abacus/collection_replica.py +124 -0
- rucio/daemons/abacus/rse.py +117 -0
- rucio/daemons/atropos/__init__.py +13 -0
- rucio/daemons/atropos/atropos.py +242 -0
- rucio/daemons/auditor/__init__.py +289 -0
- rucio/daemons/auditor/hdfs.py +97 -0
- rucio/daemons/auditor/srmdumps.py +355 -0
- rucio/daemons/automatix/__init__.py +13 -0
- rucio/daemons/automatix/automatix.py +304 -0
- rucio/daemons/badreplicas/__init__.py +13 -0
- rucio/daemons/badreplicas/minos.py +322 -0
- rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
- rucio/daemons/badreplicas/necromancer.py +196 -0
- rucio/daemons/bb8/__init__.py +13 -0
- rucio/daemons/bb8/bb8.py +353 -0
- rucio/daemons/bb8/common.py +759 -0
- rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
- rucio/daemons/bb8/t2_background_rebalance.py +153 -0
- rucio/daemons/cache/__init__.py +13 -0
- rucio/daemons/cache/consumer.py +133 -0
- rucio/daemons/common.py +405 -0
- rucio/daemons/conveyor/__init__.py +13 -0
- rucio/daemons/conveyor/common.py +562 -0
- rucio/daemons/conveyor/finisher.py +529 -0
- rucio/daemons/conveyor/poller.py +394 -0
- rucio/daemons/conveyor/preparer.py +205 -0
- rucio/daemons/conveyor/receiver.py +179 -0
- rucio/daemons/conveyor/stager.py +133 -0
- rucio/daemons/conveyor/submitter.py +403 -0
- rucio/daemons/conveyor/throttler.py +532 -0
- rucio/daemons/follower/__init__.py +13 -0
- rucio/daemons/follower/follower.py +101 -0
- rucio/daemons/hermes/__init__.py +13 -0
- rucio/daemons/hermes/hermes.py +534 -0
- rucio/daemons/judge/__init__.py +13 -0
- rucio/daemons/judge/cleaner.py +159 -0
- rucio/daemons/judge/evaluator.py +185 -0
- rucio/daemons/judge/injector.py +162 -0
- rucio/daemons/judge/repairer.py +154 -0
- rucio/daemons/oauthmanager/__init__.py +13 -0
- rucio/daemons/oauthmanager/oauthmanager.py +198 -0
- rucio/daemons/reaper/__init__.py +13 -0
- rucio/daemons/reaper/dark_reaper.py +282 -0
- rucio/daemons/reaper/reaper.py +739 -0
- rucio/daemons/replicarecoverer/__init__.py +13 -0
- rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
- rucio/daemons/rsedecommissioner/__init__.py +13 -0
- rucio/daemons/rsedecommissioner/config.py +81 -0
- rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
- rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
- rucio/daemons/rsedecommissioner/profiles/generic.py +452 -0
- rucio/daemons/rsedecommissioner/profiles/types.py +93 -0
- rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
- rucio/daemons/storage/__init__.py +13 -0
- rucio/daemons/storage/consistency/__init__.py +13 -0
- rucio/daemons/storage/consistency/actions.py +848 -0
- rucio/daemons/tracer/__init__.py +13 -0
- rucio/daemons/tracer/kronos.py +511 -0
- rucio/daemons/transmogrifier/__init__.py +13 -0
- rucio/daemons/transmogrifier/transmogrifier.py +762 -0
- rucio/daemons/undertaker/__init__.py +13 -0
- rucio/daemons/undertaker/undertaker.py +137 -0
- rucio/db/__init__.py +13 -0
- rucio/db/sqla/__init__.py +52 -0
- rucio/db/sqla/constants.py +206 -0
- rucio/db/sqla/migrate_repo/__init__.py +13 -0
- rucio/db/sqla/migrate_repo/env.py +110 -0
- rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
- rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
- rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
- rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
- rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
- rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
- rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
- rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
- rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
- rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
- rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
- rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
- rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
- rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
- rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
- rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
- rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
- rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
- rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
- rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
- rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
- rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
- rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
- rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
- rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
- rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
- rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
- rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
- rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
- rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
- rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
- rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
- rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
- rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
- rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
- rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
- rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
- rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
- rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
- rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
- rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
- rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
- rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
- rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
- rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
- rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
- rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
- rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
- rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
- rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
- rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
- rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
- rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
- rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
- rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
- rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
- rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
- rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
- rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
- rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
- rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
- rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
- rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
- rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
- rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
- rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
- rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
- rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
- rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
- rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
- rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
- rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
- rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
- rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
- rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
- rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
- rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
- rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
- rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
- rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
- rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
- rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
- rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
- rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
- rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
- rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
- rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
- rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
- rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
- rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
- rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
- rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
- rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
- rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
- rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
- rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
- rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
- rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
- rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
- rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
- rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
- rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
- rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
- rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
- rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
- rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
- rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
- rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
- rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
- rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
- rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
- rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
- rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
- rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
- rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
- rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
- rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
- rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
- rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
- rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
- rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
- rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
- rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
- rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
- rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
- rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
- rucio/db/sqla/models.py +1743 -0
- rucio/db/sqla/sautils.py +55 -0
- rucio/db/sqla/session.py +529 -0
- rucio/db/sqla/types.py +206 -0
- rucio/db/sqla/util.py +543 -0
- rucio/gateway/__init__.py +13 -0
- rucio/gateway/account.py +345 -0
- rucio/gateway/account_limit.py +363 -0
- rucio/gateway/authentication.py +381 -0
- rucio/gateway/config.py +227 -0
- rucio/gateway/credential.py +70 -0
- rucio/gateway/did.py +987 -0
- rucio/gateway/dirac.py +83 -0
- rucio/gateway/exporter.py +60 -0
- rucio/gateway/heartbeat.py +76 -0
- rucio/gateway/identity.py +189 -0
- rucio/gateway/importer.py +46 -0
- rucio/gateway/lifetime_exception.py +121 -0
- rucio/gateway/lock.py +153 -0
- rucio/gateway/meta_conventions.py +98 -0
- rucio/gateway/permission.py +74 -0
- rucio/gateway/quarantined_replica.py +79 -0
- rucio/gateway/replica.py +538 -0
- rucio/gateway/request.py +330 -0
- rucio/gateway/rse.py +632 -0
- rucio/gateway/rule.py +437 -0
- rucio/gateway/scope.py +100 -0
- rucio/gateway/subscription.py +280 -0
- rucio/gateway/vo.py +126 -0
- rucio/rse/__init__.py +96 -0
- rucio/rse/protocols/__init__.py +13 -0
- rucio/rse/protocols/bittorrent.py +194 -0
- rucio/rse/protocols/cache.py +111 -0
- rucio/rse/protocols/dummy.py +100 -0
- rucio/rse/protocols/gfal.py +708 -0
- rucio/rse/protocols/globus.py +243 -0
- rucio/rse/protocols/http_cache.py +82 -0
- rucio/rse/protocols/mock.py +123 -0
- rucio/rse/protocols/ngarc.py +209 -0
- rucio/rse/protocols/posix.py +250 -0
- rucio/rse/protocols/protocol.py +361 -0
- rucio/rse/protocols/rclone.py +365 -0
- rucio/rse/protocols/rfio.py +145 -0
- rucio/rse/protocols/srm.py +338 -0
- rucio/rse/protocols/ssh.py +414 -0
- rucio/rse/protocols/storm.py +195 -0
- rucio/rse/protocols/webdav.py +594 -0
- rucio/rse/protocols/xrootd.py +302 -0
- rucio/rse/rsemanager.py +881 -0
- rucio/rse/translation.py +260 -0
- rucio/tests/__init__.py +13 -0
- rucio/tests/common.py +280 -0
- rucio/tests/common_server.py +149 -0
- rucio/transfertool/__init__.py +13 -0
- rucio/transfertool/bittorrent.py +200 -0
- rucio/transfertool/bittorrent_driver.py +50 -0
- rucio/transfertool/bittorrent_driver_qbittorrent.py +134 -0
- rucio/transfertool/fts3.py +1600 -0
- rucio/transfertool/fts3_plugins.py +152 -0
- rucio/transfertool/globus.py +201 -0
- rucio/transfertool/globus_library.py +181 -0
- rucio/transfertool/mock.py +89 -0
- rucio/transfertool/transfertool.py +221 -0
- rucio/vcsversion.py +11 -0
- rucio/version.py +45 -0
- rucio/web/__init__.py +13 -0
- rucio/web/rest/__init__.py +13 -0
- rucio/web/rest/flaskapi/__init__.py +13 -0
- rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
- rucio/web/rest/flaskapi/v1/__init__.py +13 -0
- rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
- rucio/web/rest/flaskapi/v1/accounts.py +1103 -0
- rucio/web/rest/flaskapi/v1/archives.py +102 -0
- rucio/web/rest/flaskapi/v1/auth.py +1644 -0
- rucio/web/rest/flaskapi/v1/common.py +426 -0
- rucio/web/rest/flaskapi/v1/config.py +304 -0
- rucio/web/rest/flaskapi/v1/credentials.py +213 -0
- rucio/web/rest/flaskapi/v1/dids.py +2340 -0
- rucio/web/rest/flaskapi/v1/dirac.py +116 -0
- rucio/web/rest/flaskapi/v1/export.py +75 -0
- rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
- rucio/web/rest/flaskapi/v1/identities.py +285 -0
- rucio/web/rest/flaskapi/v1/import.py +132 -0
- rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
- rucio/web/rest/flaskapi/v1/locks.py +358 -0
- rucio/web/rest/flaskapi/v1/main.py +91 -0
- rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
- rucio/web/rest/flaskapi/v1/metrics.py +36 -0
- rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
- rucio/web/rest/flaskapi/v1/ping.py +88 -0
- rucio/web/rest/flaskapi/v1/redirect.py +366 -0
- rucio/web/rest/flaskapi/v1/replicas.py +1894 -0
- rucio/web/rest/flaskapi/v1/requests.py +998 -0
- rucio/web/rest/flaskapi/v1/rses.py +2250 -0
- rucio/web/rest/flaskapi/v1/rules.py +854 -0
- rucio/web/rest/flaskapi/v1/scopes.py +159 -0
- rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
- rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
- rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
- rucio/web/rest/flaskapi/v1/traces.py +137 -0
- rucio/web/rest/flaskapi/v1/types.py +20 -0
- rucio/web/rest/flaskapi/v1/vos.py +278 -0
- rucio/web/rest/main.py +18 -0
- rucio/web/rest/metrics.py +27 -0
- rucio/web/rest/ping.py +27 -0
- rucio-37.0.0rc1.data/data/rucio/etc/alembic.ini.template +71 -0
- rucio-37.0.0rc1.data/data/rucio/etc/alembic_offline.ini.template +74 -0
- rucio-37.0.0rc1.data/data/rucio/etc/globus-config.yml.template +5 -0
- rucio-37.0.0rc1.data/data/rucio/etc/ldap.cfg.template +30 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
- rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
- rucio-37.0.0rc1.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
- rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.atlas.client.template +43 -0
- rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.template +241 -0
- rucio-37.0.0rc1.data/data/rucio/etc/rucio_multi_vo.cfg.template +217 -0
- rucio-37.0.0rc1.data/data/rucio/requirements.server.txt +297 -0
- rucio-37.0.0rc1.data/data/rucio/tools/bootstrap.py +34 -0
- rucio-37.0.0rc1.data/data/rucio/tools/merge_rucio_configs.py +144 -0
- rucio-37.0.0rc1.data/data/rucio/tools/reset_database.py +40 -0
- rucio-37.0.0rc1.data/scripts/rucio +133 -0
- rucio-37.0.0rc1.data/scripts/rucio-abacus-account +74 -0
- rucio-37.0.0rc1.data/scripts/rucio-abacus-collection-replica +46 -0
- rucio-37.0.0rc1.data/scripts/rucio-abacus-rse +78 -0
- rucio-37.0.0rc1.data/scripts/rucio-admin +97 -0
- rucio-37.0.0rc1.data/scripts/rucio-atropos +60 -0
- rucio-37.0.0rc1.data/scripts/rucio-auditor +206 -0
- rucio-37.0.0rc1.data/scripts/rucio-automatix +50 -0
- rucio-37.0.0rc1.data/scripts/rucio-bb8 +57 -0
- rucio-37.0.0rc1.data/scripts/rucio-cache-client +141 -0
- rucio-37.0.0rc1.data/scripts/rucio-cache-consumer +42 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-finisher +58 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-poller +66 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-preparer +37 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-receiver +44 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-stager +76 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-submitter +139 -0
- rucio-37.0.0rc1.data/scripts/rucio-conveyor-throttler +104 -0
- rucio-37.0.0rc1.data/scripts/rucio-dark-reaper +53 -0
- rucio-37.0.0rc1.data/scripts/rucio-dumper +160 -0
- rucio-37.0.0rc1.data/scripts/rucio-follower +44 -0
- rucio-37.0.0rc1.data/scripts/rucio-hermes +54 -0
- rucio-37.0.0rc1.data/scripts/rucio-judge-cleaner +89 -0
- rucio-37.0.0rc1.data/scripts/rucio-judge-evaluator +137 -0
- rucio-37.0.0rc1.data/scripts/rucio-judge-injector +44 -0
- rucio-37.0.0rc1.data/scripts/rucio-judge-repairer +44 -0
- rucio-37.0.0rc1.data/scripts/rucio-kronos +44 -0
- rucio-37.0.0rc1.data/scripts/rucio-minos +53 -0
- rucio-37.0.0rc1.data/scripts/rucio-minos-temporary-expiration +50 -0
- rucio-37.0.0rc1.data/scripts/rucio-necromancer +120 -0
- rucio-37.0.0rc1.data/scripts/rucio-oauth-manager +63 -0
- rucio-37.0.0rc1.data/scripts/rucio-reaper +83 -0
- rucio-37.0.0rc1.data/scripts/rucio-replica-recoverer +248 -0
- rucio-37.0.0rc1.data/scripts/rucio-rse-decommissioner +66 -0
- rucio-37.0.0rc1.data/scripts/rucio-storage-consistency-actions +74 -0
- rucio-37.0.0rc1.data/scripts/rucio-transmogrifier +77 -0
- rucio-37.0.0rc1.data/scripts/rucio-undertaker +76 -0
- rucio-37.0.0rc1.dist-info/METADATA +92 -0
- rucio-37.0.0rc1.dist-info/RECORD +487 -0
- rucio-37.0.0rc1.dist-info/WHEEL +5 -0
- rucio-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
- rucio-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
- rucio-37.0.0rc1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1894 @@
|
|
|
1
|
+
# Copyright European Organization for Nuclear Research (CERN) since 2012
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
from itertools import chain
|
|
17
|
+
from json import dumps, loads
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
from urllib.parse import parse_qs, unquote
|
|
20
|
+
from xml.sax.saxutils import escape
|
|
21
|
+
|
|
22
|
+
from flask import Flask, Response, request
|
|
23
|
+
|
|
24
|
+
from rucio.common.config import config_get, config_get_int
|
|
25
|
+
from rucio.common.constants import SUPPORTED_PROTOCOLS
|
|
26
|
+
from rucio.common.exception import (
|
|
27
|
+
AccessDenied,
|
|
28
|
+
DataIdentifierAlreadyExists,
|
|
29
|
+
DataIdentifierNotFound,
|
|
30
|
+
Duplicate,
|
|
31
|
+
InvalidObject,
|
|
32
|
+
InvalidPath,
|
|
33
|
+
InvalidType,
|
|
34
|
+
ReplicaIsLocked,
|
|
35
|
+
ReplicaNotFound,
|
|
36
|
+
ResourceTemporaryUnavailable,
|
|
37
|
+
RSENotFound,
|
|
38
|
+
ScopeNotFound,
|
|
39
|
+
SortingAlgorithmNotSupported,
|
|
40
|
+
)
|
|
41
|
+
from rucio.common.utils import APIEncoder, parse_response, render_json
|
|
42
|
+
from rucio.core.replica_sorter import sort_replicas
|
|
43
|
+
from rucio.db.sqla.constants import BadFilesStatus
|
|
44
|
+
from rucio.gateway.quarantined_replica import quarantine_file_replicas
|
|
45
|
+
from rucio.gateway.replica import (
|
|
46
|
+
add_bad_dids,
|
|
47
|
+
add_bad_pfns,
|
|
48
|
+
add_replicas,
|
|
49
|
+
declare_bad_file_replicas,
|
|
50
|
+
declare_suspicious_file_replicas,
|
|
51
|
+
delete_replicas,
|
|
52
|
+
get_bad_replicas_summary,
|
|
53
|
+
get_did_from_pfns,
|
|
54
|
+
get_suspicious_files,
|
|
55
|
+
list_bad_replicas_status,
|
|
56
|
+
list_dataset_replicas,
|
|
57
|
+
list_dataset_replicas_bulk,
|
|
58
|
+
list_dataset_replicas_vp,
|
|
59
|
+
list_datasets_per_rse,
|
|
60
|
+
list_replicas,
|
|
61
|
+
set_tombstone,
|
|
62
|
+
update_replicas_states,
|
|
63
|
+
)
|
|
64
|
+
from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
|
|
65
|
+
from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
|
|
66
|
+
|
|
67
|
+
if TYPE_CHECKING:
|
|
68
|
+
from rucio.common.types import IPDict
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _sorted_with_priorities(replicas, sorted_pfns, limit=None):
|
|
72
|
+
"""
|
|
73
|
+
Pick up to "limit" replicas from "replicas" in the order given by sorted_pfns.
|
|
74
|
+
Sets the corresponding priority in returned replicas.
|
|
75
|
+
|
|
76
|
+
:param replicas: Dictionary {pfn: replica_definition}.
|
|
77
|
+
:param sorted_pfns: Sorted list of pfns.
|
|
78
|
+
:param limit: only return this many replicas
|
|
79
|
+
:yields: index and corresponding pfn
|
|
80
|
+
"""
|
|
81
|
+
for idx, pfn in enumerate(sorted_pfns, start=1):
|
|
82
|
+
if limit is None or idx <= limit:
|
|
83
|
+
replica = replicas[pfn]
|
|
84
|
+
replica['priority'] = idx
|
|
85
|
+
yield pfn, replica
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _generate_one_metalink_file(rfile, policy_schema, detailed_url=True):
|
|
89
|
+
yield ' <file name="' + rfile['name'] + '">\n'
|
|
90
|
+
|
|
91
|
+
if 'parents' in rfile and rfile['parents']:
|
|
92
|
+
yield ' <parents>\n'
|
|
93
|
+
for parent in rfile['parents']:
|
|
94
|
+
yield ' <did>' + parent + '</did>\n'
|
|
95
|
+
yield ' </parents>\n'
|
|
96
|
+
|
|
97
|
+
yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
|
|
98
|
+
if rfile['adler32'] is not None:
|
|
99
|
+
yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
|
|
100
|
+
if rfile['md5'] is not None:
|
|
101
|
+
yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
|
|
102
|
+
|
|
103
|
+
yield ' <size>' + str(rfile['bytes']) + '</size>\n'
|
|
104
|
+
|
|
105
|
+
yield f' <glfn name="/{policy_schema}/rucio/{rfile["scope"]}:{rfile["name"]}"></glfn>\n'
|
|
106
|
+
|
|
107
|
+
for pfn, replica in rfile['pfns'].items():
|
|
108
|
+
if detailed_url:
|
|
109
|
+
yield (
|
|
110
|
+
' '
|
|
111
|
+
f'<url location="{replica["rse"]}"'
|
|
112
|
+
f' domain="{replica["domain"]}"'
|
|
113
|
+
f' priority="{replica["priority"]}"'
|
|
114
|
+
f' client_extract="{str(replica["client_extract"]).lower()}"'
|
|
115
|
+
f'>{escape(pfn)}</url>\n'
|
|
116
|
+
)
|
|
117
|
+
else:
|
|
118
|
+
yield (
|
|
119
|
+
' '
|
|
120
|
+
f'<url location="{replica["rse"]}"'
|
|
121
|
+
f' priority="{replica["priority"]}"'
|
|
122
|
+
f'>{escape(pfn)}</url>\n'
|
|
123
|
+
)
|
|
124
|
+
yield ' </file>\n'
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _generate_metalink_response(rfiles, policy_schema, detailed_url=True):
|
|
128
|
+
first = True
|
|
129
|
+
for rfile in rfiles:
|
|
130
|
+
if first:
|
|
131
|
+
# first, set the appropriate content type, and stream the header
|
|
132
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
|
|
133
|
+
first = False
|
|
134
|
+
|
|
135
|
+
yield from _generate_one_metalink_file(rfile, policy_schema=policy_schema, detailed_url=detailed_url)
|
|
136
|
+
|
|
137
|
+
if first:
|
|
138
|
+
# if still first output, i.e. there were no replicas
|
|
139
|
+
yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
|
|
140
|
+
else:
|
|
141
|
+
# don't forget to send the metalink footer
|
|
142
|
+
yield '</metalink>\n'
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _generate_json_response(rfiles):
|
|
146
|
+
for rfile in rfiles:
|
|
147
|
+
yield dumps(rfile) + '\n'
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class Replicas(ErrorHandlingMethodView):
|
|
151
|
+
|
|
152
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
|
|
153
|
+
def get(self, scope_name):
|
|
154
|
+
"""
|
|
155
|
+
---
|
|
156
|
+
summary: Get Replicas
|
|
157
|
+
description: List all replicas for data identifiers.
|
|
158
|
+
tags:
|
|
159
|
+
- Replicas
|
|
160
|
+
parameters:
|
|
161
|
+
- name: scope_name
|
|
162
|
+
in: path
|
|
163
|
+
description: The DID associated with the replicas.
|
|
164
|
+
schema:
|
|
165
|
+
type: string
|
|
166
|
+
style: simple
|
|
167
|
+
- name: X-Forwarded-For
|
|
168
|
+
in: header
|
|
169
|
+
description: The client ip
|
|
170
|
+
schema:
|
|
171
|
+
type: string
|
|
172
|
+
- name: schemes
|
|
173
|
+
in: query
|
|
174
|
+
description: The schemes of the replicas.
|
|
175
|
+
schema:
|
|
176
|
+
type: string
|
|
177
|
+
- name: select
|
|
178
|
+
in: query
|
|
179
|
+
description: The sorting algorithm.
|
|
180
|
+
schema:
|
|
181
|
+
type: string
|
|
182
|
+
enum: ["geoip", "random"]
|
|
183
|
+
- name: limit
|
|
184
|
+
in: query
|
|
185
|
+
description: The maximum number of replicas returned.
|
|
186
|
+
schema:
|
|
187
|
+
type: integer
|
|
188
|
+
responses:
|
|
189
|
+
200:
|
|
190
|
+
description: OK
|
|
191
|
+
content:
|
|
192
|
+
application/x-json-stream:
|
|
193
|
+
schema:
|
|
194
|
+
description: A list with all replicas.
|
|
195
|
+
type: array
|
|
196
|
+
items:
|
|
197
|
+
description: A replica. Possibly contains more information.
|
|
198
|
+
type: object
|
|
199
|
+
properties:
|
|
200
|
+
scope:
|
|
201
|
+
description: The scope of the replica.
|
|
202
|
+
type: string
|
|
203
|
+
name:
|
|
204
|
+
description: The name of the replica.
|
|
205
|
+
type: string
|
|
206
|
+
bytes:
|
|
207
|
+
description: The size of the replica in bytes.
|
|
208
|
+
type: integer
|
|
209
|
+
md5:
|
|
210
|
+
description: The md5 checksum of the replica.
|
|
211
|
+
type: string
|
|
212
|
+
adler32:
|
|
213
|
+
description: The adler32 checksum of the replica.
|
|
214
|
+
type: string
|
|
215
|
+
pfns:
|
|
216
|
+
description: The pfns associated with the replica.
|
|
217
|
+
type: array
|
|
218
|
+
rses:
|
|
219
|
+
description: The rse associated with the replica.
|
|
220
|
+
type: string
|
|
221
|
+
401:
|
|
222
|
+
description: Invalid Auth Token
|
|
223
|
+
404:
|
|
224
|
+
description: Did not found
|
|
225
|
+
406:
|
|
226
|
+
description: Not acceptable
|
|
227
|
+
"""
|
|
228
|
+
try:
|
|
229
|
+
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
|
|
230
|
+
except ValueError as error:
|
|
231
|
+
return generate_http_error_flask(400, error)
|
|
232
|
+
|
|
233
|
+
content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
|
|
234
|
+
metalink = (content_type == 'application/metalink4+xml')
|
|
235
|
+
dids = [{'scope': scope, 'name': name}]
|
|
236
|
+
select, limit = None, None
|
|
237
|
+
|
|
238
|
+
client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
|
|
239
|
+
client_location: 'IPDict' = {'ip': client_ip, 'fqdn': None, 'site': None}
|
|
240
|
+
|
|
241
|
+
schemes = request.args.get('schemes', default=None)
|
|
242
|
+
select = request.args.get('select', default=None)
|
|
243
|
+
limit = request.args.get('limit', default=None)
|
|
244
|
+
if limit:
|
|
245
|
+
limit = int(limit)
|
|
246
|
+
|
|
247
|
+
# Resolve all reasonable protocols when doing metalink for maximum access possibilities
|
|
248
|
+
if metalink and schemes is None:
|
|
249
|
+
schemes = SUPPORTED_PROTOCOLS
|
|
250
|
+
|
|
251
|
+
try:
|
|
252
|
+
def _list_and_sort_replicas(vo):
|
|
253
|
+
# we need to call list_replicas before starting to reply
|
|
254
|
+
# otherwise the exceptions won't be propagated correctly
|
|
255
|
+
for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
|
|
256
|
+
replicas = []
|
|
257
|
+
dictreplica = {}
|
|
258
|
+
for rse in rfile['rses']:
|
|
259
|
+
for replica in rfile['rses'][rse]:
|
|
260
|
+
replicas.append(replica)
|
|
261
|
+
dictreplica[replica] = rse
|
|
262
|
+
|
|
263
|
+
replicas = sort_replicas(dictreplica, client_location, selection=select)
|
|
264
|
+
rfile['pfns'] = dict(_sorted_with_priorities(rfile['pfns'], replicas, limit=limit))
|
|
265
|
+
yield rfile
|
|
266
|
+
|
|
267
|
+
rfiles = _list_and_sort_replicas(vo=request.environ.get('vo'))
|
|
268
|
+
if metalink:
|
|
269
|
+
response_generator = _generate_metalink_response(rfiles, 'atlas', detailed_url=False)
|
|
270
|
+
else:
|
|
271
|
+
response_generator = _generate_json_response(rfiles)
|
|
272
|
+
return try_stream(response_generator, content_type=content_type)
|
|
273
|
+
except (DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
|
|
274
|
+
return generate_http_error_flask(404, error)
|
|
275
|
+
|
|
276
|
+
def post(self):
|
|
277
|
+
"""
|
|
278
|
+
---
|
|
279
|
+
summary: Create File Replicas
|
|
280
|
+
description: Create file replicas at a given RSE.
|
|
281
|
+
tags:
|
|
282
|
+
- Replicas
|
|
283
|
+
requestBody:
|
|
284
|
+
content:
|
|
285
|
+
application/json:
|
|
286
|
+
schema:
|
|
287
|
+
type: object
|
|
288
|
+
required:
|
|
289
|
+
- rse
|
|
290
|
+
- files
|
|
291
|
+
properties:
|
|
292
|
+
rse:
|
|
293
|
+
description: The rse for the replication
|
|
294
|
+
type: string
|
|
295
|
+
files:
|
|
296
|
+
description: The files to replicate
|
|
297
|
+
type: array
|
|
298
|
+
items:
|
|
299
|
+
type: object
|
|
300
|
+
required:
|
|
301
|
+
- pfn
|
|
302
|
+
- bytes
|
|
303
|
+
- name
|
|
304
|
+
properties:
|
|
305
|
+
pfn:
|
|
306
|
+
description: The pfn of the replica.
|
|
307
|
+
type: string
|
|
308
|
+
name:
|
|
309
|
+
description: The DID name.
|
|
310
|
+
type: string
|
|
311
|
+
bytes:
|
|
312
|
+
description: The size of the replica in bytes.
|
|
313
|
+
type: integer
|
|
314
|
+
state:
|
|
315
|
+
description: The state of the replica.
|
|
316
|
+
type: string
|
|
317
|
+
path:
|
|
318
|
+
description: The path of the new replica.
|
|
319
|
+
type: string
|
|
320
|
+
md5:
|
|
321
|
+
description: The md5 checksum.
|
|
322
|
+
type: string
|
|
323
|
+
adler32:
|
|
324
|
+
description: The adler32 checksum.
|
|
325
|
+
type: string
|
|
326
|
+
lcok_cnt:
|
|
327
|
+
description: The lock count.
|
|
328
|
+
type: integer
|
|
329
|
+
tombstone:
|
|
330
|
+
description: The tombstone.
|
|
331
|
+
type: string
|
|
332
|
+
ignore_availability:
|
|
333
|
+
description: The ignore availability.
|
|
334
|
+
type: boolean
|
|
335
|
+
responses:
|
|
336
|
+
201:
|
|
337
|
+
description: OK
|
|
338
|
+
content:
|
|
339
|
+
application/json:
|
|
340
|
+
schema:
|
|
341
|
+
type: string
|
|
342
|
+
enum: ["Created"]
|
|
343
|
+
400:
|
|
344
|
+
description: Invalid Path
|
|
345
|
+
401:
|
|
346
|
+
description: Invalid Auth Token
|
|
347
|
+
404:
|
|
348
|
+
description: Rse or scope not found
|
|
349
|
+
409:
|
|
350
|
+
description: Replica or Did already exists
|
|
351
|
+
503:
|
|
352
|
+
description: Resource temporary unavailable
|
|
353
|
+
"""
|
|
354
|
+
parameters = json_parameters(parse_response)
|
|
355
|
+
rse = param_get(parameters, 'rse')
|
|
356
|
+
files = param_get(parameters, 'files')
|
|
357
|
+
|
|
358
|
+
try:
|
|
359
|
+
add_replicas(
|
|
360
|
+
rse=rse,
|
|
361
|
+
files=files,
|
|
362
|
+
issuer=request.environ.get('issuer'),
|
|
363
|
+
vo=request.environ.get('vo'),
|
|
364
|
+
ignore_availability=param_get(parameters, 'ignore_availability', default=False),
|
|
365
|
+
)
|
|
366
|
+
except InvalidPath as error:
|
|
367
|
+
return generate_http_error_flask(400, error)
|
|
368
|
+
except AccessDenied as error:
|
|
369
|
+
return generate_http_error_flask(401, error)
|
|
370
|
+
except (Duplicate, DataIdentifierAlreadyExists) as error:
|
|
371
|
+
return generate_http_error_flask(409, error)
|
|
372
|
+
except (RSENotFound, ScopeNotFound) as error:
|
|
373
|
+
return generate_http_error_flask(404, error)
|
|
374
|
+
except ResourceTemporaryUnavailable as error:
|
|
375
|
+
return generate_http_error_flask(503, error)
|
|
376
|
+
|
|
377
|
+
return 'Created', 201
|
|
378
|
+
|
|
379
|
+
def put(self):
|
|
380
|
+
"""
|
|
381
|
+
---
|
|
382
|
+
summary: Update File Replicas
|
|
383
|
+
description: Update file replicas state at a given RSE.
|
|
384
|
+
tags:
|
|
385
|
+
- Replicas
|
|
386
|
+
requestBody:
|
|
387
|
+
content:
|
|
388
|
+
application/json:
|
|
389
|
+
schema:
|
|
390
|
+
type: object
|
|
391
|
+
required:
|
|
392
|
+
- rse
|
|
393
|
+
- files
|
|
394
|
+
properties:
|
|
395
|
+
rse:
|
|
396
|
+
description: The rse for the replication
|
|
397
|
+
type: string
|
|
398
|
+
files:
|
|
399
|
+
description: The files to replicate
|
|
400
|
+
type: array
|
|
401
|
+
items:
|
|
402
|
+
type: object
|
|
403
|
+
properties:
|
|
404
|
+
name:
|
|
405
|
+
description: The pfn of the replica.
|
|
406
|
+
type: string
|
|
407
|
+
state:
|
|
408
|
+
description: The pfn of the replica.
|
|
409
|
+
type: string
|
|
410
|
+
path:
|
|
411
|
+
description: The pfn of the replica.
|
|
412
|
+
type: string
|
|
413
|
+
error_message:
|
|
414
|
+
description: The error message if an error occurred.
|
|
415
|
+
type: string
|
|
416
|
+
broken_rule_id:
|
|
417
|
+
description: The id of the broken rule if one was found.
|
|
418
|
+
type: string
|
|
419
|
+
broken_message:
|
|
420
|
+
description: The message of the broken rule.
|
|
421
|
+
type: string
|
|
422
|
+
responses:
|
|
423
|
+
200:
|
|
424
|
+
description: OK
|
|
425
|
+
400:
|
|
426
|
+
description: Cannot decode json parameter list
|
|
427
|
+
401:
|
|
428
|
+
description: Invalid Auth Token
|
|
429
|
+
"""
|
|
430
|
+
parameters = json_parameters(parse_response)
|
|
431
|
+
rse = param_get(parameters, 'rse')
|
|
432
|
+
files = param_get(parameters, 'files')
|
|
433
|
+
|
|
434
|
+
try:
|
|
435
|
+
update_replicas_states(rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
|
|
436
|
+
except AccessDenied as error:
|
|
437
|
+
return generate_http_error_flask(401, error)
|
|
438
|
+
|
|
439
|
+
return '', 200
|
|
440
|
+
|
|
441
|
+
def delete(self):
|
|
442
|
+
"""
|
|
443
|
+
---
|
|
444
|
+
summary: Delete File Replicas
|
|
445
|
+
description: Delete file replicas at a given RSE.
|
|
446
|
+
tags:
|
|
447
|
+
- Replicas
|
|
448
|
+
requestBody:
|
|
449
|
+
content:
|
|
450
|
+
application/json:
|
|
451
|
+
schema:
|
|
452
|
+
type: object
|
|
453
|
+
required:
|
|
454
|
+
- rse
|
|
455
|
+
- files
|
|
456
|
+
properties:
|
|
457
|
+
rse:
|
|
458
|
+
description: The rse name.
|
|
459
|
+
type: string
|
|
460
|
+
files:
|
|
461
|
+
description: The files to delete.
|
|
462
|
+
type: array
|
|
463
|
+
items:
|
|
464
|
+
type: object
|
|
465
|
+
required:
|
|
466
|
+
- name
|
|
467
|
+
properties:
|
|
468
|
+
name:
|
|
469
|
+
description: The name of the replica.
|
|
470
|
+
type: string
|
|
471
|
+
responses:
|
|
472
|
+
200:
|
|
473
|
+
description: OK
|
|
474
|
+
400:
|
|
475
|
+
description: Cannot decode json parameter list.
|
|
476
|
+
401:
|
|
477
|
+
description: Invalid Auth Token
|
|
478
|
+
404:
|
|
479
|
+
description: Rse or Replica not found
|
|
480
|
+
"""
|
|
481
|
+
parameters = json_parameters(parse_response)
|
|
482
|
+
rse = param_get(parameters, 'rse')
|
|
483
|
+
files = param_get(parameters, 'files')
|
|
484
|
+
|
|
485
|
+
try:
|
|
486
|
+
delete_replicas(
|
|
487
|
+
rse=rse,
|
|
488
|
+
files=files,
|
|
489
|
+
issuer=request.environ.get('issuer'),
|
|
490
|
+
vo=request.environ.get('vo'),
|
|
491
|
+
ignore_availability=param_get(parameters, 'ignore_availability', default=False),
|
|
492
|
+
)
|
|
493
|
+
except AccessDenied as error:
|
|
494
|
+
return generate_http_error_flask(401, error)
|
|
495
|
+
except (RSENotFound, ReplicaNotFound) as error:
|
|
496
|
+
return generate_http_error_flask(404, error)
|
|
497
|
+
except ResourceTemporaryUnavailable as error:
|
|
498
|
+
return generate_http_error_flask(503, error)
|
|
499
|
+
|
|
500
|
+
return '', 200
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
class ListReplicas(ErrorHandlingMethodView):
|
|
504
|
+
|
|
505
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
|
|
506
|
+
def post(self):
|
|
507
|
+
"""
|
|
508
|
+
---
|
|
509
|
+
summary: List Replicas
|
|
510
|
+
description: List all replicas for a DID.
|
|
511
|
+
tags:
|
|
512
|
+
- Replicas
|
|
513
|
+
parameters:
|
|
514
|
+
- name: X-Forwarded-For
|
|
515
|
+
in: header
|
|
516
|
+
description: The client ip address.
|
|
517
|
+
schema:
|
|
518
|
+
type: string
|
|
519
|
+
- name: limit
|
|
520
|
+
in: query
|
|
521
|
+
description: The maximum number pfns per replica to return.
|
|
522
|
+
schema:
|
|
523
|
+
type: integer
|
|
524
|
+
- name: select
|
|
525
|
+
in: query
|
|
526
|
+
description: Requested sorting of the result, e.g., 'geoip', 'random'.
|
|
527
|
+
schema:
|
|
528
|
+
type: string
|
|
529
|
+
- name: sort
|
|
530
|
+
in: query
|
|
531
|
+
description: Requested sorting of the result, e.g., 'geoip', 'random'.
|
|
532
|
+
schema:
|
|
533
|
+
type: string
|
|
534
|
+
requestBody:
|
|
535
|
+
content:
|
|
536
|
+
application/json:
|
|
537
|
+
schema:
|
|
538
|
+
type: object
|
|
539
|
+
properties:
|
|
540
|
+
client_location:
|
|
541
|
+
description: The clients location.
|
|
542
|
+
type: string
|
|
543
|
+
dids:
|
|
544
|
+
description: List of Dids.
|
|
545
|
+
type: array
|
|
546
|
+
items:
|
|
547
|
+
type: object
|
|
548
|
+
properties:
|
|
549
|
+
scope:
|
|
550
|
+
description: The scope of the did.
|
|
551
|
+
type: string
|
|
552
|
+
name:
|
|
553
|
+
description: The name of the did.
|
|
554
|
+
type: string
|
|
555
|
+
schemes:
|
|
556
|
+
description: A list of schemes to filter the replicas.
|
|
557
|
+
type: array
|
|
558
|
+
items:
|
|
559
|
+
type: string
|
|
560
|
+
sort:
|
|
561
|
+
description: Requested sorting of the result, e.g., 'geoip', 'random'.
|
|
562
|
+
type: string
|
|
563
|
+
unavailable:
|
|
564
|
+
description: If unavailable rse should be considered.
|
|
565
|
+
type: boolean
|
|
566
|
+
deprecated: true
|
|
567
|
+
ignore_availability:
|
|
568
|
+
description: If the availability should be ignored.
|
|
569
|
+
type: boolean
|
|
570
|
+
rse_expression:
|
|
571
|
+
description: The RSE expression to restrict on a list of RSEs.
|
|
572
|
+
type: string
|
|
573
|
+
all_states:
|
|
574
|
+
description: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
|
|
575
|
+
type: boolean
|
|
576
|
+
domain:
|
|
577
|
+
description: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
|
|
578
|
+
type: string
|
|
579
|
+
signature_lifetime:
|
|
580
|
+
description: If supported, in seconds, restrict the lifetime of the signed PFN.
|
|
581
|
+
type: integer
|
|
582
|
+
resolve_archives:
|
|
583
|
+
description: When set to True, find archives which contain the replicas.
|
|
584
|
+
type: boolean
|
|
585
|
+
resolve_parents:
|
|
586
|
+
description: When set to True, find all parent datasets which contain the replicas.
|
|
587
|
+
type: boolean
|
|
588
|
+
updated_after:
|
|
589
|
+
description: datetime object (UTC time), only return replicas updated after this time
|
|
590
|
+
type: string
|
|
591
|
+
nrandom:
|
|
592
|
+
description: The maximum number of replicas to return.
|
|
593
|
+
type: integer
|
|
594
|
+
responses:
|
|
595
|
+
200:
|
|
596
|
+
description: OK
|
|
597
|
+
content:
|
|
598
|
+
application/json:
|
|
599
|
+
schema:
|
|
600
|
+
type: array
|
|
601
|
+
items:
|
|
602
|
+
type: object
|
|
603
|
+
properties:
|
|
604
|
+
scope:
|
|
605
|
+
description: The scope of the replica.
|
|
606
|
+
type: string
|
|
607
|
+
name:
|
|
608
|
+
description: The name of the replica.
|
|
609
|
+
type: string
|
|
610
|
+
bytes:
|
|
611
|
+
description: The size of the replica in bytes.
|
|
612
|
+
type: integer
|
|
613
|
+
md5:
|
|
614
|
+
description: The md5 checksum.
|
|
615
|
+
type: string
|
|
616
|
+
adler32:
|
|
617
|
+
description: The adler32 checksum.
|
|
618
|
+
type: string
|
|
619
|
+
pfns:
|
|
620
|
+
description: The pfns.
|
|
621
|
+
type: array
|
|
622
|
+
rses:
|
|
623
|
+
description: The RSESs.
|
|
624
|
+
type: array
|
|
625
|
+
application/metalink4+xml:
|
|
626
|
+
schema:
|
|
627
|
+
type: object
|
|
628
|
+
properties:
|
|
629
|
+
scope:
|
|
630
|
+
description: The scope of the replica.
|
|
631
|
+
type: string
|
|
632
|
+
name:
|
|
633
|
+
description: The name of the replica.
|
|
634
|
+
type: string
|
|
635
|
+
bytes:
|
|
636
|
+
description: The size of the replica in bytes.
|
|
637
|
+
type: integer
|
|
638
|
+
md5:
|
|
639
|
+
description: The md5 checksum.
|
|
640
|
+
type: string
|
|
641
|
+
adler32:
|
|
642
|
+
description: The adler32 checksum.
|
|
643
|
+
type: string
|
|
644
|
+
pfns:
|
|
645
|
+
description: The pfns.
|
|
646
|
+
type: array
|
|
647
|
+
rses:
|
|
648
|
+
description: The RSESs.
|
|
649
|
+
type: array
|
|
650
|
+
400:
|
|
651
|
+
description: Cannot decode json parameter list.
|
|
652
|
+
401:
|
|
653
|
+
description: Invalid Auth Token
|
|
654
|
+
404:
|
|
655
|
+
description: Did not found.
|
|
656
|
+
406:
|
|
657
|
+
description: Not acceptable
|
|
658
|
+
"""
|
|
659
|
+
content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
|
|
660
|
+
metalink = (content_type == 'application/metalink4+xml')
|
|
661
|
+
|
|
662
|
+
client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
|
|
663
|
+
|
|
664
|
+
parameters = json_parameters(parse_response)
|
|
665
|
+
|
|
666
|
+
client_location: 'IPDict' = {'ip': client_ip,
|
|
667
|
+
'fqdn': None,
|
|
668
|
+
'site': None}
|
|
669
|
+
client_location.update(param_get(parameters, 'client_location', default={}))
|
|
670
|
+
|
|
671
|
+
# making sure IP address is not overwritten
|
|
672
|
+
client_location['ip'] = client_ip
|
|
673
|
+
|
|
674
|
+
dids = param_get(parameters, 'dids', default=[])
|
|
675
|
+
schemes = param_get(parameters, 'schemes', default=None)
|
|
676
|
+
select = param_get(parameters, 'sort', default=None)
|
|
677
|
+
unavailable = param_get(parameters, 'unavailable', default=False)
|
|
678
|
+
ignore_availability = param_get(parameters, 'ignore_availability', default='unavailable' in parameters)
|
|
679
|
+
rse_expression = param_get(parameters, 'rse_expression', default=None)
|
|
680
|
+
all_states = param_get(parameters, 'all_states', default=False)
|
|
681
|
+
domain = param_get(parameters, 'domain', default=None)
|
|
682
|
+
if 'signature_lifetime' in parameters:
|
|
683
|
+
signature_lifetime = param_get(parameters, 'signature_lifetime')
|
|
684
|
+
else:
|
|
685
|
+
# hardcoded default of 10 minutes if config is not parseable
|
|
686
|
+
signature_lifetime = config_get_int('credentials', 'signature_lifetime', raise_exception=False, default=600)
|
|
687
|
+
resolve_archives = param_get(parameters, 'resolve_archives', default=True)
|
|
688
|
+
resolve_parents = param_get(parameters, 'resolve_parents', default=False)
|
|
689
|
+
updated_after = param_get(parameters, 'updated_after', default=None)
|
|
690
|
+
if updated_after is not None:
|
|
691
|
+
if isinstance(updated_after, (int, float)):
|
|
692
|
+
# convert from epoch time stamp to datetime object
|
|
693
|
+
updated_after = datetime.utcfromtimestamp(updated_after)
|
|
694
|
+
else:
|
|
695
|
+
# attempt UTC format '%Y-%m-%dT%H:%M:%S' conversion
|
|
696
|
+
updated_after = datetime.strptime(updated_after, '%Y-%m-%dT%H:%M:%S')
|
|
697
|
+
nrandom = param_get(parameters, 'nrandom', default=None)
|
|
698
|
+
if nrandom:
|
|
699
|
+
nrandom = int(nrandom)
|
|
700
|
+
|
|
701
|
+
limit = request.args.get('limit', default=None)
|
|
702
|
+
select = request.args.get('select', default=select)
|
|
703
|
+
select = request.args.get('sort', default=select)
|
|
704
|
+
|
|
705
|
+
# Resolve all reasonable protocols when doing metalink for maximum access possibilities
|
|
706
|
+
if metalink and schemes is None:
|
|
707
|
+
schemes = SUPPORTED_PROTOCOLS
|
|
708
|
+
|
|
709
|
+
content_type = 'application/metalink4+xml' if metalink else 'application/x-json-stream'
|
|
710
|
+
|
|
711
|
+
try:
|
|
712
|
+
def _list_and_sort_replicas(request_id, issuer, vo):
|
|
713
|
+
# we need to call list_replicas before starting to reply
|
|
714
|
+
# otherwise the exceptions won't be propagated correctly
|
|
715
|
+
for rfile in list_replicas(dids=dids, schemes=schemes,
|
|
716
|
+
unavailable=unavailable,
|
|
717
|
+
request_id=request_id,
|
|
718
|
+
ignore_availability=ignore_availability,
|
|
719
|
+
all_states=all_states,
|
|
720
|
+
rse_expression=rse_expression,
|
|
721
|
+
client_location=client_location,
|
|
722
|
+
domain=domain,
|
|
723
|
+
signature_lifetime=signature_lifetime,
|
|
724
|
+
resolve_archives=resolve_archives,
|
|
725
|
+
resolve_parents=resolve_parents,
|
|
726
|
+
nrandom=nrandom,
|
|
727
|
+
updated_after=updated_after,
|
|
728
|
+
issuer=issuer,
|
|
729
|
+
vo=vo):
|
|
730
|
+
|
|
731
|
+
# Sort rfile['pfns'] and limit its size according to "limit" parameter
|
|
732
|
+
lanreplicas = {}
|
|
733
|
+
wanreplicas = {}
|
|
734
|
+
for pfn, replica in rfile['pfns'].items():
|
|
735
|
+
replica_tuple = (replica['domain'], replica['priority'], replica['rse'], replica['client_extract'])
|
|
736
|
+
if replica_tuple[0] == 'lan':
|
|
737
|
+
lanreplicas[pfn] = replica_tuple
|
|
738
|
+
else:
|
|
739
|
+
wanreplicas[pfn] = replica_tuple
|
|
740
|
+
|
|
741
|
+
rfile['pfns'] = dict(_sorted_with_priorities(replicas=rfile['pfns'],
|
|
742
|
+
# Lan replicas sorted by priority; followed by wan replicas sorted by selection criteria
|
|
743
|
+
sorted_pfns=chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
|
|
744
|
+
sort_replicas(wanreplicas, client_location, selection=select)),
|
|
745
|
+
limit=limit))
|
|
746
|
+
yield rfile
|
|
747
|
+
|
|
748
|
+
rfiles = _list_and_sort_replicas(request_id=request.environ.get('request_id'),
|
|
749
|
+
issuer=request.environ.get('issuer'),
|
|
750
|
+
vo=request.environ.get('vo'))
|
|
751
|
+
if metalink:
|
|
752
|
+
policy_schema = config_get('policy', 'schema', raise_exception=False, default='generic')
|
|
753
|
+
response_generator = _generate_metalink_response(rfiles, policy_schema)
|
|
754
|
+
else:
|
|
755
|
+
response_generator = _generate_json_response(rfiles)
|
|
756
|
+
return try_stream(response_generator, content_type=content_type)
|
|
757
|
+
except (InvalidObject, DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
|
|
758
|
+
return generate_http_error_flask(400, error)
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
class ReplicasDIDs(ErrorHandlingMethodView):
|
|
762
|
+
|
|
763
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
764
|
+
def post(self):
|
|
765
|
+
"""
|
|
766
|
+
---
|
|
767
|
+
summary: List Replicas Dids
|
|
768
|
+
description: List the DIDs associated to a list of replicas.
|
|
769
|
+
tags:
|
|
770
|
+
- Replicas
|
|
771
|
+
requestBody:
|
|
772
|
+
content:
|
|
773
|
+
application/json:
|
|
774
|
+
schema:
|
|
775
|
+
type: object
|
|
776
|
+
required:
|
|
777
|
+
- rse
|
|
778
|
+
properties:
|
|
779
|
+
pfns:
|
|
780
|
+
description: The list of pfns.
|
|
781
|
+
type: array
|
|
782
|
+
items:
|
|
783
|
+
type: string
|
|
784
|
+
rse:
|
|
785
|
+
description: The RSE name.
|
|
786
|
+
type: string
|
|
787
|
+
responses:
|
|
788
|
+
200:
|
|
789
|
+
description: OK
|
|
790
|
+
content:
|
|
791
|
+
application/x-json-stream:
|
|
792
|
+
schema:
|
|
793
|
+
type: array
|
|
794
|
+
items:
|
|
795
|
+
type: object
|
|
796
|
+
additionalProperties:
|
|
797
|
+
x-additionalPropertiesName: mapped PFNs to DIDs
|
|
798
|
+
description: A mapping from a pfn to a did.
|
|
799
|
+
type: object
|
|
800
|
+
properties:
|
|
801
|
+
scope:
|
|
802
|
+
description: The scope of the DID.
|
|
803
|
+
type: string
|
|
804
|
+
name:
|
|
805
|
+
description: The name of the DID.
|
|
806
|
+
type: string
|
|
807
|
+
400:
|
|
808
|
+
description: Cannot decode json parameter list.
|
|
809
|
+
401:
|
|
810
|
+
description: Invalid Auth Token
|
|
811
|
+
404:
|
|
812
|
+
description: Not found
|
|
813
|
+
406:
|
|
814
|
+
description: Not acceptable
|
|
815
|
+
"""
|
|
816
|
+
parameters = json_parameters()
|
|
817
|
+
pfns = param_get(parameters, 'pfns', default=[])
|
|
818
|
+
rse = param_get(parameters, 'rse')
|
|
819
|
+
|
|
820
|
+
try:
|
|
821
|
+
def generate(vo):
|
|
822
|
+
for pfn in get_did_from_pfns(pfns, rse, vo=vo):
|
|
823
|
+
yield dumps(pfn) + '\n'
|
|
824
|
+
|
|
825
|
+
return try_stream(generate(vo=request.environ.get('vo')))
|
|
826
|
+
except AccessDenied as error:
|
|
827
|
+
return generate_http_error_flask(401, error)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
class BadReplicas(ErrorHandlingMethodView):
|
|
831
|
+
|
|
832
|
+
@check_accept_header_wrapper_flask(['application/json'])
|
|
833
|
+
def post(self):
|
|
834
|
+
"""
|
|
835
|
+
---
|
|
836
|
+
summary: Declare Bad Replicas
|
|
837
|
+
description: Declares a list of bad replicas.
|
|
838
|
+
tags:
|
|
839
|
+
- Replicas
|
|
840
|
+
requestBody:
|
|
841
|
+
content:
|
|
842
|
+
application/json:
|
|
843
|
+
schema:
|
|
844
|
+
type: object
|
|
845
|
+
properties:
|
|
846
|
+
replicas:
|
|
847
|
+
description: The list of pfns or list of dicts with "scope", "name", "rse_id"/"rse"
|
|
848
|
+
type: array
|
|
849
|
+
items:
|
|
850
|
+
type: string
|
|
851
|
+
pfns:
|
|
852
|
+
deprecated: true
|
|
853
|
+
description: The list of pfns, for backward compatibility with older versions of the ReplicaClient
|
|
854
|
+
type: array
|
|
855
|
+
items:
|
|
856
|
+
type: string
|
|
857
|
+
reason:
|
|
858
|
+
description: The reason for the declaration.
|
|
859
|
+
type: string
|
|
860
|
+
force:
|
|
861
|
+
description: If true, ignore existing replica status in the bad_replicas table.
|
|
862
|
+
type: boolean
|
|
863
|
+
responses:
|
|
864
|
+
201:
|
|
865
|
+
description: OK
|
|
866
|
+
content:
|
|
867
|
+
application/json:
|
|
868
|
+
schema:
|
|
869
|
+
description: Returns the not declared files.
|
|
870
|
+
type: array
|
|
871
|
+
400:
|
|
872
|
+
description: Can not decode json parameter list.
|
|
873
|
+
404:
|
|
874
|
+
description: Not found
|
|
875
|
+
406:
|
|
876
|
+
description: Not acceptable
|
|
877
|
+
"""
|
|
878
|
+
parameters = json_parameters()
|
|
879
|
+
replicas = param_get(parameters, 'replicas', default=[]) or param_get(parameters, 'pfns', default=[])
|
|
880
|
+
reason = param_get(parameters, 'reason', default=None)
|
|
881
|
+
force = param_get(parameters, 'force', default=False)
|
|
882
|
+
|
|
883
|
+
try:
|
|
884
|
+
not_declared_files = declare_bad_file_replicas(replicas, reason=reason,
|
|
885
|
+
issuer=request.environ.get('issuer'), vo=request.environ.get('vo'),
|
|
886
|
+
force=force)
|
|
887
|
+
return not_declared_files, 201
|
|
888
|
+
except AccessDenied as error:
|
|
889
|
+
return generate_http_error_flask(401, error)
|
|
890
|
+
except (RSENotFound, ReplicaNotFound) as error:
|
|
891
|
+
return generate_http_error_flask(404, error)
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
class QuarantineReplicas(ErrorHandlingMethodView):
|
|
895
|
+
|
|
896
|
+
def post(self):
|
|
897
|
+
"""
|
|
898
|
+
---
|
|
899
|
+
summary: Quarantine replicas
|
|
900
|
+
description: Quarantine replicas.
|
|
901
|
+
tags:
|
|
902
|
+
- Replicas
|
|
903
|
+
requestBody:
|
|
904
|
+
content:
|
|
905
|
+
application/json:
|
|
906
|
+
schema:
|
|
907
|
+
type: object
|
|
908
|
+
required:
|
|
909
|
+
- replicas
|
|
910
|
+
properties:
|
|
911
|
+
replicas:
|
|
912
|
+
description: replicas
|
|
913
|
+
type: array
|
|
914
|
+
items:
|
|
915
|
+
type: object
|
|
916
|
+
required:
|
|
917
|
+
- path
|
|
918
|
+
properties:
|
|
919
|
+
path:
|
|
920
|
+
description: path
|
|
921
|
+
type: string
|
|
922
|
+
scope:
|
|
923
|
+
description: scope
|
|
924
|
+
type: string
|
|
925
|
+
name:
|
|
926
|
+
description: name
|
|
927
|
+
type: string
|
|
928
|
+
rse:
|
|
929
|
+
description: RSE name
|
|
930
|
+
type: string
|
|
931
|
+
rse_id:
|
|
932
|
+
description: RSE id
|
|
933
|
+
type: string
|
|
934
|
+
responses:
|
|
935
|
+
200:
|
|
936
|
+
description: OK
|
|
937
|
+
403:
|
|
938
|
+
description: Forbidden.
|
|
939
|
+
404:
|
|
940
|
+
description: Not found
|
|
941
|
+
"""
|
|
942
|
+
|
|
943
|
+
parameters = json_parameters()
|
|
944
|
+
replicas = param_get(parameters, 'replicas', default=[])
|
|
945
|
+
rse = param_get(parameters, 'rse', default=None)
|
|
946
|
+
rse_id = param_get(parameters, 'rse_id', default=None)
|
|
947
|
+
vo = request.environ.get('vo')
|
|
948
|
+
issuer = request.environ.get('issuer')
|
|
949
|
+
|
|
950
|
+
if replicas:
|
|
951
|
+
try:
|
|
952
|
+
quarantine_file_replicas(replicas, issuer, rse=rse, rse_id=rse_id, vo=vo)
|
|
953
|
+
except AccessDenied as error:
|
|
954
|
+
return generate_http_error_flask(403, error)
|
|
955
|
+
except (RSENotFound, ReplicaNotFound) as error:
|
|
956
|
+
return generate_http_error_flask(404, error)
|
|
957
|
+
|
|
958
|
+
return '', 200
|
|
959
|
+
|
|
960
|
+
|
|
961
|
+
class SuspiciousReplicas(ErrorHandlingMethodView):
|
|
962
|
+
|
|
963
|
+
@check_accept_header_wrapper_flask(['application/json'])
|
|
964
|
+
def post(self):
|
|
965
|
+
"""
|
|
966
|
+
---
|
|
967
|
+
summary: Declare Suspicious Replicas
|
|
968
|
+
description: Declare a list of suspicious replicas.
|
|
969
|
+
tags:
|
|
970
|
+
- Replicas
|
|
971
|
+
requestBody:
|
|
972
|
+
content:
|
|
973
|
+
application/json:
|
|
974
|
+
schema:
|
|
975
|
+
type: object
|
|
976
|
+
properties:
|
|
977
|
+
pfns:
|
|
978
|
+
description: The list of pfns.
|
|
979
|
+
type: array
|
|
980
|
+
items:
|
|
981
|
+
type: string
|
|
982
|
+
reason:
|
|
983
|
+
description: The reason for the declaration.
|
|
984
|
+
type: string
|
|
985
|
+
responses:
|
|
986
|
+
201:
|
|
987
|
+
description: OK
|
|
988
|
+
content:
|
|
989
|
+
application/json:
|
|
990
|
+
schema:
|
|
991
|
+
description: Returns the not declared files.
|
|
992
|
+
type: array
|
|
993
|
+
400:
|
|
994
|
+
description: Can not decode json parameter list.
|
|
995
|
+
404:
|
|
996
|
+
description: Not found
|
|
997
|
+
406:
|
|
998
|
+
description: Not acceptable
|
|
999
|
+
"""
|
|
1000
|
+
parameters = json_parameters(parse_response)
|
|
1001
|
+
pfns = param_get(parameters, 'pfns', default=[])
|
|
1002
|
+
reason = param_get(parameters, 'reason', default=None)
|
|
1003
|
+
|
|
1004
|
+
try:
|
|
1005
|
+
not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
|
|
1006
|
+
return not_declared_files, 201
|
|
1007
|
+
except AccessDenied as error:
|
|
1008
|
+
return generate_http_error_flask(401, error)
|
|
1009
|
+
|
|
1010
|
+
@check_accept_header_wrapper_flask(['application/json'])
|
|
1011
|
+
def get(self):
|
|
1012
|
+
"""
|
|
1013
|
+
---
|
|
1014
|
+
summary: List Suspicious Replicas
|
|
1015
|
+
description: List the suspicious replicas on a list of RSEs.
|
|
1016
|
+
tags:
|
|
1017
|
+
- Replicas
|
|
1018
|
+
parameters:
|
|
1019
|
+
- name: rse_expression
|
|
1020
|
+
in: query
|
|
1021
|
+
description: The RSE expression to filter for.
|
|
1022
|
+
schema:
|
|
1023
|
+
type: string
|
|
1024
|
+
- name: younger_than
|
|
1025
|
+
in: query
|
|
1026
|
+
description: Date to filter for.
|
|
1027
|
+
schema:
|
|
1028
|
+
type: string
|
|
1029
|
+
- name: nattempts
|
|
1030
|
+
in: query
|
|
1031
|
+
description: The maximum number of attempts to make.
|
|
1032
|
+
schema:
|
|
1033
|
+
type: integer
|
|
1034
|
+
responses:
|
|
1035
|
+
200:
|
|
1036
|
+
description: OK
|
|
1037
|
+
content:
|
|
1038
|
+
application/json:
|
|
1039
|
+
schema:
|
|
1040
|
+
type: array
|
|
1041
|
+
items:
|
|
1042
|
+
type: object
|
|
1043
|
+
properties:
|
|
1044
|
+
scope:
|
|
1045
|
+
description: The scope of the Replica.
|
|
1046
|
+
type: string
|
|
1047
|
+
name:
|
|
1048
|
+
description: The name of the Replica.
|
|
1049
|
+
type: string
|
|
1050
|
+
rse:
|
|
1051
|
+
description: The rse name.
|
|
1052
|
+
type: string
|
|
1053
|
+
rse_id:
|
|
1054
|
+
description: The id of the rse.
|
|
1055
|
+
type: string
|
|
1056
|
+
cnt:
|
|
1057
|
+
description: The number of replicas.
|
|
1058
|
+
type: integer
|
|
1059
|
+
created_at:
|
|
1060
|
+
description: The time when the replica was created.
|
|
1061
|
+
type: string
|
|
1062
|
+
401:
|
|
1063
|
+
description: Invalid Auth Token
|
|
1064
|
+
404:
|
|
1065
|
+
description: Not found
|
|
1066
|
+
406:
|
|
1067
|
+
description: Not acceptable
|
|
1068
|
+
"""
|
|
1069
|
+
rse_expression, younger_than, nattempts = None, None, None
|
|
1070
|
+
if request.query_string:
|
|
1071
|
+
query_string = request.query_string.decode(encoding='utf-8')
|
|
1072
|
+
try:
|
|
1073
|
+
params = loads(unquote(query_string))
|
|
1074
|
+
except ValueError:
|
|
1075
|
+
params = parse_qs(query_string)
|
|
1076
|
+
|
|
1077
|
+
if 'rse_expression' in params:
|
|
1078
|
+
rse_expression = params['rse_expression'][0]
|
|
1079
|
+
if 'younger_than' in params and params['younger_than'][0]:
|
|
1080
|
+
younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S")
|
|
1081
|
+
if 'nattempts' in params:
|
|
1082
|
+
nattempts = int(params['nattempts'][0])
|
|
1083
|
+
|
|
1084
|
+
result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
|
|
1085
|
+
return Response(render_json(result), 200, content_type='application/json')
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
class BadReplicasStates(ErrorHandlingMethodView):
|
|
1089
|
+
|
|
1090
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1091
|
+
def get(self):
|
|
1092
|
+
"""
|
|
1093
|
+
---
|
|
1094
|
+
summary: List Bad Replicas By States
|
|
1095
|
+
description: List the bad or suspicious replicas by states.
|
|
1096
|
+
tags:
|
|
1097
|
+
- Replicas
|
|
1098
|
+
parameters:
|
|
1099
|
+
- name: state
|
|
1100
|
+
in: query
|
|
1101
|
+
description: The state of the file.
|
|
1102
|
+
schema:
|
|
1103
|
+
type: string
|
|
1104
|
+
enum: [SUSPICIOUS, BAD]
|
|
1105
|
+
- name: rse
|
|
1106
|
+
in: query
|
|
1107
|
+
description: The rse name.
|
|
1108
|
+
schema:
|
|
1109
|
+
type: string
|
|
1110
|
+
- name: younger_than
|
|
1111
|
+
in: query
|
|
1112
|
+
description: Date to select bad replicas younger than this date.
|
|
1113
|
+
schema:
|
|
1114
|
+
type: string
|
|
1115
|
+
format: date-time
|
|
1116
|
+
- name: older_than
|
|
1117
|
+
in: query
|
|
1118
|
+
description: Date to select bad replicas older than this date.
|
|
1119
|
+
schema:
|
|
1120
|
+
type: string
|
|
1121
|
+
format: date-time
|
|
1122
|
+
- name: limit
|
|
1123
|
+
in: query
|
|
1124
|
+
description: The maximum number of replicas returned.
|
|
1125
|
+
schema:
|
|
1126
|
+
type: integer
|
|
1127
|
+
- name: list_pfns
|
|
1128
|
+
in: query
|
|
1129
|
+
description: Flag to include pfns.
|
|
1130
|
+
schema:
|
|
1131
|
+
type: boolean
|
|
1132
|
+
responses:
|
|
1133
|
+
200:
|
|
1134
|
+
description: OK
|
|
1135
|
+
content:
|
|
1136
|
+
application/x-json-stream:
|
|
1137
|
+
schema:
|
|
1138
|
+
description: A list of all result replicas.
|
|
1139
|
+
type: array
|
|
1140
|
+
items:
|
|
1141
|
+
oneOf:
|
|
1142
|
+
- type: object
|
|
1143
|
+
properties:
|
|
1144
|
+
scope:
|
|
1145
|
+
description: The scope of the replica.
|
|
1146
|
+
type: string
|
|
1147
|
+
name:
|
|
1148
|
+
description: The name of the replica.
|
|
1149
|
+
type: string
|
|
1150
|
+
type:
|
|
1151
|
+
description: The type of the replica.
|
|
1152
|
+
type: string
|
|
1153
|
+
- type: object
|
|
1154
|
+
properties:
|
|
1155
|
+
scope:
|
|
1156
|
+
description: The scope of the replica.
|
|
1157
|
+
type: string
|
|
1158
|
+
name:
|
|
1159
|
+
description: The name of the replica.
|
|
1160
|
+
type: string
|
|
1161
|
+
rse:
|
|
1162
|
+
description: The name of the associated rse.
|
|
1163
|
+
type: string
|
|
1164
|
+
rse_id:
|
|
1165
|
+
description: The id of the associated rse.
|
|
1166
|
+
type: string
|
|
1167
|
+
state:
|
|
1168
|
+
description: The state of the replica.
|
|
1169
|
+
type: string
|
|
1170
|
+
created_at:
|
|
1171
|
+
description: The date-time the replica was created.
|
|
1172
|
+
type: string
|
|
1173
|
+
format: date-time
|
|
1174
|
+
updated_at:
|
|
1175
|
+
description: The date-time the replica was updated.
|
|
1176
|
+
type: string
|
|
1177
|
+
format: date-time
|
|
1178
|
+
401:
|
|
1179
|
+
description: Invalid Auth Token
|
|
1180
|
+
406:
|
|
1181
|
+
description: Not acceptable
|
|
1182
|
+
"""
|
|
1183
|
+
state, rse, younger_than, older_than, limit, list_pfns = None, None, None, None, None, None
|
|
1184
|
+
if request.query_string:
|
|
1185
|
+
query_string = request.query_string.decode(encoding='utf-8')
|
|
1186
|
+
try:
|
|
1187
|
+
params = loads(unquote(query_string))
|
|
1188
|
+
except ValueError:
|
|
1189
|
+
params = parse_qs(query_string)
|
|
1190
|
+
if 'state' in params:
|
|
1191
|
+
state = params['state'][0]
|
|
1192
|
+
if isinstance(state, str):
|
|
1193
|
+
state = BadFilesStatus(state)
|
|
1194
|
+
if 'rse' in params:
|
|
1195
|
+
rse = params['rse'][0]
|
|
1196
|
+
if 'younger_than' in params:
|
|
1197
|
+
younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
|
|
1198
|
+
if 'older_than' in params and params['older_than']:
|
|
1199
|
+
older_than = datetime.strptime(params['older_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
|
|
1200
|
+
if 'limit' in params:
|
|
1201
|
+
limit = int(params['limit'][0])
|
|
1202
|
+
if 'list_pfns' in params:
|
|
1203
|
+
list_pfns = bool(params['list_pfns'][0])
|
|
1204
|
+
|
|
1205
|
+
def generate(vo):
|
|
1206
|
+
for row in list_bad_replicas_status(state=state, rse=rse, younger_than=younger_than,
|
|
1207
|
+
older_than=older_than, limit=limit, list_pfns=list_pfns,
|
|
1208
|
+
vo=vo):
|
|
1209
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1210
|
+
|
|
1211
|
+
return try_stream(generate(vo=request.environ.get('vo')))
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
class BadReplicasSummary(ErrorHandlingMethodView):
|
|
1215
|
+
|
|
1216
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1217
|
+
def get(self):
|
|
1218
|
+
"""
|
|
1219
|
+
---
|
|
1220
|
+
summary: Bad Replicas Summary
|
|
1221
|
+
description: Return a summary of the bad replicas by incident.
|
|
1222
|
+
tags:
|
|
1223
|
+
- Replicas
|
|
1224
|
+
parameters:
|
|
1225
|
+
- name: rse_expression
|
|
1226
|
+
in: query
|
|
1227
|
+
description: The RSE expression.
|
|
1228
|
+
schema:
|
|
1229
|
+
type: string
|
|
1230
|
+
- name: from_date
|
|
1231
|
+
in: query
|
|
1232
|
+
description: The start date.
|
|
1233
|
+
schema:
|
|
1234
|
+
type: string
|
|
1235
|
+
format: date-time
|
|
1236
|
+
- name: to_date
|
|
1237
|
+
in: query
|
|
1238
|
+
description: The end date.
|
|
1239
|
+
schema:
|
|
1240
|
+
type: string
|
|
1241
|
+
format: date-time
|
|
1242
|
+
responses:
|
|
1243
|
+
200:
|
|
1244
|
+
description: OK
|
|
1245
|
+
content:
|
|
1246
|
+
application/x-json-stream:
|
|
1247
|
+
schema:
|
|
1248
|
+
description: A list of summaries.
|
|
1249
|
+
type: array
|
|
1250
|
+
items:
|
|
1251
|
+
type: object
|
|
1252
|
+
properties:
|
|
1253
|
+
rse:
|
|
1254
|
+
description: The name of the associated RSE.
|
|
1255
|
+
type: string
|
|
1256
|
+
rse_id:
|
|
1257
|
+
description: The id of the associated RSE.
|
|
1258
|
+
type: string
|
|
1259
|
+
created_at:
|
|
1260
|
+
description: The creation date-time.
|
|
1261
|
+
type: string
|
|
1262
|
+
format: date-time
|
|
1263
|
+
reason:
|
|
1264
|
+
description: The reason for the incident.
|
|
1265
|
+
type: string
|
|
1266
|
+
401:
|
|
1267
|
+
description: Invalid Auth Token
|
|
1268
|
+
406:
|
|
1269
|
+
description: Not acceptable
|
|
1270
|
+
"""
|
|
1271
|
+
rse_expression, from_date, to_date = None, None, None
|
|
1272
|
+
if request.query_string:
|
|
1273
|
+
query_string = request.query_string.decode(encoding='utf-8')
|
|
1274
|
+
try:
|
|
1275
|
+
params = loads(unquote(query_string))
|
|
1276
|
+
except ValueError:
|
|
1277
|
+
params = parse_qs(query_string)
|
|
1278
|
+
if 'rse_expression' in params:
|
|
1279
|
+
rse_expression = params['rse_expression'][0]
|
|
1280
|
+
if 'from_date' in params and params['from_date'][0]:
|
|
1281
|
+
from_date = datetime.strptime(params['from_date'][0], "%Y-%m-%d")
|
|
1282
|
+
if 'to_date' in params:
|
|
1283
|
+
to_date = datetime.strptime(params['to_date'][0], "%Y-%m-%d")
|
|
1284
|
+
|
|
1285
|
+
def generate(vo):
|
|
1286
|
+
for row in get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date,
|
|
1287
|
+
to_date=to_date, vo=vo):
|
|
1288
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1289
|
+
|
|
1290
|
+
return try_stream(generate(vo=request.environ.get('vo')))
|
|
1291
|
+
|
|
1292
|
+
|
|
1293
|
+
class DatasetReplicas(ErrorHandlingMethodView):
|
|
1294
|
+
|
|
1295
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1296
|
+
def get(self, scope_name):
|
|
1297
|
+
"""
|
|
1298
|
+
---
|
|
1299
|
+
summary: List Dataset Replicas
|
|
1300
|
+
description: List dataset replicas.
|
|
1301
|
+
tags:
|
|
1302
|
+
- Replicas
|
|
1303
|
+
parameters:
|
|
1304
|
+
- name: scope_name
|
|
1305
|
+
in: path
|
|
1306
|
+
description: data identifier (scope)/(name).
|
|
1307
|
+
schema:
|
|
1308
|
+
type: string
|
|
1309
|
+
style: simple
|
|
1310
|
+
- name: deep
|
|
1311
|
+
in: query
|
|
1312
|
+
description: Flag to ennable lookup at the file level.
|
|
1313
|
+
schema:
|
|
1314
|
+
type: boolean
|
|
1315
|
+
responses:
|
|
1316
|
+
200:
|
|
1317
|
+
description: OK
|
|
1318
|
+
content:
|
|
1319
|
+
application/x-json-stream:
|
|
1320
|
+
schema:
|
|
1321
|
+
description: A list of dataset replicas.
|
|
1322
|
+
type: array
|
|
1323
|
+
items:
|
|
1324
|
+
type: object
|
|
1325
|
+
properties:
|
|
1326
|
+
scope:
|
|
1327
|
+
description: The scope of the replica.
|
|
1328
|
+
type: string
|
|
1329
|
+
name:
|
|
1330
|
+
description: The name of the replica.
|
|
1331
|
+
type: string
|
|
1332
|
+
rse:
|
|
1333
|
+
description: The name of the associated RSE.
|
|
1334
|
+
type: string
|
|
1335
|
+
rse_id:
|
|
1336
|
+
description: The id of the associated RSE.
|
|
1337
|
+
type: string
|
|
1338
|
+
bytes:
|
|
1339
|
+
description: The size of the replica.
|
|
1340
|
+
type: integer
|
|
1341
|
+
length:
|
|
1342
|
+
description: The length of the replica.
|
|
1343
|
+
type: integer
|
|
1344
|
+
available_bytes:
|
|
1345
|
+
description: The number of available bytes of the replica.
|
|
1346
|
+
type: integer
|
|
1347
|
+
available_length:
|
|
1348
|
+
description: The available length of the replica.
|
|
1349
|
+
type: integer
|
|
1350
|
+
state:
|
|
1351
|
+
description: The state of the replica.
|
|
1352
|
+
type: string
|
|
1353
|
+
created_at:
|
|
1354
|
+
description: The date-time the replica was created.
|
|
1355
|
+
type: string
|
|
1356
|
+
format: date-time
|
|
1357
|
+
updated_at:
|
|
1358
|
+
description: The date-time the replica was updated.
|
|
1359
|
+
type: string
|
|
1360
|
+
format: date-time
|
|
1361
|
+
accessed_at:
|
|
1362
|
+
description: The date-time the replica was accessed.
|
|
1363
|
+
type: string
|
|
1364
|
+
format: date-time
|
|
1365
|
+
401:
|
|
1366
|
+
description: Invalid Auth Token
|
|
1367
|
+
404:
|
|
1368
|
+
description: Not found
|
|
1369
|
+
406:
|
|
1370
|
+
description: Not acceptable
|
|
1371
|
+
"""
|
|
1372
|
+
try:
|
|
1373
|
+
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
|
|
1374
|
+
|
|
1375
|
+
def generate(_deep, vo):
|
|
1376
|
+
for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
|
|
1377
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1378
|
+
|
|
1379
|
+
deep = request.args.get('deep', default=False)
|
|
1380
|
+
|
|
1381
|
+
return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
|
|
1382
|
+
except ValueError as error:
|
|
1383
|
+
return generate_http_error_flask(400, error)
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
class DatasetReplicasBulk(ErrorHandlingMethodView):
|
|
1387
|
+
|
|
1388
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1389
|
+
def post(self):
|
|
1390
|
+
"""
|
|
1391
|
+
---
|
|
1392
|
+
summary: List Dataset Replicas for Multiple DIDs
|
|
1393
|
+
description: List dataset replicas for multiple dids.
|
|
1394
|
+
tags:
|
|
1395
|
+
- Replicas
|
|
1396
|
+
requestBody:
|
|
1397
|
+
content:
|
|
1398
|
+
application/json:
|
|
1399
|
+
schema:
|
|
1400
|
+
type: object
|
|
1401
|
+
required:
|
|
1402
|
+
- dids
|
|
1403
|
+
properties:
|
|
1404
|
+
dids:
|
|
1405
|
+
description: A list of dids.
|
|
1406
|
+
type: array
|
|
1407
|
+
items:
|
|
1408
|
+
description: A did.
|
|
1409
|
+
type: object
|
|
1410
|
+
properties:
|
|
1411
|
+
scope:
|
|
1412
|
+
description: The scope of the did.
|
|
1413
|
+
type: string
|
|
1414
|
+
name:
|
|
1415
|
+
description: The name of the did.
|
|
1416
|
+
type: string
|
|
1417
|
+
responses:
|
|
1418
|
+
200:
|
|
1419
|
+
description: OK
|
|
1420
|
+
content:
|
|
1421
|
+
application/x-json-stream:
|
|
1422
|
+
schema:
|
|
1423
|
+
description: A list of dataset replicas.
|
|
1424
|
+
type: array
|
|
1425
|
+
items:
|
|
1426
|
+
type: object
|
|
1427
|
+
properties:
|
|
1428
|
+
scope:
|
|
1429
|
+
description: The scope of the replica.
|
|
1430
|
+
type: string
|
|
1431
|
+
name:
|
|
1432
|
+
description: The name of the replica.
|
|
1433
|
+
type: string
|
|
1434
|
+
rse:
|
|
1435
|
+
description: The name of the associated RSE.
|
|
1436
|
+
type: string
|
|
1437
|
+
rse_id:
|
|
1438
|
+
description: The id of the associated RSE.
|
|
1439
|
+
type: string
|
|
1440
|
+
bytes:
|
|
1441
|
+
description: The size of the replica.
|
|
1442
|
+
type: integer
|
|
1443
|
+
length:
|
|
1444
|
+
description: The length of the replica.
|
|
1445
|
+
type: integer
|
|
1446
|
+
available_bytes:
|
|
1447
|
+
description: The number of available bytes of the replica.
|
|
1448
|
+
type: integer
|
|
1449
|
+
available_length:
|
|
1450
|
+
description: The available length of the replica.
|
|
1451
|
+
type: integer
|
|
1452
|
+
state:
|
|
1453
|
+
description: The state of the replica.
|
|
1454
|
+
type: string
|
|
1455
|
+
created_at:
|
|
1456
|
+
description: The date-time the replica was created.
|
|
1457
|
+
type: string
|
|
1458
|
+
format: date-time
|
|
1459
|
+
updated_at:
|
|
1460
|
+
description: The date-time the replica was updated.
|
|
1461
|
+
type: string
|
|
1462
|
+
format: date-time
|
|
1463
|
+
accessed_at:
|
|
1464
|
+
description: The date-time the replica was accessed.
|
|
1465
|
+
type: string
|
|
1466
|
+
format: date-time
|
|
1467
|
+
400:
|
|
1468
|
+
description: Bad Request.
|
|
1469
|
+
401:
|
|
1470
|
+
description: Invalid Auth Token
|
|
1471
|
+
404:
|
|
1472
|
+
description: Not found
|
|
1473
|
+
406:
|
|
1474
|
+
description: Not acceptable
|
|
1475
|
+
"""
|
|
1476
|
+
parameters = json_parameters(parse_response)
|
|
1477
|
+
dids = param_get(parameters, 'dids')
|
|
1478
|
+
if len(dids) == 0:
|
|
1479
|
+
return generate_http_error_flask(400, ValueError.__name__, 'List of DIDs is empty')
|
|
1480
|
+
|
|
1481
|
+
try:
|
|
1482
|
+
def generate(vo):
|
|
1483
|
+
for row in list_dataset_replicas_bulk(dids=dids, vo=vo):
|
|
1484
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1485
|
+
|
|
1486
|
+
return try_stream(generate(vo=request.environ.get('vo')))
|
|
1487
|
+
except InvalidObject as error:
|
|
1488
|
+
return generate_http_error_flask(400, error, f'Cannot validate DIDs: {error}')
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
class DatasetReplicasVP(ErrorHandlingMethodView):
|
|
1492
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1493
|
+
def get(self, scope_name):
|
|
1494
|
+
"""
|
|
1495
|
+
---
|
|
1496
|
+
summary: List Dataset Replicas VP
|
|
1497
|
+
description: |
|
|
1498
|
+
List dataset replicas using the Virtual Placement service.
|
|
1499
|
+
This is an RnD function and might change or go away at any time.
|
|
1500
|
+
tags:
|
|
1501
|
+
- Replicas
|
|
1502
|
+
parameters:
|
|
1503
|
+
- name: scope_name
|
|
1504
|
+
in: path
|
|
1505
|
+
description: data identifier (scope)/(name).
|
|
1506
|
+
schema:
|
|
1507
|
+
type: string
|
|
1508
|
+
style: simple
|
|
1509
|
+
- name: deep
|
|
1510
|
+
in: query
|
|
1511
|
+
description: Flag to ennable lookup at the file level.
|
|
1512
|
+
schema:
|
|
1513
|
+
type: boolean
|
|
1514
|
+
responses:
|
|
1515
|
+
200:
|
|
1516
|
+
description: OK. This needs documentation!
|
|
1517
|
+
401:
|
|
1518
|
+
description: Invalid Auth Token
|
|
1519
|
+
406:
|
|
1520
|
+
description: Not acceptable
|
|
1521
|
+
"""
|
|
1522
|
+
try:
|
|
1523
|
+
scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
|
|
1524
|
+
|
|
1525
|
+
def generate(_deep, vo):
|
|
1526
|
+
for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
|
|
1527
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1528
|
+
|
|
1529
|
+
deep = request.args.get('deep', default=False)
|
|
1530
|
+
|
|
1531
|
+
return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
|
|
1532
|
+
except ValueError as error:
|
|
1533
|
+
return generate_http_error_flask(400, error)
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
class ReplicasRSE(ErrorHandlingMethodView):
|
|
1537
|
+
|
|
1538
|
+
@check_accept_header_wrapper_flask(['application/x-json-stream'])
|
|
1539
|
+
def get(self, rse):
|
|
1540
|
+
"""
|
|
1541
|
+
---
|
|
1542
|
+
summary: List Dataset Replicas per RSE
|
|
1543
|
+
description: List dataset replicas per RSE.
|
|
1544
|
+
tags:
|
|
1545
|
+
- Replicas
|
|
1546
|
+
parameters:
|
|
1547
|
+
- name: rse
|
|
1548
|
+
in: path
|
|
1549
|
+
description: The rse to filter for.
|
|
1550
|
+
schema:
|
|
1551
|
+
type: string
|
|
1552
|
+
style: simple
|
|
1553
|
+
responses:
|
|
1554
|
+
200:
|
|
1555
|
+
description: OK
|
|
1556
|
+
content:
|
|
1557
|
+
application/x-json-stream:
|
|
1558
|
+
schema:
|
|
1559
|
+
description: A list of dataset replicas.
|
|
1560
|
+
type: array
|
|
1561
|
+
items:
|
|
1562
|
+
type: object
|
|
1563
|
+
properties:
|
|
1564
|
+
scope:
|
|
1565
|
+
description: The scope of the replica.
|
|
1566
|
+
type: string
|
|
1567
|
+
name:
|
|
1568
|
+
description: The name of the replica.
|
|
1569
|
+
type: string
|
|
1570
|
+
rse:
|
|
1571
|
+
description: The name of the associated RSE.
|
|
1572
|
+
type: string
|
|
1573
|
+
rse_id:
|
|
1574
|
+
description: The id of the associated RSE.
|
|
1575
|
+
type: string
|
|
1576
|
+
bytes:
|
|
1577
|
+
description: The size of the replica.
|
|
1578
|
+
type: integer
|
|
1579
|
+
length:
|
|
1580
|
+
description: The length of the replica.
|
|
1581
|
+
type: integer
|
|
1582
|
+
available_bytes:
|
|
1583
|
+
description: The number of available bytes of the replica.
|
|
1584
|
+
type: integer
|
|
1585
|
+
available_length:
|
|
1586
|
+
description: The available length of the replica.
|
|
1587
|
+
type: integer
|
|
1588
|
+
state:
|
|
1589
|
+
description: The state of the replica.
|
|
1590
|
+
type: string
|
|
1591
|
+
created_at:
|
|
1592
|
+
description: The date-time the replica was created.
|
|
1593
|
+
type: string
|
|
1594
|
+
format: date-time
|
|
1595
|
+
updated_at:
|
|
1596
|
+
description: The date-time the replica was updated.
|
|
1597
|
+
type: string
|
|
1598
|
+
format: date-time
|
|
1599
|
+
accessed_at:
|
|
1600
|
+
description: The date-time the replica was accessed.
|
|
1601
|
+
type: string
|
|
1602
|
+
format: date-time
|
|
1603
|
+
401:
|
|
1604
|
+
description: Invalid Auth Token
|
|
1605
|
+
406:
|
|
1606
|
+
description: Not acceptable
|
|
1607
|
+
"""
|
|
1608
|
+
|
|
1609
|
+
def generate(vo):
|
|
1610
|
+
for row in list_datasets_per_rse(rse=rse, vo=vo):
|
|
1611
|
+
yield dumps(row, cls=APIEncoder) + '\n'
|
|
1612
|
+
|
|
1613
|
+
return try_stream(generate(vo=request.environ.get('vo')))
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
class BadDIDs(ErrorHandlingMethodView):
|
|
1617
|
+
|
|
1618
|
+
def post(self):
|
|
1619
|
+
"""
|
|
1620
|
+
---
|
|
1621
|
+
summary: Mark Bad by DID
|
|
1622
|
+
description: Declare a list of bad replicas by DID.
|
|
1623
|
+
tags:
|
|
1624
|
+
- Replicas
|
|
1625
|
+
requestBody:
|
|
1626
|
+
content:
|
|
1627
|
+
application/json:
|
|
1628
|
+
schema:
|
|
1629
|
+
type: object
|
|
1630
|
+
properties:
|
|
1631
|
+
expires_at:
|
|
1632
|
+
description: The expires at value.
|
|
1633
|
+
type: string
|
|
1634
|
+
format: date-time
|
|
1635
|
+
dids:
|
|
1636
|
+
description: The list of dids associated with the bad replicas.
|
|
1637
|
+
type: array
|
|
1638
|
+
items:
|
|
1639
|
+
type: object
|
|
1640
|
+
properties:
|
|
1641
|
+
scope:
|
|
1642
|
+
description: The scope of the did.
|
|
1643
|
+
type: string
|
|
1644
|
+
name:
|
|
1645
|
+
description: The name of the did.
|
|
1646
|
+
type: string
|
|
1647
|
+
rse:
|
|
1648
|
+
description: The name of the rse.
|
|
1649
|
+
type: string
|
|
1650
|
+
reason:
|
|
1651
|
+
description: The reason for the change.
|
|
1652
|
+
type: string
|
|
1653
|
+
responses:
|
|
1654
|
+
201:
|
|
1655
|
+
description: OK
|
|
1656
|
+
content:
|
|
1657
|
+
application/json:
|
|
1658
|
+
schema:
|
|
1659
|
+
description: All files not declared as bad.
|
|
1660
|
+
type: array
|
|
1661
|
+
items:
|
|
1662
|
+
type: string
|
|
1663
|
+
400:
|
|
1664
|
+
description: Cannot decode json parameter list.
|
|
1665
|
+
401:
|
|
1666
|
+
description: Invalid Auth Token
|
|
1667
|
+
404:
|
|
1668
|
+
description: Not found
|
|
1669
|
+
"""
|
|
1670
|
+
parameters = json_parameters(parse_response)
|
|
1671
|
+
expires_at = param_get(parameters, 'expires_at', default=None)
|
|
1672
|
+
if expires_at:
|
|
1673
|
+
expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
|
|
1674
|
+
|
|
1675
|
+
try:
|
|
1676
|
+
not_declared_files = add_bad_dids(
|
|
1677
|
+
dids=param_get(parameters, 'dids', default=[]),
|
|
1678
|
+
rse=param_get(parameters, 'rse', default=None),
|
|
1679
|
+
issuer=request.environ.get('issuer'),
|
|
1680
|
+
state=BadFilesStatus.BAD,
|
|
1681
|
+
reason=param_get(parameters, 'reason', default=None),
|
|
1682
|
+
expires_at=expires_at,
|
|
1683
|
+
vo=request.environ.get('vo'),
|
|
1684
|
+
)
|
|
1685
|
+
except (ValueError, InvalidType) as error:
|
|
1686
|
+
return generate_http_error_flask(400, ValueError.__name__, error.args[0])
|
|
1687
|
+
except AccessDenied as error:
|
|
1688
|
+
return generate_http_error_flask(401, error)
|
|
1689
|
+
except ReplicaNotFound as error:
|
|
1690
|
+
return generate_http_error_flask(404, error)
|
|
1691
|
+
except Duplicate as error:
|
|
1692
|
+
return generate_http_error_flask(409, error)
|
|
1693
|
+
|
|
1694
|
+
return Response(dumps(not_declared_files), status=201, content_type='application/json')
|
|
1695
|
+
|
|
1696
|
+
|
|
1697
|
+
class BadPFNs(ErrorHandlingMethodView):
|
|
1698
|
+
|
|
1699
|
+
def post(self):
|
|
1700
|
+
"""
|
|
1701
|
+
---
|
|
1702
|
+
summary: Declare Bad PFNs
|
|
1703
|
+
description: Declare a list of bad PFNs.
|
|
1704
|
+
tags:
|
|
1705
|
+
- Replicas
|
|
1706
|
+
requestBody:
|
|
1707
|
+
content:
|
|
1708
|
+
application/json:
|
|
1709
|
+
schema:
|
|
1710
|
+
type: object
|
|
1711
|
+
properties:
|
|
1712
|
+
expires_at:
|
|
1713
|
+
description: The expires at value. Only apply to TEMPORARY_UNAVAILABLE.
|
|
1714
|
+
type: string
|
|
1715
|
+
format: date-time
|
|
1716
|
+
pfns:
|
|
1717
|
+
description: The list of pfns associated with the bad PFNs.
|
|
1718
|
+
type: array
|
|
1719
|
+
items:
|
|
1720
|
+
type: string
|
|
1721
|
+
state:
|
|
1722
|
+
description: The state to set the PFNs to.
|
|
1723
|
+
type: string
|
|
1724
|
+
enum: ["BAD", "SUSPICIOUS", "TEMPORARY_UNAVAILABLE"]
|
|
1725
|
+
reason:
|
|
1726
|
+
description: The reason for the change.
|
|
1727
|
+
type: string
|
|
1728
|
+
responses:
|
|
1729
|
+
201:
|
|
1730
|
+
description: Created
|
|
1731
|
+
400:
|
|
1732
|
+
description: Cannot decode json parameter list.
|
|
1733
|
+
401:
|
|
1734
|
+
description: Invalid Auth Token
|
|
1735
|
+
404:
|
|
1736
|
+
description: Replica not found
|
|
1737
|
+
409:
|
|
1738
|
+
description: Duplicate
|
|
1739
|
+
"""
|
|
1740
|
+
parameters = json_parameters(parse_response)
|
|
1741
|
+
expires_at = param_get(parameters, 'expires_at', default=None)
|
|
1742
|
+
if expires_at:
|
|
1743
|
+
expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
|
|
1744
|
+
|
|
1745
|
+
try:
|
|
1746
|
+
add_bad_pfns(
|
|
1747
|
+
pfns=param_get(parameters, 'pfns', default=[]),
|
|
1748
|
+
issuer=request.environ.get('issuer'),
|
|
1749
|
+
state=param_get(parameters, 'state', default=None),
|
|
1750
|
+
reason=param_get(parameters, 'reason', default=None),
|
|
1751
|
+
expires_at=expires_at,
|
|
1752
|
+
vo=request.environ.get('vo'),
|
|
1753
|
+
)
|
|
1754
|
+
except (ValueError, InvalidType) as error:
|
|
1755
|
+
return generate_http_error_flask(400, ValueError.__name__, error.args[0])
|
|
1756
|
+
except AccessDenied as error:
|
|
1757
|
+
return generate_http_error_flask(401, error)
|
|
1758
|
+
except ReplicaNotFound as error:
|
|
1759
|
+
return generate_http_error_flask(404, error)
|
|
1760
|
+
except Duplicate as error:
|
|
1761
|
+
return generate_http_error_flask(409, error)
|
|
1762
|
+
|
|
1763
|
+
return 'Created', 201
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
class Tombstone(ErrorHandlingMethodView):
|
|
1767
|
+
|
|
1768
|
+
def post(self):
|
|
1769
|
+
"""
|
|
1770
|
+
---
|
|
1771
|
+
summary: Set Tombstone
|
|
1772
|
+
description: Set a tombstone on a list of replicas.
|
|
1773
|
+
tags:
|
|
1774
|
+
- Replicas
|
|
1775
|
+
requestBody:
|
|
1776
|
+
content:
|
|
1777
|
+
application/json:
|
|
1778
|
+
schema:
|
|
1779
|
+
type: object
|
|
1780
|
+
properties:
|
|
1781
|
+
replicas:
|
|
1782
|
+
description: The replicas to set the tombstone to.
|
|
1783
|
+
type: array
|
|
1784
|
+
items:
|
|
1785
|
+
type: object
|
|
1786
|
+
required:
|
|
1787
|
+
- rse
|
|
1788
|
+
- scope
|
|
1789
|
+
- name
|
|
1790
|
+
properties:
|
|
1791
|
+
rse:
|
|
1792
|
+
description: The rse associated with the tombstone.
|
|
1793
|
+
type: string
|
|
1794
|
+
scope:
|
|
1795
|
+
description: The scope of the replica
|
|
1796
|
+
type: string
|
|
1797
|
+
name:
|
|
1798
|
+
description: The name of the replica.
|
|
1799
|
+
type: string
|
|
1800
|
+
responses:
|
|
1801
|
+
201:
|
|
1802
|
+
description: Created
|
|
1803
|
+
401:
|
|
1804
|
+
description: Invalid Auth Token
|
|
1805
|
+
404:
|
|
1806
|
+
description: Not found
|
|
1807
|
+
423:
|
|
1808
|
+
description: Replica is locked.
|
|
1809
|
+
"""
|
|
1810
|
+
parameters = json_parameters(parse_response)
|
|
1811
|
+
replicas = param_get(parameters, 'replicas', default=[])
|
|
1812
|
+
|
|
1813
|
+
try:
|
|
1814
|
+
for replica in replicas:
|
|
1815
|
+
set_tombstone(
|
|
1816
|
+
rse=replica['rse'],
|
|
1817
|
+
scope=replica['scope'],
|
|
1818
|
+
name=replica['name'],
|
|
1819
|
+
issuer=request.environ.get('issuer'),
|
|
1820
|
+
vo=request.environ.get('vo'),
|
|
1821
|
+
)
|
|
1822
|
+
except ReplicaNotFound as error:
|
|
1823
|
+
return generate_http_error_flask(404, error)
|
|
1824
|
+
except ReplicaIsLocked as error:
|
|
1825
|
+
return generate_http_error_flask(423, error)
|
|
1826
|
+
|
|
1827
|
+
return 'Created', 201
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
def blueprint(with_doc=False):
|
|
1831
|
+
bp = AuthenticatedBlueprint('replicas', __name__, url_prefix='/replicas')
|
|
1832
|
+
|
|
1833
|
+
list_replicas_view = ListReplicas.as_view('list_replicas')
|
|
1834
|
+
bp.add_url_rule('/list', view_func=list_replicas_view, methods=['post', ])
|
|
1835
|
+
replicas_view = Replicas.as_view('replicas')
|
|
1836
|
+
if not with_doc:
|
|
1837
|
+
# rule without trailing slash needs to be added before rule with trailing slash
|
|
1838
|
+
bp.add_url_rule('', view_func=replicas_view, methods=['post', 'put', 'delete'])
|
|
1839
|
+
bp.add_url_rule('/', view_func=replicas_view, methods=['post', 'put', 'delete'])
|
|
1840
|
+
suspicious_replicas_view = SuspiciousReplicas.as_view('suspicious_replicas')
|
|
1841
|
+
bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['get', 'post'])
|
|
1842
|
+
bad_replicas_states_view = BadReplicasStates.as_view('bad_replicas_states')
|
|
1843
|
+
bp.add_url_rule('/bad/states', view_func=bad_replicas_states_view, methods=['get', ])
|
|
1844
|
+
bad_replicas_summary_view = BadReplicasSummary.as_view('bad_replicas_summary')
|
|
1845
|
+
bp.add_url_rule('/bad/summary', view_func=bad_replicas_summary_view, methods=['get', ])
|
|
1846
|
+
bad_replicas_pfn_view = BadPFNs.as_view('add_bad_pfns')
|
|
1847
|
+
bp.add_url_rule('/bad/pfns', view_func=bad_replicas_pfn_view, methods=['post', ])
|
|
1848
|
+
bad_replicas_dids_view = BadDIDs.as_view('add_bad_dids')
|
|
1849
|
+
bp.add_url_rule('/bad/dids', view_func=bad_replicas_dids_view, methods=['post', ])
|
|
1850
|
+
replicas_rse_view = ReplicasRSE.as_view('replicas_rse')
|
|
1851
|
+
bp.add_url_rule('/rse/<rse>', view_func=replicas_rse_view, methods=['get', ])
|
|
1852
|
+
|
|
1853
|
+
bad_replicas_view = BadReplicas.as_view('bad_replicas')
|
|
1854
|
+
bp.add_url_rule('/bad', view_func=bad_replicas_view, methods=['post', ])
|
|
1855
|
+
|
|
1856
|
+
quarantine_replicas_view = QuarantineReplicas.as_view('quarantine_replicas')
|
|
1857
|
+
bp.add_url_rule('/quarantine', view_func=quarantine_replicas_view, methods=['post', ])
|
|
1858
|
+
|
|
1859
|
+
replicas_dids_view = ReplicasDIDs.as_view('replicas_dids')
|
|
1860
|
+
bp.add_url_rule('/dids', view_func=replicas_dids_view, methods=['post', ])
|
|
1861
|
+
dataset_replicas_view = DatasetReplicas.as_view('dataset_replicas')
|
|
1862
|
+
bp.add_url_rule('/<path:scope_name>/datasets', view_func=dataset_replicas_view, methods=['get', ])
|
|
1863
|
+
dataset_replicas_bulk_view = DatasetReplicasBulk.as_view('dataset_replicas_bulk')
|
|
1864
|
+
bp.add_url_rule('/datasets_bulk', view_func=dataset_replicas_bulk_view, methods=['post', ])
|
|
1865
|
+
dataset_replicas_vp_view = DatasetReplicasVP.as_view('dataset_replicas_vp')
|
|
1866
|
+
bp.add_url_rule('/<path:scope_name>', view_func=replicas_view, methods=['get', ])
|
|
1867
|
+
set_tombstone_view = Tombstone.as_view('set_tombstone')
|
|
1868
|
+
bp.add_url_rule('/tombstone', view_func=set_tombstone_view, methods=['post', ])
|
|
1869
|
+
|
|
1870
|
+
if not with_doc:
|
|
1871
|
+
bp.add_url_rule('/list/', view_func=list_replicas_view, methods=['post', ])
|
|
1872
|
+
bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['get', 'post'])
|
|
1873
|
+
bp.add_url_rule('/bad/states/', view_func=bad_replicas_states_view, methods=['get', ])
|
|
1874
|
+
bp.add_url_rule('/bad/summary/', view_func=bad_replicas_summary_view, methods=['get', ])
|
|
1875
|
+
bp.add_url_rule('/bad/pfns/', view_func=bad_replicas_pfn_view, methods=['post', ])
|
|
1876
|
+
bp.add_url_rule('/bad/dids/', view_func=bad_replicas_dids_view, methods=['post', ])
|
|
1877
|
+
bp.add_url_rule('/rse/<rse>/', view_func=replicas_rse_view, methods=['get', ])
|
|
1878
|
+
bp.add_url_rule('/bad/', view_func=bad_replicas_view, methods=['post', ])
|
|
1879
|
+
bp.add_url_rule('/dids/', view_func=replicas_dids_view, methods=['post', ])
|
|
1880
|
+
bp.add_url_rule('/datasets_bulk/', view_func=dataset_replicas_bulk_view, methods=['post', ])
|
|
1881
|
+
bp.add_url_rule('/<path:scope_name>/datasets_vp', view_func=dataset_replicas_vp_view, methods=['get', ])
|
|
1882
|
+
bp.add_url_rule('/<path:scope_name>/', view_func=replicas_view, methods=['get', ])
|
|
1883
|
+
bp.add_url_rule('/tombstone/', view_func=set_tombstone_view, methods=['post', ])
|
|
1884
|
+
bp.add_url_rule('/quarantine/', view_func=quarantine_replicas_view, methods=['post', ])
|
|
1885
|
+
|
|
1886
|
+
bp.after_request(response_headers)
|
|
1887
|
+
return bp
|
|
1888
|
+
|
|
1889
|
+
|
|
1890
|
+
def make_doc():
|
|
1891
|
+
""" Only used for sphinx documentation """
|
|
1892
|
+
doc_app = Flask(__name__)
|
|
1893
|
+
doc_app.register_blueprint(blueprint(with_doc=True))
|
|
1894
|
+
return doc_app
|