rucio 37.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (487) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/cli/__init__.py +14 -0
  4. rucio/cli/account.py +216 -0
  5. rucio/cli/bin_legacy/__init__.py +13 -0
  6. rucio/cli/bin_legacy/rucio.py +2825 -0
  7. rucio/cli/bin_legacy/rucio_admin.py +2500 -0
  8. rucio/cli/command.py +272 -0
  9. rucio/cli/config.py +72 -0
  10. rucio/cli/did.py +191 -0
  11. rucio/cli/download.py +128 -0
  12. rucio/cli/lifetime_exception.py +33 -0
  13. rucio/cli/replica.py +162 -0
  14. rucio/cli/rse.py +293 -0
  15. rucio/cli/rule.py +158 -0
  16. rucio/cli/scope.py +40 -0
  17. rucio/cli/subscription.py +73 -0
  18. rucio/cli/upload.py +60 -0
  19. rucio/cli/utils.py +226 -0
  20. rucio/client/__init__.py +15 -0
  21. rucio/client/accountclient.py +432 -0
  22. rucio/client/accountlimitclient.py +183 -0
  23. rucio/client/baseclient.py +983 -0
  24. rucio/client/client.py +120 -0
  25. rucio/client/configclient.py +126 -0
  26. rucio/client/credentialclient.py +59 -0
  27. rucio/client/didclient.py +868 -0
  28. rucio/client/diracclient.py +56 -0
  29. rucio/client/downloadclient.py +1783 -0
  30. rucio/client/exportclient.py +44 -0
  31. rucio/client/fileclient.py +50 -0
  32. rucio/client/importclient.py +42 -0
  33. rucio/client/lifetimeclient.py +90 -0
  34. rucio/client/lockclient.py +109 -0
  35. rucio/client/metaconventionsclient.py +140 -0
  36. rucio/client/pingclient.py +44 -0
  37. rucio/client/replicaclient.py +452 -0
  38. rucio/client/requestclient.py +125 -0
  39. rucio/client/richclient.py +317 -0
  40. rucio/client/rseclient.py +746 -0
  41. rucio/client/ruleclient.py +294 -0
  42. rucio/client/scopeclient.py +90 -0
  43. rucio/client/subscriptionclient.py +173 -0
  44. rucio/client/touchclient.py +82 -0
  45. rucio/client/uploadclient.py +969 -0
  46. rucio/common/__init__.py +13 -0
  47. rucio/common/bittorrent.py +234 -0
  48. rucio/common/cache.py +111 -0
  49. rucio/common/checksum.py +168 -0
  50. rucio/common/client.py +122 -0
  51. rucio/common/config.py +788 -0
  52. rucio/common/constants.py +217 -0
  53. rucio/common/constraints.py +17 -0
  54. rucio/common/didtype.py +237 -0
  55. rucio/common/dumper/__init__.py +342 -0
  56. rucio/common/dumper/consistency.py +497 -0
  57. rucio/common/dumper/data_models.py +362 -0
  58. rucio/common/dumper/path_parsing.py +75 -0
  59. rucio/common/exception.py +1208 -0
  60. rucio/common/extra.py +31 -0
  61. rucio/common/logging.py +420 -0
  62. rucio/common/pcache.py +1409 -0
  63. rucio/common/plugins.py +185 -0
  64. rucio/common/policy.py +93 -0
  65. rucio/common/schema/__init__.py +200 -0
  66. rucio/common/schema/generic.py +416 -0
  67. rucio/common/schema/generic_multi_vo.py +395 -0
  68. rucio/common/stomp_utils.py +423 -0
  69. rucio/common/stopwatch.py +55 -0
  70. rucio/common/test_rucio_server.py +154 -0
  71. rucio/common/types.py +483 -0
  72. rucio/common/utils.py +1688 -0
  73. rucio/core/__init__.py +13 -0
  74. rucio/core/account.py +496 -0
  75. rucio/core/account_counter.py +236 -0
  76. rucio/core/account_limit.py +425 -0
  77. rucio/core/authentication.py +620 -0
  78. rucio/core/config.py +437 -0
  79. rucio/core/credential.py +224 -0
  80. rucio/core/did.py +3004 -0
  81. rucio/core/did_meta_plugins/__init__.py +252 -0
  82. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  83. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  84. rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
  85. rucio/core/did_meta_plugins/filter_engine.py +672 -0
  86. rucio/core/did_meta_plugins/json_meta.py +240 -0
  87. rucio/core/did_meta_plugins/mongo_meta.py +229 -0
  88. rucio/core/did_meta_plugins/postgres_meta.py +352 -0
  89. rucio/core/dirac.py +237 -0
  90. rucio/core/distance.py +187 -0
  91. rucio/core/exporter.py +59 -0
  92. rucio/core/heartbeat.py +363 -0
  93. rucio/core/identity.py +301 -0
  94. rucio/core/importer.py +260 -0
  95. rucio/core/lifetime_exception.py +377 -0
  96. rucio/core/lock.py +577 -0
  97. rucio/core/message.py +288 -0
  98. rucio/core/meta_conventions.py +203 -0
  99. rucio/core/monitor.py +448 -0
  100. rucio/core/naming_convention.py +195 -0
  101. rucio/core/nongrid_trace.py +136 -0
  102. rucio/core/oidc.py +1463 -0
  103. rucio/core/permission/__init__.py +161 -0
  104. rucio/core/permission/generic.py +1124 -0
  105. rucio/core/permission/generic_multi_vo.py +1144 -0
  106. rucio/core/quarantined_replica.py +224 -0
  107. rucio/core/replica.py +4483 -0
  108. rucio/core/replica_sorter.py +362 -0
  109. rucio/core/request.py +3091 -0
  110. rucio/core/rse.py +2079 -0
  111. rucio/core/rse_counter.py +185 -0
  112. rucio/core/rse_expression_parser.py +459 -0
  113. rucio/core/rse_selector.py +304 -0
  114. rucio/core/rule.py +4484 -0
  115. rucio/core/rule_grouping.py +1620 -0
  116. rucio/core/scope.py +181 -0
  117. rucio/core/subscription.py +362 -0
  118. rucio/core/topology.py +490 -0
  119. rucio/core/trace.py +375 -0
  120. rucio/core/transfer.py +1531 -0
  121. rucio/core/vo.py +169 -0
  122. rucio/core/volatile_replica.py +151 -0
  123. rucio/daemons/__init__.py +13 -0
  124. rucio/daemons/abacus/__init__.py +13 -0
  125. rucio/daemons/abacus/account.py +116 -0
  126. rucio/daemons/abacus/collection_replica.py +124 -0
  127. rucio/daemons/abacus/rse.py +117 -0
  128. rucio/daemons/atropos/__init__.py +13 -0
  129. rucio/daemons/atropos/atropos.py +242 -0
  130. rucio/daemons/auditor/__init__.py +289 -0
  131. rucio/daemons/auditor/hdfs.py +97 -0
  132. rucio/daemons/auditor/srmdumps.py +355 -0
  133. rucio/daemons/automatix/__init__.py +13 -0
  134. rucio/daemons/automatix/automatix.py +304 -0
  135. rucio/daemons/badreplicas/__init__.py +13 -0
  136. rucio/daemons/badreplicas/minos.py +322 -0
  137. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  138. rucio/daemons/badreplicas/necromancer.py +196 -0
  139. rucio/daemons/bb8/__init__.py +13 -0
  140. rucio/daemons/bb8/bb8.py +353 -0
  141. rucio/daemons/bb8/common.py +759 -0
  142. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  143. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  144. rucio/daemons/cache/__init__.py +13 -0
  145. rucio/daemons/cache/consumer.py +133 -0
  146. rucio/daemons/common.py +405 -0
  147. rucio/daemons/conveyor/__init__.py +13 -0
  148. rucio/daemons/conveyor/common.py +562 -0
  149. rucio/daemons/conveyor/finisher.py +529 -0
  150. rucio/daemons/conveyor/poller.py +394 -0
  151. rucio/daemons/conveyor/preparer.py +205 -0
  152. rucio/daemons/conveyor/receiver.py +179 -0
  153. rucio/daemons/conveyor/stager.py +133 -0
  154. rucio/daemons/conveyor/submitter.py +403 -0
  155. rucio/daemons/conveyor/throttler.py +532 -0
  156. rucio/daemons/follower/__init__.py +13 -0
  157. rucio/daemons/follower/follower.py +101 -0
  158. rucio/daemons/hermes/__init__.py +13 -0
  159. rucio/daemons/hermes/hermes.py +534 -0
  160. rucio/daemons/judge/__init__.py +13 -0
  161. rucio/daemons/judge/cleaner.py +159 -0
  162. rucio/daemons/judge/evaluator.py +185 -0
  163. rucio/daemons/judge/injector.py +162 -0
  164. rucio/daemons/judge/repairer.py +154 -0
  165. rucio/daemons/oauthmanager/__init__.py +13 -0
  166. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  167. rucio/daemons/reaper/__init__.py +13 -0
  168. rucio/daemons/reaper/dark_reaper.py +282 -0
  169. rucio/daemons/reaper/reaper.py +739 -0
  170. rucio/daemons/replicarecoverer/__init__.py +13 -0
  171. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  172. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  173. rucio/daemons/rsedecommissioner/config.py +81 -0
  174. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  175. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  176. rucio/daemons/rsedecommissioner/profiles/generic.py +452 -0
  177. rucio/daemons/rsedecommissioner/profiles/types.py +93 -0
  178. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  179. rucio/daemons/storage/__init__.py +13 -0
  180. rucio/daemons/storage/consistency/__init__.py +13 -0
  181. rucio/daemons/storage/consistency/actions.py +848 -0
  182. rucio/daemons/tracer/__init__.py +13 -0
  183. rucio/daemons/tracer/kronos.py +511 -0
  184. rucio/daemons/transmogrifier/__init__.py +13 -0
  185. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  186. rucio/daemons/undertaker/__init__.py +13 -0
  187. rucio/daemons/undertaker/undertaker.py +137 -0
  188. rucio/db/__init__.py +13 -0
  189. rucio/db/sqla/__init__.py +52 -0
  190. rucio/db/sqla/constants.py +206 -0
  191. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  192. rucio/db/sqla/migrate_repo/env.py +110 -0
  193. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  194. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  195. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  196. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  197. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  198. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  199. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  200. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  201. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  202. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  203. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  204. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  205. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  206. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  207. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  208. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  209. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  210. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  211. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  212. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  213. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  214. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  215. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  216. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  217. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  218. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  219. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  220. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  221. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  222. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  223. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  224. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  225. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  226. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  227. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  228. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  229. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  230. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  231. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  232. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  233. rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
  234. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  235. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  236. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  237. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  238. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  239. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  240. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  241. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  242. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  243. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  244. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  245. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  246. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  247. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  248. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  249. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  251. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  252. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  253. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  254. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  255. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  256. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  257. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  258. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  259. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  260. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  261. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  262. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  263. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  264. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  265. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  266. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  267. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  268. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  269. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  270. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  271. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  272. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  273. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  274. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  275. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  276. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  278. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  279. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  280. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  281. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  282. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  283. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  284. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  285. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  286. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  287. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  288. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  289. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  290. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  291. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  292. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  293. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  294. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  295. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  296. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  297. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  298. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  299. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  300. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  301. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  302. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  303. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  304. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  305. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  306. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  307. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  308. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  309. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  310. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  311. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  312. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  313. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  314. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  316. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  317. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  318. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  319. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  320. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  321. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  322. rucio/db/sqla/models.py +1743 -0
  323. rucio/db/sqla/sautils.py +55 -0
  324. rucio/db/sqla/session.py +529 -0
  325. rucio/db/sqla/types.py +206 -0
  326. rucio/db/sqla/util.py +543 -0
  327. rucio/gateway/__init__.py +13 -0
  328. rucio/gateway/account.py +345 -0
  329. rucio/gateway/account_limit.py +363 -0
  330. rucio/gateway/authentication.py +381 -0
  331. rucio/gateway/config.py +227 -0
  332. rucio/gateway/credential.py +70 -0
  333. rucio/gateway/did.py +987 -0
  334. rucio/gateway/dirac.py +83 -0
  335. rucio/gateway/exporter.py +60 -0
  336. rucio/gateway/heartbeat.py +76 -0
  337. rucio/gateway/identity.py +189 -0
  338. rucio/gateway/importer.py +46 -0
  339. rucio/gateway/lifetime_exception.py +121 -0
  340. rucio/gateway/lock.py +153 -0
  341. rucio/gateway/meta_conventions.py +98 -0
  342. rucio/gateway/permission.py +74 -0
  343. rucio/gateway/quarantined_replica.py +79 -0
  344. rucio/gateway/replica.py +538 -0
  345. rucio/gateway/request.py +330 -0
  346. rucio/gateway/rse.py +632 -0
  347. rucio/gateway/rule.py +437 -0
  348. rucio/gateway/scope.py +100 -0
  349. rucio/gateway/subscription.py +280 -0
  350. rucio/gateway/vo.py +126 -0
  351. rucio/rse/__init__.py +96 -0
  352. rucio/rse/protocols/__init__.py +13 -0
  353. rucio/rse/protocols/bittorrent.py +194 -0
  354. rucio/rse/protocols/cache.py +111 -0
  355. rucio/rse/protocols/dummy.py +100 -0
  356. rucio/rse/protocols/gfal.py +708 -0
  357. rucio/rse/protocols/globus.py +243 -0
  358. rucio/rse/protocols/http_cache.py +82 -0
  359. rucio/rse/protocols/mock.py +123 -0
  360. rucio/rse/protocols/ngarc.py +209 -0
  361. rucio/rse/protocols/posix.py +250 -0
  362. rucio/rse/protocols/protocol.py +361 -0
  363. rucio/rse/protocols/rclone.py +365 -0
  364. rucio/rse/protocols/rfio.py +145 -0
  365. rucio/rse/protocols/srm.py +338 -0
  366. rucio/rse/protocols/ssh.py +414 -0
  367. rucio/rse/protocols/storm.py +195 -0
  368. rucio/rse/protocols/webdav.py +594 -0
  369. rucio/rse/protocols/xrootd.py +302 -0
  370. rucio/rse/rsemanager.py +881 -0
  371. rucio/rse/translation.py +260 -0
  372. rucio/tests/__init__.py +13 -0
  373. rucio/tests/common.py +280 -0
  374. rucio/tests/common_server.py +149 -0
  375. rucio/transfertool/__init__.py +13 -0
  376. rucio/transfertool/bittorrent.py +200 -0
  377. rucio/transfertool/bittorrent_driver.py +50 -0
  378. rucio/transfertool/bittorrent_driver_qbittorrent.py +134 -0
  379. rucio/transfertool/fts3.py +1600 -0
  380. rucio/transfertool/fts3_plugins.py +152 -0
  381. rucio/transfertool/globus.py +201 -0
  382. rucio/transfertool/globus_library.py +181 -0
  383. rucio/transfertool/mock.py +89 -0
  384. rucio/transfertool/transfertool.py +221 -0
  385. rucio/vcsversion.py +11 -0
  386. rucio/version.py +45 -0
  387. rucio/web/__init__.py +13 -0
  388. rucio/web/rest/__init__.py +13 -0
  389. rucio/web/rest/flaskapi/__init__.py +13 -0
  390. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  391. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  392. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  393. rucio/web/rest/flaskapi/v1/accounts.py +1103 -0
  394. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  395. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  396. rucio/web/rest/flaskapi/v1/common.py +426 -0
  397. rucio/web/rest/flaskapi/v1/config.py +304 -0
  398. rucio/web/rest/flaskapi/v1/credentials.py +213 -0
  399. rucio/web/rest/flaskapi/v1/dids.py +2340 -0
  400. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  401. rucio/web/rest/flaskapi/v1/export.py +75 -0
  402. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  403. rucio/web/rest/flaskapi/v1/identities.py +285 -0
  404. rucio/web/rest/flaskapi/v1/import.py +132 -0
  405. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  406. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  407. rucio/web/rest/flaskapi/v1/main.py +91 -0
  408. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  409. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  410. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  411. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  412. rucio/web/rest/flaskapi/v1/redirect.py +366 -0
  413. rucio/web/rest/flaskapi/v1/replicas.py +1894 -0
  414. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  415. rucio/web/rest/flaskapi/v1/rses.py +2250 -0
  416. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  417. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  418. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  419. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  420. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  421. rucio/web/rest/flaskapi/v1/traces.py +137 -0
  422. rucio/web/rest/flaskapi/v1/types.py +20 -0
  423. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  424. rucio/web/rest/main.py +18 -0
  425. rucio/web/rest/metrics.py +27 -0
  426. rucio/web/rest/ping.py +27 -0
  427. rucio-37.0.0rc1.data/data/rucio/etc/alembic.ini.template +71 -0
  428. rucio-37.0.0rc1.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  429. rucio-37.0.0rc1.data/data/rucio/etc/globus-config.yml.template +5 -0
  430. rucio-37.0.0rc1.data/data/rucio/etc/ldap.cfg.template +30 -0
  431. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  432. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  433. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  434. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  435. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  436. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  437. rucio-37.0.0rc1.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  438. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.atlas.client.template +43 -0
  439. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.template +241 -0
  440. rucio-37.0.0rc1.data/data/rucio/etc/rucio_multi_vo.cfg.template +217 -0
  441. rucio-37.0.0rc1.data/data/rucio/requirements.server.txt +297 -0
  442. rucio-37.0.0rc1.data/data/rucio/tools/bootstrap.py +34 -0
  443. rucio-37.0.0rc1.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  444. rucio-37.0.0rc1.data/data/rucio/tools/reset_database.py +40 -0
  445. rucio-37.0.0rc1.data/scripts/rucio +133 -0
  446. rucio-37.0.0rc1.data/scripts/rucio-abacus-account +74 -0
  447. rucio-37.0.0rc1.data/scripts/rucio-abacus-collection-replica +46 -0
  448. rucio-37.0.0rc1.data/scripts/rucio-abacus-rse +78 -0
  449. rucio-37.0.0rc1.data/scripts/rucio-admin +97 -0
  450. rucio-37.0.0rc1.data/scripts/rucio-atropos +60 -0
  451. rucio-37.0.0rc1.data/scripts/rucio-auditor +206 -0
  452. rucio-37.0.0rc1.data/scripts/rucio-automatix +50 -0
  453. rucio-37.0.0rc1.data/scripts/rucio-bb8 +57 -0
  454. rucio-37.0.0rc1.data/scripts/rucio-cache-client +141 -0
  455. rucio-37.0.0rc1.data/scripts/rucio-cache-consumer +42 -0
  456. rucio-37.0.0rc1.data/scripts/rucio-conveyor-finisher +58 -0
  457. rucio-37.0.0rc1.data/scripts/rucio-conveyor-poller +66 -0
  458. rucio-37.0.0rc1.data/scripts/rucio-conveyor-preparer +37 -0
  459. rucio-37.0.0rc1.data/scripts/rucio-conveyor-receiver +44 -0
  460. rucio-37.0.0rc1.data/scripts/rucio-conveyor-stager +76 -0
  461. rucio-37.0.0rc1.data/scripts/rucio-conveyor-submitter +139 -0
  462. rucio-37.0.0rc1.data/scripts/rucio-conveyor-throttler +104 -0
  463. rucio-37.0.0rc1.data/scripts/rucio-dark-reaper +53 -0
  464. rucio-37.0.0rc1.data/scripts/rucio-dumper +160 -0
  465. rucio-37.0.0rc1.data/scripts/rucio-follower +44 -0
  466. rucio-37.0.0rc1.data/scripts/rucio-hermes +54 -0
  467. rucio-37.0.0rc1.data/scripts/rucio-judge-cleaner +89 -0
  468. rucio-37.0.0rc1.data/scripts/rucio-judge-evaluator +137 -0
  469. rucio-37.0.0rc1.data/scripts/rucio-judge-injector +44 -0
  470. rucio-37.0.0rc1.data/scripts/rucio-judge-repairer +44 -0
  471. rucio-37.0.0rc1.data/scripts/rucio-kronos +44 -0
  472. rucio-37.0.0rc1.data/scripts/rucio-minos +53 -0
  473. rucio-37.0.0rc1.data/scripts/rucio-minos-temporary-expiration +50 -0
  474. rucio-37.0.0rc1.data/scripts/rucio-necromancer +120 -0
  475. rucio-37.0.0rc1.data/scripts/rucio-oauth-manager +63 -0
  476. rucio-37.0.0rc1.data/scripts/rucio-reaper +83 -0
  477. rucio-37.0.0rc1.data/scripts/rucio-replica-recoverer +248 -0
  478. rucio-37.0.0rc1.data/scripts/rucio-rse-decommissioner +66 -0
  479. rucio-37.0.0rc1.data/scripts/rucio-storage-consistency-actions +74 -0
  480. rucio-37.0.0rc1.data/scripts/rucio-transmogrifier +77 -0
  481. rucio-37.0.0rc1.data/scripts/rucio-undertaker +76 -0
  482. rucio-37.0.0rc1.dist-info/METADATA +92 -0
  483. rucio-37.0.0rc1.dist-info/RECORD +487 -0
  484. rucio-37.0.0rc1.dist-info/WHEEL +5 -0
  485. rucio-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
  486. rucio-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
  487. rucio-37.0.0rc1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1894 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from datetime import datetime
16
+ from itertools import chain
17
+ from json import dumps, loads
18
+ from typing import TYPE_CHECKING
19
+ from urllib.parse import parse_qs, unquote
20
+ from xml.sax.saxutils import escape
21
+
22
+ from flask import Flask, Response, request
23
+
24
+ from rucio.common.config import config_get, config_get_int
25
+ from rucio.common.constants import SUPPORTED_PROTOCOLS
26
+ from rucio.common.exception import (
27
+ AccessDenied,
28
+ DataIdentifierAlreadyExists,
29
+ DataIdentifierNotFound,
30
+ Duplicate,
31
+ InvalidObject,
32
+ InvalidPath,
33
+ InvalidType,
34
+ ReplicaIsLocked,
35
+ ReplicaNotFound,
36
+ ResourceTemporaryUnavailable,
37
+ RSENotFound,
38
+ ScopeNotFound,
39
+ SortingAlgorithmNotSupported,
40
+ )
41
+ from rucio.common.utils import APIEncoder, parse_response, render_json
42
+ from rucio.core.replica_sorter import sort_replicas
43
+ from rucio.db.sqla.constants import BadFilesStatus
44
+ from rucio.gateway.quarantined_replica import quarantine_file_replicas
45
+ from rucio.gateway.replica import (
46
+ add_bad_dids,
47
+ add_bad_pfns,
48
+ add_replicas,
49
+ declare_bad_file_replicas,
50
+ declare_suspicious_file_replicas,
51
+ delete_replicas,
52
+ get_bad_replicas_summary,
53
+ get_did_from_pfns,
54
+ get_suspicious_files,
55
+ list_bad_replicas_status,
56
+ list_dataset_replicas,
57
+ list_dataset_replicas_bulk,
58
+ list_dataset_replicas_vp,
59
+ list_datasets_per_rse,
60
+ list_replicas,
61
+ set_tombstone,
62
+ update_replicas_states,
63
+ )
64
+ from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
65
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
66
+
67
+ if TYPE_CHECKING:
68
+ from rucio.common.types import IPDict
69
+
70
+
71
+ def _sorted_with_priorities(replicas, sorted_pfns, limit=None):
72
+ """
73
+ Pick up to "limit" replicas from "replicas" in the order given by sorted_pfns.
74
+ Sets the corresponding priority in returned replicas.
75
+
76
+ :param replicas: Dictionary {pfn: replica_definition}.
77
+ :param sorted_pfns: Sorted list of pfns.
78
+ :param limit: only return this many replicas
79
+ :yields: index and corresponding pfn
80
+ """
81
+ for idx, pfn in enumerate(sorted_pfns, start=1):
82
+ if limit is None or idx <= limit:
83
+ replica = replicas[pfn]
84
+ replica['priority'] = idx
85
+ yield pfn, replica
86
+
87
+
88
+ def _generate_one_metalink_file(rfile, policy_schema, detailed_url=True):
89
+ yield ' <file name="' + rfile['name'] + '">\n'
90
+
91
+ if 'parents' in rfile and rfile['parents']:
92
+ yield ' <parents>\n'
93
+ for parent in rfile['parents']:
94
+ yield ' <did>' + parent + '</did>\n'
95
+ yield ' </parents>\n'
96
+
97
+ yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
98
+ if rfile['adler32'] is not None:
99
+ yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
100
+ if rfile['md5'] is not None:
101
+ yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
102
+
103
+ yield ' <size>' + str(rfile['bytes']) + '</size>\n'
104
+
105
+ yield f' <glfn name="/{policy_schema}/rucio/{rfile["scope"]}:{rfile["name"]}"></glfn>\n'
106
+
107
+ for pfn, replica in rfile['pfns'].items():
108
+ if detailed_url:
109
+ yield (
110
+ ' '
111
+ f'<url location="{replica["rse"]}"'
112
+ f' domain="{replica["domain"]}"'
113
+ f' priority="{replica["priority"]}"'
114
+ f' client_extract="{str(replica["client_extract"]).lower()}"'
115
+ f'>{escape(pfn)}</url>\n'
116
+ )
117
+ else:
118
+ yield (
119
+ ' '
120
+ f'<url location="{replica["rse"]}"'
121
+ f' priority="{replica["priority"]}"'
122
+ f'>{escape(pfn)}</url>\n'
123
+ )
124
+ yield ' </file>\n'
125
+
126
+
127
+ def _generate_metalink_response(rfiles, policy_schema, detailed_url=True):
128
+ first = True
129
+ for rfile in rfiles:
130
+ if first:
131
+ # first, set the appropriate content type, and stream the header
132
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
133
+ first = False
134
+
135
+ yield from _generate_one_metalink_file(rfile, policy_schema=policy_schema, detailed_url=detailed_url)
136
+
137
+ if first:
138
+ # if still first output, i.e. there were no replicas
139
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
140
+ else:
141
+ # don't forget to send the metalink footer
142
+ yield '</metalink>\n'
143
+
144
+
145
+ def _generate_json_response(rfiles):
146
+ for rfile in rfiles:
147
+ yield dumps(rfile) + '\n'
148
+
149
+
150
+ class Replicas(ErrorHandlingMethodView):
151
+
152
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
153
+ def get(self, scope_name):
154
+ """
155
+ ---
156
+ summary: Get Replicas
157
+ description: List all replicas for data identifiers.
158
+ tags:
159
+ - Replicas
160
+ parameters:
161
+ - name: scope_name
162
+ in: path
163
+ description: The DID associated with the replicas.
164
+ schema:
165
+ type: string
166
+ style: simple
167
+ - name: X-Forwarded-For
168
+ in: header
169
+ description: The client ip
170
+ schema:
171
+ type: string
172
+ - name: schemes
173
+ in: query
174
+ description: The schemes of the replicas.
175
+ schema:
176
+ type: string
177
+ - name: select
178
+ in: query
179
+ description: The sorting algorithm.
180
+ schema:
181
+ type: string
182
+ enum: ["geoip", "random"]
183
+ - name: limit
184
+ in: query
185
+ description: The maximum number of replicas returned.
186
+ schema:
187
+ type: integer
188
+ responses:
189
+ 200:
190
+ description: OK
191
+ content:
192
+ application/x-json-stream:
193
+ schema:
194
+ description: A list with all replicas.
195
+ type: array
196
+ items:
197
+ description: A replica. Possibly contains more information.
198
+ type: object
199
+ properties:
200
+ scope:
201
+ description: The scope of the replica.
202
+ type: string
203
+ name:
204
+ description: The name of the replica.
205
+ type: string
206
+ bytes:
207
+ description: The size of the replica in bytes.
208
+ type: integer
209
+ md5:
210
+ description: The md5 checksum of the replica.
211
+ type: string
212
+ adler32:
213
+ description: The adler32 checksum of the replica.
214
+ type: string
215
+ pfns:
216
+ description: The pfns associated with the replica.
217
+ type: array
218
+ rses:
219
+ description: The rse associated with the replica.
220
+ type: string
221
+ 401:
222
+ description: Invalid Auth Token
223
+ 404:
224
+ description: Did not found
225
+ 406:
226
+ description: Not acceptable
227
+ """
228
+ try:
229
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
230
+ except ValueError as error:
231
+ return generate_http_error_flask(400, error)
232
+
233
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
234
+ metalink = (content_type == 'application/metalink4+xml')
235
+ dids = [{'scope': scope, 'name': name}]
236
+ select, limit = None, None
237
+
238
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
239
+ client_location: 'IPDict' = {'ip': client_ip, 'fqdn': None, 'site': None}
240
+
241
+ schemes = request.args.get('schemes', default=None)
242
+ select = request.args.get('select', default=None)
243
+ limit = request.args.get('limit', default=None)
244
+ if limit:
245
+ limit = int(limit)
246
+
247
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
248
+ if metalink and schemes is None:
249
+ schemes = SUPPORTED_PROTOCOLS
250
+
251
+ try:
252
+ def _list_and_sort_replicas(vo):
253
+ # we need to call list_replicas before starting to reply
254
+ # otherwise the exceptions won't be propagated correctly
255
+ for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
256
+ replicas = []
257
+ dictreplica = {}
258
+ for rse in rfile['rses']:
259
+ for replica in rfile['rses'][rse]:
260
+ replicas.append(replica)
261
+ dictreplica[replica] = rse
262
+
263
+ replicas = sort_replicas(dictreplica, client_location, selection=select)
264
+ rfile['pfns'] = dict(_sorted_with_priorities(rfile['pfns'], replicas, limit=limit))
265
+ yield rfile
266
+
267
+ rfiles = _list_and_sort_replicas(vo=request.environ.get('vo'))
268
+ if metalink:
269
+ response_generator = _generate_metalink_response(rfiles, 'atlas', detailed_url=False)
270
+ else:
271
+ response_generator = _generate_json_response(rfiles)
272
+ return try_stream(response_generator, content_type=content_type)
273
+ except (DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
274
+ return generate_http_error_flask(404, error)
275
+
276
+ def post(self):
277
+ """
278
+ ---
279
+ summary: Create File Replicas
280
+ description: Create file replicas at a given RSE.
281
+ tags:
282
+ - Replicas
283
+ requestBody:
284
+ content:
285
+ application/json:
286
+ schema:
287
+ type: object
288
+ required:
289
+ - rse
290
+ - files
291
+ properties:
292
+ rse:
293
+ description: The rse for the replication
294
+ type: string
295
+ files:
296
+ description: The files to replicate
297
+ type: array
298
+ items:
299
+ type: object
300
+ required:
301
+ - pfn
302
+ - bytes
303
+ - name
304
+ properties:
305
+ pfn:
306
+ description: The pfn of the replica.
307
+ type: string
308
+ name:
309
+ description: The DID name.
310
+ type: string
311
+ bytes:
312
+ description: The size of the replica in bytes.
313
+ type: integer
314
+ state:
315
+ description: The state of the replica.
316
+ type: string
317
+ path:
318
+ description: The path of the new replica.
319
+ type: string
320
+ md5:
321
+ description: The md5 checksum.
322
+ type: string
323
+ adler32:
324
+ description: The adler32 checksum.
325
+ type: string
326
+ lcok_cnt:
327
+ description: The lock count.
328
+ type: integer
329
+ tombstone:
330
+ description: The tombstone.
331
+ type: string
332
+ ignore_availability:
333
+ description: The ignore availability.
334
+ type: boolean
335
+ responses:
336
+ 201:
337
+ description: OK
338
+ content:
339
+ application/json:
340
+ schema:
341
+ type: string
342
+ enum: ["Created"]
343
+ 400:
344
+ description: Invalid Path
345
+ 401:
346
+ description: Invalid Auth Token
347
+ 404:
348
+ description: Rse or scope not found
349
+ 409:
350
+ description: Replica or Did already exists
351
+ 503:
352
+ description: Resource temporary unavailable
353
+ """
354
+ parameters = json_parameters(parse_response)
355
+ rse = param_get(parameters, 'rse')
356
+ files = param_get(parameters, 'files')
357
+
358
+ try:
359
+ add_replicas(
360
+ rse=rse,
361
+ files=files,
362
+ issuer=request.environ.get('issuer'),
363
+ vo=request.environ.get('vo'),
364
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
365
+ )
366
+ except InvalidPath as error:
367
+ return generate_http_error_flask(400, error)
368
+ except AccessDenied as error:
369
+ return generate_http_error_flask(401, error)
370
+ except (Duplicate, DataIdentifierAlreadyExists) as error:
371
+ return generate_http_error_flask(409, error)
372
+ except (RSENotFound, ScopeNotFound) as error:
373
+ return generate_http_error_flask(404, error)
374
+ except ResourceTemporaryUnavailable as error:
375
+ return generate_http_error_flask(503, error)
376
+
377
+ return 'Created', 201
378
+
379
+ def put(self):
380
+ """
381
+ ---
382
+ summary: Update File Replicas
383
+ description: Update file replicas state at a given RSE.
384
+ tags:
385
+ - Replicas
386
+ requestBody:
387
+ content:
388
+ application/json:
389
+ schema:
390
+ type: object
391
+ required:
392
+ - rse
393
+ - files
394
+ properties:
395
+ rse:
396
+ description: The rse for the replication
397
+ type: string
398
+ files:
399
+ description: The files to replicate
400
+ type: array
401
+ items:
402
+ type: object
403
+ properties:
404
+ name:
405
+ description: The pfn of the replica.
406
+ type: string
407
+ state:
408
+ description: The pfn of the replica.
409
+ type: string
410
+ path:
411
+ description: The pfn of the replica.
412
+ type: string
413
+ error_message:
414
+ description: The error message if an error occurred.
415
+ type: string
416
+ broken_rule_id:
417
+ description: The id of the broken rule if one was found.
418
+ type: string
419
+ broken_message:
420
+ description: The message of the broken rule.
421
+ type: string
422
+ responses:
423
+ 200:
424
+ description: OK
425
+ 400:
426
+ description: Cannot decode json parameter list
427
+ 401:
428
+ description: Invalid Auth Token
429
+ """
430
+ parameters = json_parameters(parse_response)
431
+ rse = param_get(parameters, 'rse')
432
+ files = param_get(parameters, 'files')
433
+
434
+ try:
435
+ update_replicas_states(rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
436
+ except AccessDenied as error:
437
+ return generate_http_error_flask(401, error)
438
+
439
+ return '', 200
440
+
441
+ def delete(self):
442
+ """
443
+ ---
444
+ summary: Delete File Replicas
445
+ description: Delete file replicas at a given RSE.
446
+ tags:
447
+ - Replicas
448
+ requestBody:
449
+ content:
450
+ application/json:
451
+ schema:
452
+ type: object
453
+ required:
454
+ - rse
455
+ - files
456
+ properties:
457
+ rse:
458
+ description: The rse name.
459
+ type: string
460
+ files:
461
+ description: The files to delete.
462
+ type: array
463
+ items:
464
+ type: object
465
+ required:
466
+ - name
467
+ properties:
468
+ name:
469
+ description: The name of the replica.
470
+ type: string
471
+ responses:
472
+ 200:
473
+ description: OK
474
+ 400:
475
+ description: Cannot decode json parameter list.
476
+ 401:
477
+ description: Invalid Auth Token
478
+ 404:
479
+ description: Rse or Replica not found
480
+ """
481
+ parameters = json_parameters(parse_response)
482
+ rse = param_get(parameters, 'rse')
483
+ files = param_get(parameters, 'files')
484
+
485
+ try:
486
+ delete_replicas(
487
+ rse=rse,
488
+ files=files,
489
+ issuer=request.environ.get('issuer'),
490
+ vo=request.environ.get('vo'),
491
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
492
+ )
493
+ except AccessDenied as error:
494
+ return generate_http_error_flask(401, error)
495
+ except (RSENotFound, ReplicaNotFound) as error:
496
+ return generate_http_error_flask(404, error)
497
+ except ResourceTemporaryUnavailable as error:
498
+ return generate_http_error_flask(503, error)
499
+
500
+ return '', 200
501
+
502
+
503
+ class ListReplicas(ErrorHandlingMethodView):
504
+
505
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
506
+ def post(self):
507
+ """
508
+ ---
509
+ summary: List Replicas
510
+ description: List all replicas for a DID.
511
+ tags:
512
+ - Replicas
513
+ parameters:
514
+ - name: X-Forwarded-For
515
+ in: header
516
+ description: The client ip address.
517
+ schema:
518
+ type: string
519
+ - name: limit
520
+ in: query
521
+ description: The maximum number pfns per replica to return.
522
+ schema:
523
+ type: integer
524
+ - name: select
525
+ in: query
526
+ description: Requested sorting of the result, e.g., 'geoip', 'random'.
527
+ schema:
528
+ type: string
529
+ - name: sort
530
+ in: query
531
+ description: Requested sorting of the result, e.g., 'geoip', 'random'.
532
+ schema:
533
+ type: string
534
+ requestBody:
535
+ content:
536
+ application/json:
537
+ schema:
538
+ type: object
539
+ properties:
540
+ client_location:
541
+ description: The clients location.
542
+ type: string
543
+ dids:
544
+ description: List of Dids.
545
+ type: array
546
+ items:
547
+ type: object
548
+ properties:
549
+ scope:
550
+ description: The scope of the did.
551
+ type: string
552
+ name:
553
+ description: The name of the did.
554
+ type: string
555
+ schemes:
556
+ description: A list of schemes to filter the replicas.
557
+ type: array
558
+ items:
559
+ type: string
560
+ sort:
561
+ description: Requested sorting of the result, e.g., 'geoip', 'random'.
562
+ type: string
563
+ unavailable:
564
+ description: If unavailable rse should be considered.
565
+ type: boolean
566
+ deprecated: true
567
+ ignore_availability:
568
+ description: If the availability should be ignored.
569
+ type: boolean
570
+ rse_expression:
571
+ description: The RSE expression to restrict on a list of RSEs.
572
+ type: string
573
+ all_states:
574
+ description: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
575
+ type: boolean
576
+ domain:
577
+ description: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
578
+ type: string
579
+ signature_lifetime:
580
+ description: If supported, in seconds, restrict the lifetime of the signed PFN.
581
+ type: integer
582
+ resolve_archives:
583
+ description: When set to True, find archives which contain the replicas.
584
+ type: boolean
585
+ resolve_parents:
586
+ description: When set to True, find all parent datasets which contain the replicas.
587
+ type: boolean
588
+ updated_after:
589
+ description: datetime object (UTC time), only return replicas updated after this time
590
+ type: string
591
+ nrandom:
592
+ description: The maximum number of replicas to return.
593
+ type: integer
594
+ responses:
595
+ 200:
596
+ description: OK
597
+ content:
598
+ application/json:
599
+ schema:
600
+ type: array
601
+ items:
602
+ type: object
603
+ properties:
604
+ scope:
605
+ description: The scope of the replica.
606
+ type: string
607
+ name:
608
+ description: The name of the replica.
609
+ type: string
610
+ bytes:
611
+ description: The size of the replica in bytes.
612
+ type: integer
613
+ md5:
614
+ description: The md5 checksum.
615
+ type: string
616
+ adler32:
617
+ description: The adler32 checksum.
618
+ type: string
619
+ pfns:
620
+ description: The pfns.
621
+ type: array
622
+ rses:
623
+ description: The RSESs.
624
+ type: array
625
+ application/metalink4+xml:
626
+ schema:
627
+ type: object
628
+ properties:
629
+ scope:
630
+ description: The scope of the replica.
631
+ type: string
632
+ name:
633
+ description: The name of the replica.
634
+ type: string
635
+ bytes:
636
+ description: The size of the replica in bytes.
637
+ type: integer
638
+ md5:
639
+ description: The md5 checksum.
640
+ type: string
641
+ adler32:
642
+ description: The adler32 checksum.
643
+ type: string
644
+ pfns:
645
+ description: The pfns.
646
+ type: array
647
+ rses:
648
+ description: The RSESs.
649
+ type: array
650
+ 400:
651
+ description: Cannot decode json parameter list.
652
+ 401:
653
+ description: Invalid Auth Token
654
+ 404:
655
+ description: Did not found.
656
+ 406:
657
+ description: Not acceptable
658
+ """
659
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
660
+ metalink = (content_type == 'application/metalink4+xml')
661
+
662
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
663
+
664
+ parameters = json_parameters(parse_response)
665
+
666
+ client_location: 'IPDict' = {'ip': client_ip,
667
+ 'fqdn': None,
668
+ 'site': None}
669
+ client_location.update(param_get(parameters, 'client_location', default={}))
670
+
671
+ # making sure IP address is not overwritten
672
+ client_location['ip'] = client_ip
673
+
674
+ dids = param_get(parameters, 'dids', default=[])
675
+ schemes = param_get(parameters, 'schemes', default=None)
676
+ select = param_get(parameters, 'sort', default=None)
677
+ unavailable = param_get(parameters, 'unavailable', default=False)
678
+ ignore_availability = param_get(parameters, 'ignore_availability', default='unavailable' in parameters)
679
+ rse_expression = param_get(parameters, 'rse_expression', default=None)
680
+ all_states = param_get(parameters, 'all_states', default=False)
681
+ domain = param_get(parameters, 'domain', default=None)
682
+ if 'signature_lifetime' in parameters:
683
+ signature_lifetime = param_get(parameters, 'signature_lifetime')
684
+ else:
685
+ # hardcoded default of 10 minutes if config is not parseable
686
+ signature_lifetime = config_get_int('credentials', 'signature_lifetime', raise_exception=False, default=600)
687
+ resolve_archives = param_get(parameters, 'resolve_archives', default=True)
688
+ resolve_parents = param_get(parameters, 'resolve_parents', default=False)
689
+ updated_after = param_get(parameters, 'updated_after', default=None)
690
+ if updated_after is not None:
691
+ if isinstance(updated_after, (int, float)):
692
+ # convert from epoch time stamp to datetime object
693
+ updated_after = datetime.utcfromtimestamp(updated_after)
694
+ else:
695
+ # attempt UTC format '%Y-%m-%dT%H:%M:%S' conversion
696
+ updated_after = datetime.strptime(updated_after, '%Y-%m-%dT%H:%M:%S')
697
+ nrandom = param_get(parameters, 'nrandom', default=None)
698
+ if nrandom:
699
+ nrandom = int(nrandom)
700
+
701
+ limit = request.args.get('limit', default=None)
702
+ select = request.args.get('select', default=select)
703
+ select = request.args.get('sort', default=select)
704
+
705
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
706
+ if metalink and schemes is None:
707
+ schemes = SUPPORTED_PROTOCOLS
708
+
709
+ content_type = 'application/metalink4+xml' if metalink else 'application/x-json-stream'
710
+
711
+ try:
712
+ def _list_and_sort_replicas(request_id, issuer, vo):
713
+ # we need to call list_replicas before starting to reply
714
+ # otherwise the exceptions won't be propagated correctly
715
+ for rfile in list_replicas(dids=dids, schemes=schemes,
716
+ unavailable=unavailable,
717
+ request_id=request_id,
718
+ ignore_availability=ignore_availability,
719
+ all_states=all_states,
720
+ rse_expression=rse_expression,
721
+ client_location=client_location,
722
+ domain=domain,
723
+ signature_lifetime=signature_lifetime,
724
+ resolve_archives=resolve_archives,
725
+ resolve_parents=resolve_parents,
726
+ nrandom=nrandom,
727
+ updated_after=updated_after,
728
+ issuer=issuer,
729
+ vo=vo):
730
+
731
+ # Sort rfile['pfns'] and limit its size according to "limit" parameter
732
+ lanreplicas = {}
733
+ wanreplicas = {}
734
+ for pfn, replica in rfile['pfns'].items():
735
+ replica_tuple = (replica['domain'], replica['priority'], replica['rse'], replica['client_extract'])
736
+ if replica_tuple[0] == 'lan':
737
+ lanreplicas[pfn] = replica_tuple
738
+ else:
739
+ wanreplicas[pfn] = replica_tuple
740
+
741
+ rfile['pfns'] = dict(_sorted_with_priorities(replicas=rfile['pfns'],
742
+ # Lan replicas sorted by priority; followed by wan replicas sorted by selection criteria
743
+ sorted_pfns=chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
744
+ sort_replicas(wanreplicas, client_location, selection=select)),
745
+ limit=limit))
746
+ yield rfile
747
+
748
+ rfiles = _list_and_sort_replicas(request_id=request.environ.get('request_id'),
749
+ issuer=request.environ.get('issuer'),
750
+ vo=request.environ.get('vo'))
751
+ if metalink:
752
+ policy_schema = config_get('policy', 'schema', raise_exception=False, default='generic')
753
+ response_generator = _generate_metalink_response(rfiles, policy_schema)
754
+ else:
755
+ response_generator = _generate_json_response(rfiles)
756
+ return try_stream(response_generator, content_type=content_type)
757
+ except (InvalidObject, DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
758
+ return generate_http_error_flask(400, error)
759
+
760
+
761
+ class ReplicasDIDs(ErrorHandlingMethodView):
762
+
763
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
764
+ def post(self):
765
+ """
766
+ ---
767
+ summary: List Replicas Dids
768
+ description: List the DIDs associated to a list of replicas.
769
+ tags:
770
+ - Replicas
771
+ requestBody:
772
+ content:
773
+ application/json:
774
+ schema:
775
+ type: object
776
+ required:
777
+ - rse
778
+ properties:
779
+ pfns:
780
+ description: The list of pfns.
781
+ type: array
782
+ items:
783
+ type: string
784
+ rse:
785
+ description: The RSE name.
786
+ type: string
787
+ responses:
788
+ 200:
789
+ description: OK
790
+ content:
791
+ application/x-json-stream:
792
+ schema:
793
+ type: array
794
+ items:
795
+ type: object
796
+ additionalProperties:
797
+ x-additionalPropertiesName: mapped PFNs to DIDs
798
+ description: A mapping from a pfn to a did.
799
+ type: object
800
+ properties:
801
+ scope:
802
+ description: The scope of the DID.
803
+ type: string
804
+ name:
805
+ description: The name of the DID.
806
+ type: string
807
+ 400:
808
+ description: Cannot decode json parameter list.
809
+ 401:
810
+ description: Invalid Auth Token
811
+ 404:
812
+ description: Not found
813
+ 406:
814
+ description: Not acceptable
815
+ """
816
+ parameters = json_parameters()
817
+ pfns = param_get(parameters, 'pfns', default=[])
818
+ rse = param_get(parameters, 'rse')
819
+
820
+ try:
821
+ def generate(vo):
822
+ for pfn in get_did_from_pfns(pfns, rse, vo=vo):
823
+ yield dumps(pfn) + '\n'
824
+
825
+ return try_stream(generate(vo=request.environ.get('vo')))
826
+ except AccessDenied as error:
827
+ return generate_http_error_flask(401, error)
828
+
829
+
830
+ class BadReplicas(ErrorHandlingMethodView):
831
+
832
+ @check_accept_header_wrapper_flask(['application/json'])
833
+ def post(self):
834
+ """
835
+ ---
836
+ summary: Declare Bad Replicas
837
+ description: Declares a list of bad replicas.
838
+ tags:
839
+ - Replicas
840
+ requestBody:
841
+ content:
842
+ application/json:
843
+ schema:
844
+ type: object
845
+ properties:
846
+ replicas:
847
+ description: The list of pfns or list of dicts with "scope", "name", "rse_id"/"rse"
848
+ type: array
849
+ items:
850
+ type: string
851
+ pfns:
852
+ deprecated: true
853
+ description: The list of pfns, for backward compatibility with older versions of the ReplicaClient
854
+ type: array
855
+ items:
856
+ type: string
857
+ reason:
858
+ description: The reason for the declaration.
859
+ type: string
860
+ force:
861
+ description: If true, ignore existing replica status in the bad_replicas table.
862
+ type: boolean
863
+ responses:
864
+ 201:
865
+ description: OK
866
+ content:
867
+ application/json:
868
+ schema:
869
+ description: Returns the not declared files.
870
+ type: array
871
+ 400:
872
+ description: Can not decode json parameter list.
873
+ 404:
874
+ description: Not found
875
+ 406:
876
+ description: Not acceptable
877
+ """
878
+ parameters = json_parameters()
879
+ replicas = param_get(parameters, 'replicas', default=[]) or param_get(parameters, 'pfns', default=[])
880
+ reason = param_get(parameters, 'reason', default=None)
881
+ force = param_get(parameters, 'force', default=False)
882
+
883
+ try:
884
+ not_declared_files = declare_bad_file_replicas(replicas, reason=reason,
885
+ issuer=request.environ.get('issuer'), vo=request.environ.get('vo'),
886
+ force=force)
887
+ return not_declared_files, 201
888
+ except AccessDenied as error:
889
+ return generate_http_error_flask(401, error)
890
+ except (RSENotFound, ReplicaNotFound) as error:
891
+ return generate_http_error_flask(404, error)
892
+
893
+
894
+ class QuarantineReplicas(ErrorHandlingMethodView):
895
+
896
+ def post(self):
897
+ """
898
+ ---
899
+ summary: Quarantine replicas
900
+ description: Quarantine replicas.
901
+ tags:
902
+ - Replicas
903
+ requestBody:
904
+ content:
905
+ application/json:
906
+ schema:
907
+ type: object
908
+ required:
909
+ - replicas
910
+ properties:
911
+ replicas:
912
+ description: replicas
913
+ type: array
914
+ items:
915
+ type: object
916
+ required:
917
+ - path
918
+ properties:
919
+ path:
920
+ description: path
921
+ type: string
922
+ scope:
923
+ description: scope
924
+ type: string
925
+ name:
926
+ description: name
927
+ type: string
928
+ rse:
929
+ description: RSE name
930
+ type: string
931
+ rse_id:
932
+ description: RSE id
933
+ type: string
934
+ responses:
935
+ 200:
936
+ description: OK
937
+ 403:
938
+ description: Forbidden.
939
+ 404:
940
+ description: Not found
941
+ """
942
+
943
+ parameters = json_parameters()
944
+ replicas = param_get(parameters, 'replicas', default=[])
945
+ rse = param_get(parameters, 'rse', default=None)
946
+ rse_id = param_get(parameters, 'rse_id', default=None)
947
+ vo = request.environ.get('vo')
948
+ issuer = request.environ.get('issuer')
949
+
950
+ if replicas:
951
+ try:
952
+ quarantine_file_replicas(replicas, issuer, rse=rse, rse_id=rse_id, vo=vo)
953
+ except AccessDenied as error:
954
+ return generate_http_error_flask(403, error)
955
+ except (RSENotFound, ReplicaNotFound) as error:
956
+ return generate_http_error_flask(404, error)
957
+
958
+ return '', 200
959
+
960
+
961
+ class SuspiciousReplicas(ErrorHandlingMethodView):
962
+
963
+ @check_accept_header_wrapper_flask(['application/json'])
964
+ def post(self):
965
+ """
966
+ ---
967
+ summary: Declare Suspicious Replicas
968
+ description: Declare a list of suspicious replicas.
969
+ tags:
970
+ - Replicas
971
+ requestBody:
972
+ content:
973
+ application/json:
974
+ schema:
975
+ type: object
976
+ properties:
977
+ pfns:
978
+ description: The list of pfns.
979
+ type: array
980
+ items:
981
+ type: string
982
+ reason:
983
+ description: The reason for the declaration.
984
+ type: string
985
+ responses:
986
+ 201:
987
+ description: OK
988
+ content:
989
+ application/json:
990
+ schema:
991
+ description: Returns the not declared files.
992
+ type: array
993
+ 400:
994
+ description: Can not decode json parameter list.
995
+ 404:
996
+ description: Not found
997
+ 406:
998
+ description: Not acceptable
999
+ """
1000
+ parameters = json_parameters(parse_response)
1001
+ pfns = param_get(parameters, 'pfns', default=[])
1002
+ reason = param_get(parameters, 'reason', default=None)
1003
+
1004
+ try:
1005
+ not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
1006
+ return not_declared_files, 201
1007
+ except AccessDenied as error:
1008
+ return generate_http_error_flask(401, error)
1009
+
1010
+ @check_accept_header_wrapper_flask(['application/json'])
1011
+ def get(self):
1012
+ """
1013
+ ---
1014
+ summary: List Suspicious Replicas
1015
+ description: List the suspicious replicas on a list of RSEs.
1016
+ tags:
1017
+ - Replicas
1018
+ parameters:
1019
+ - name: rse_expression
1020
+ in: query
1021
+ description: The RSE expression to filter for.
1022
+ schema:
1023
+ type: string
1024
+ - name: younger_than
1025
+ in: query
1026
+ description: Date to filter for.
1027
+ schema:
1028
+ type: string
1029
+ - name: nattempts
1030
+ in: query
1031
+ description: The maximum number of attempts to make.
1032
+ schema:
1033
+ type: integer
1034
+ responses:
1035
+ 200:
1036
+ description: OK
1037
+ content:
1038
+ application/json:
1039
+ schema:
1040
+ type: array
1041
+ items:
1042
+ type: object
1043
+ properties:
1044
+ scope:
1045
+ description: The scope of the Replica.
1046
+ type: string
1047
+ name:
1048
+ description: The name of the Replica.
1049
+ type: string
1050
+ rse:
1051
+ description: The rse name.
1052
+ type: string
1053
+ rse_id:
1054
+ description: The id of the rse.
1055
+ type: string
1056
+ cnt:
1057
+ description: The number of replicas.
1058
+ type: integer
1059
+ created_at:
1060
+ description: The time when the replica was created.
1061
+ type: string
1062
+ 401:
1063
+ description: Invalid Auth Token
1064
+ 404:
1065
+ description: Not found
1066
+ 406:
1067
+ description: Not acceptable
1068
+ """
1069
+ rse_expression, younger_than, nattempts = None, None, None
1070
+ if request.query_string:
1071
+ query_string = request.query_string.decode(encoding='utf-8')
1072
+ try:
1073
+ params = loads(unquote(query_string))
1074
+ except ValueError:
1075
+ params = parse_qs(query_string)
1076
+
1077
+ if 'rse_expression' in params:
1078
+ rse_expression = params['rse_expression'][0]
1079
+ if 'younger_than' in params and params['younger_than'][0]:
1080
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S")
1081
+ if 'nattempts' in params:
1082
+ nattempts = int(params['nattempts'][0])
1083
+
1084
+ result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
1085
+ return Response(render_json(result), 200, content_type='application/json')
1086
+
1087
+
1088
+ class BadReplicasStates(ErrorHandlingMethodView):
1089
+
1090
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1091
+ def get(self):
1092
+ """
1093
+ ---
1094
+ summary: List Bad Replicas By States
1095
+ description: List the bad or suspicious replicas by states.
1096
+ tags:
1097
+ - Replicas
1098
+ parameters:
1099
+ - name: state
1100
+ in: query
1101
+ description: The state of the file.
1102
+ schema:
1103
+ type: string
1104
+ enum: [SUSPICIOUS, BAD]
1105
+ - name: rse
1106
+ in: query
1107
+ description: The rse name.
1108
+ schema:
1109
+ type: string
1110
+ - name: younger_than
1111
+ in: query
1112
+ description: Date to select bad replicas younger than this date.
1113
+ schema:
1114
+ type: string
1115
+ format: date-time
1116
+ - name: older_than
1117
+ in: query
1118
+ description: Date to select bad replicas older than this date.
1119
+ schema:
1120
+ type: string
1121
+ format: date-time
1122
+ - name: limit
1123
+ in: query
1124
+ description: The maximum number of replicas returned.
1125
+ schema:
1126
+ type: integer
1127
+ - name: list_pfns
1128
+ in: query
1129
+ description: Flag to include pfns.
1130
+ schema:
1131
+ type: boolean
1132
+ responses:
1133
+ 200:
1134
+ description: OK
1135
+ content:
1136
+ application/x-json-stream:
1137
+ schema:
1138
+ description: A list of all result replicas.
1139
+ type: array
1140
+ items:
1141
+ oneOf:
1142
+ - type: object
1143
+ properties:
1144
+ scope:
1145
+ description: The scope of the replica.
1146
+ type: string
1147
+ name:
1148
+ description: The name of the replica.
1149
+ type: string
1150
+ type:
1151
+ description: The type of the replica.
1152
+ type: string
1153
+ - type: object
1154
+ properties:
1155
+ scope:
1156
+ description: The scope of the replica.
1157
+ type: string
1158
+ name:
1159
+ description: The name of the replica.
1160
+ type: string
1161
+ rse:
1162
+ description: The name of the associated rse.
1163
+ type: string
1164
+ rse_id:
1165
+ description: The id of the associated rse.
1166
+ type: string
1167
+ state:
1168
+ description: The state of the replica.
1169
+ type: string
1170
+ created_at:
1171
+ description: The date-time the replica was created.
1172
+ type: string
1173
+ format: date-time
1174
+ updated_at:
1175
+ description: The date-time the replica was updated.
1176
+ type: string
1177
+ format: date-time
1178
+ 401:
1179
+ description: Invalid Auth Token
1180
+ 406:
1181
+ description: Not acceptable
1182
+ """
1183
+ state, rse, younger_than, older_than, limit, list_pfns = None, None, None, None, None, None
1184
+ if request.query_string:
1185
+ query_string = request.query_string.decode(encoding='utf-8')
1186
+ try:
1187
+ params = loads(unquote(query_string))
1188
+ except ValueError:
1189
+ params = parse_qs(query_string)
1190
+ if 'state' in params:
1191
+ state = params['state'][0]
1192
+ if isinstance(state, str):
1193
+ state = BadFilesStatus(state)
1194
+ if 'rse' in params:
1195
+ rse = params['rse'][0]
1196
+ if 'younger_than' in params:
1197
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1198
+ if 'older_than' in params and params['older_than']:
1199
+ older_than = datetime.strptime(params['older_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1200
+ if 'limit' in params:
1201
+ limit = int(params['limit'][0])
1202
+ if 'list_pfns' in params:
1203
+ list_pfns = bool(params['list_pfns'][0])
1204
+
1205
+ def generate(vo):
1206
+ for row in list_bad_replicas_status(state=state, rse=rse, younger_than=younger_than,
1207
+ older_than=older_than, limit=limit, list_pfns=list_pfns,
1208
+ vo=vo):
1209
+ yield dumps(row, cls=APIEncoder) + '\n'
1210
+
1211
+ return try_stream(generate(vo=request.environ.get('vo')))
1212
+
1213
+
1214
+ class BadReplicasSummary(ErrorHandlingMethodView):
1215
+
1216
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1217
+ def get(self):
1218
+ """
1219
+ ---
1220
+ summary: Bad Replicas Summary
1221
+ description: Return a summary of the bad replicas by incident.
1222
+ tags:
1223
+ - Replicas
1224
+ parameters:
1225
+ - name: rse_expression
1226
+ in: query
1227
+ description: The RSE expression.
1228
+ schema:
1229
+ type: string
1230
+ - name: from_date
1231
+ in: query
1232
+ description: The start date.
1233
+ schema:
1234
+ type: string
1235
+ format: date-time
1236
+ - name: to_date
1237
+ in: query
1238
+ description: The end date.
1239
+ schema:
1240
+ type: string
1241
+ format: date-time
1242
+ responses:
1243
+ 200:
1244
+ description: OK
1245
+ content:
1246
+ application/x-json-stream:
1247
+ schema:
1248
+ description: A list of summaries.
1249
+ type: array
1250
+ items:
1251
+ type: object
1252
+ properties:
1253
+ rse:
1254
+ description: The name of the associated RSE.
1255
+ type: string
1256
+ rse_id:
1257
+ description: The id of the associated RSE.
1258
+ type: string
1259
+ created_at:
1260
+ description: The creation date-time.
1261
+ type: string
1262
+ format: date-time
1263
+ reason:
1264
+ description: The reason for the incident.
1265
+ type: string
1266
+ 401:
1267
+ description: Invalid Auth Token
1268
+ 406:
1269
+ description: Not acceptable
1270
+ """
1271
+ rse_expression, from_date, to_date = None, None, None
1272
+ if request.query_string:
1273
+ query_string = request.query_string.decode(encoding='utf-8')
1274
+ try:
1275
+ params = loads(unquote(query_string))
1276
+ except ValueError:
1277
+ params = parse_qs(query_string)
1278
+ if 'rse_expression' in params:
1279
+ rse_expression = params['rse_expression'][0]
1280
+ if 'from_date' in params and params['from_date'][0]:
1281
+ from_date = datetime.strptime(params['from_date'][0], "%Y-%m-%d")
1282
+ if 'to_date' in params:
1283
+ to_date = datetime.strptime(params['to_date'][0], "%Y-%m-%d")
1284
+
1285
+ def generate(vo):
1286
+ for row in get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date,
1287
+ to_date=to_date, vo=vo):
1288
+ yield dumps(row, cls=APIEncoder) + '\n'
1289
+
1290
+ return try_stream(generate(vo=request.environ.get('vo')))
1291
+
1292
+
1293
+ class DatasetReplicas(ErrorHandlingMethodView):
1294
+
1295
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1296
+ def get(self, scope_name):
1297
+ """
1298
+ ---
1299
+ summary: List Dataset Replicas
1300
+ description: List dataset replicas.
1301
+ tags:
1302
+ - Replicas
1303
+ parameters:
1304
+ - name: scope_name
1305
+ in: path
1306
+ description: data identifier (scope)/(name).
1307
+ schema:
1308
+ type: string
1309
+ style: simple
1310
+ - name: deep
1311
+ in: query
1312
+ description: Flag to ennable lookup at the file level.
1313
+ schema:
1314
+ type: boolean
1315
+ responses:
1316
+ 200:
1317
+ description: OK
1318
+ content:
1319
+ application/x-json-stream:
1320
+ schema:
1321
+ description: A list of dataset replicas.
1322
+ type: array
1323
+ items:
1324
+ type: object
1325
+ properties:
1326
+ scope:
1327
+ description: The scope of the replica.
1328
+ type: string
1329
+ name:
1330
+ description: The name of the replica.
1331
+ type: string
1332
+ rse:
1333
+ description: The name of the associated RSE.
1334
+ type: string
1335
+ rse_id:
1336
+ description: The id of the associated RSE.
1337
+ type: string
1338
+ bytes:
1339
+ description: The size of the replica.
1340
+ type: integer
1341
+ length:
1342
+ description: The length of the replica.
1343
+ type: integer
1344
+ available_bytes:
1345
+ description: The number of available bytes of the replica.
1346
+ type: integer
1347
+ available_length:
1348
+ description: The available length of the replica.
1349
+ type: integer
1350
+ state:
1351
+ description: The state of the replica.
1352
+ type: string
1353
+ created_at:
1354
+ description: The date-time the replica was created.
1355
+ type: string
1356
+ format: date-time
1357
+ updated_at:
1358
+ description: The date-time the replica was updated.
1359
+ type: string
1360
+ format: date-time
1361
+ accessed_at:
1362
+ description: The date-time the replica was accessed.
1363
+ type: string
1364
+ format: date-time
1365
+ 401:
1366
+ description: Invalid Auth Token
1367
+ 404:
1368
+ description: Not found
1369
+ 406:
1370
+ description: Not acceptable
1371
+ """
1372
+ try:
1373
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1374
+
1375
+ def generate(_deep, vo):
1376
+ for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
1377
+ yield dumps(row, cls=APIEncoder) + '\n'
1378
+
1379
+ deep = request.args.get('deep', default=False)
1380
+
1381
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1382
+ except ValueError as error:
1383
+ return generate_http_error_flask(400, error)
1384
+
1385
+
1386
+ class DatasetReplicasBulk(ErrorHandlingMethodView):
1387
+
1388
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1389
+ def post(self):
1390
+ """
1391
+ ---
1392
+ summary: List Dataset Replicas for Multiple DIDs
1393
+ description: List dataset replicas for multiple dids.
1394
+ tags:
1395
+ - Replicas
1396
+ requestBody:
1397
+ content:
1398
+ application/json:
1399
+ schema:
1400
+ type: object
1401
+ required:
1402
+ - dids
1403
+ properties:
1404
+ dids:
1405
+ description: A list of dids.
1406
+ type: array
1407
+ items:
1408
+ description: A did.
1409
+ type: object
1410
+ properties:
1411
+ scope:
1412
+ description: The scope of the did.
1413
+ type: string
1414
+ name:
1415
+ description: The name of the did.
1416
+ type: string
1417
+ responses:
1418
+ 200:
1419
+ description: OK
1420
+ content:
1421
+ application/x-json-stream:
1422
+ schema:
1423
+ description: A list of dataset replicas.
1424
+ type: array
1425
+ items:
1426
+ type: object
1427
+ properties:
1428
+ scope:
1429
+ description: The scope of the replica.
1430
+ type: string
1431
+ name:
1432
+ description: The name of the replica.
1433
+ type: string
1434
+ rse:
1435
+ description: The name of the associated RSE.
1436
+ type: string
1437
+ rse_id:
1438
+ description: The id of the associated RSE.
1439
+ type: string
1440
+ bytes:
1441
+ description: The size of the replica.
1442
+ type: integer
1443
+ length:
1444
+ description: The length of the replica.
1445
+ type: integer
1446
+ available_bytes:
1447
+ description: The number of available bytes of the replica.
1448
+ type: integer
1449
+ available_length:
1450
+ description: The available length of the replica.
1451
+ type: integer
1452
+ state:
1453
+ description: The state of the replica.
1454
+ type: string
1455
+ created_at:
1456
+ description: The date-time the replica was created.
1457
+ type: string
1458
+ format: date-time
1459
+ updated_at:
1460
+ description: The date-time the replica was updated.
1461
+ type: string
1462
+ format: date-time
1463
+ accessed_at:
1464
+ description: The date-time the replica was accessed.
1465
+ type: string
1466
+ format: date-time
1467
+ 400:
1468
+ description: Bad Request.
1469
+ 401:
1470
+ description: Invalid Auth Token
1471
+ 404:
1472
+ description: Not found
1473
+ 406:
1474
+ description: Not acceptable
1475
+ """
1476
+ parameters = json_parameters(parse_response)
1477
+ dids = param_get(parameters, 'dids')
1478
+ if len(dids) == 0:
1479
+ return generate_http_error_flask(400, ValueError.__name__, 'List of DIDs is empty')
1480
+
1481
+ try:
1482
+ def generate(vo):
1483
+ for row in list_dataset_replicas_bulk(dids=dids, vo=vo):
1484
+ yield dumps(row, cls=APIEncoder) + '\n'
1485
+
1486
+ return try_stream(generate(vo=request.environ.get('vo')))
1487
+ except InvalidObject as error:
1488
+ return generate_http_error_flask(400, error, f'Cannot validate DIDs: {error}')
1489
+
1490
+
1491
+ class DatasetReplicasVP(ErrorHandlingMethodView):
1492
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1493
+ def get(self, scope_name):
1494
+ """
1495
+ ---
1496
+ summary: List Dataset Replicas VP
1497
+ description: |
1498
+ List dataset replicas using the Virtual Placement service.
1499
+ This is an RnD function and might change or go away at any time.
1500
+ tags:
1501
+ - Replicas
1502
+ parameters:
1503
+ - name: scope_name
1504
+ in: path
1505
+ description: data identifier (scope)/(name).
1506
+ schema:
1507
+ type: string
1508
+ style: simple
1509
+ - name: deep
1510
+ in: query
1511
+ description: Flag to ennable lookup at the file level.
1512
+ schema:
1513
+ type: boolean
1514
+ responses:
1515
+ 200:
1516
+ description: OK. This needs documentation!
1517
+ 401:
1518
+ description: Invalid Auth Token
1519
+ 406:
1520
+ description: Not acceptable
1521
+ """
1522
+ try:
1523
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1524
+
1525
+ def generate(_deep, vo):
1526
+ for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
1527
+ yield dumps(row, cls=APIEncoder) + '\n'
1528
+
1529
+ deep = request.args.get('deep', default=False)
1530
+
1531
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1532
+ except ValueError as error:
1533
+ return generate_http_error_flask(400, error)
1534
+
1535
+
1536
+ class ReplicasRSE(ErrorHandlingMethodView):
1537
+
1538
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1539
+ def get(self, rse):
1540
+ """
1541
+ ---
1542
+ summary: List Dataset Replicas per RSE
1543
+ description: List dataset replicas per RSE.
1544
+ tags:
1545
+ - Replicas
1546
+ parameters:
1547
+ - name: rse
1548
+ in: path
1549
+ description: The rse to filter for.
1550
+ schema:
1551
+ type: string
1552
+ style: simple
1553
+ responses:
1554
+ 200:
1555
+ description: OK
1556
+ content:
1557
+ application/x-json-stream:
1558
+ schema:
1559
+ description: A list of dataset replicas.
1560
+ type: array
1561
+ items:
1562
+ type: object
1563
+ properties:
1564
+ scope:
1565
+ description: The scope of the replica.
1566
+ type: string
1567
+ name:
1568
+ description: The name of the replica.
1569
+ type: string
1570
+ rse:
1571
+ description: The name of the associated RSE.
1572
+ type: string
1573
+ rse_id:
1574
+ description: The id of the associated RSE.
1575
+ type: string
1576
+ bytes:
1577
+ description: The size of the replica.
1578
+ type: integer
1579
+ length:
1580
+ description: The length of the replica.
1581
+ type: integer
1582
+ available_bytes:
1583
+ description: The number of available bytes of the replica.
1584
+ type: integer
1585
+ available_length:
1586
+ description: The available length of the replica.
1587
+ type: integer
1588
+ state:
1589
+ description: The state of the replica.
1590
+ type: string
1591
+ created_at:
1592
+ description: The date-time the replica was created.
1593
+ type: string
1594
+ format: date-time
1595
+ updated_at:
1596
+ description: The date-time the replica was updated.
1597
+ type: string
1598
+ format: date-time
1599
+ accessed_at:
1600
+ description: The date-time the replica was accessed.
1601
+ type: string
1602
+ format: date-time
1603
+ 401:
1604
+ description: Invalid Auth Token
1605
+ 406:
1606
+ description: Not acceptable
1607
+ """
1608
+
1609
+ def generate(vo):
1610
+ for row in list_datasets_per_rse(rse=rse, vo=vo):
1611
+ yield dumps(row, cls=APIEncoder) + '\n'
1612
+
1613
+ return try_stream(generate(vo=request.environ.get('vo')))
1614
+
1615
+
1616
+ class BadDIDs(ErrorHandlingMethodView):
1617
+
1618
+ def post(self):
1619
+ """
1620
+ ---
1621
+ summary: Mark Bad by DID
1622
+ description: Declare a list of bad replicas by DID.
1623
+ tags:
1624
+ - Replicas
1625
+ requestBody:
1626
+ content:
1627
+ application/json:
1628
+ schema:
1629
+ type: object
1630
+ properties:
1631
+ expires_at:
1632
+ description: The expires at value.
1633
+ type: string
1634
+ format: date-time
1635
+ dids:
1636
+ description: The list of dids associated with the bad replicas.
1637
+ type: array
1638
+ items:
1639
+ type: object
1640
+ properties:
1641
+ scope:
1642
+ description: The scope of the did.
1643
+ type: string
1644
+ name:
1645
+ description: The name of the did.
1646
+ type: string
1647
+ rse:
1648
+ description: The name of the rse.
1649
+ type: string
1650
+ reason:
1651
+ description: The reason for the change.
1652
+ type: string
1653
+ responses:
1654
+ 201:
1655
+ description: OK
1656
+ content:
1657
+ application/json:
1658
+ schema:
1659
+ description: All files not declared as bad.
1660
+ type: array
1661
+ items:
1662
+ type: string
1663
+ 400:
1664
+ description: Cannot decode json parameter list.
1665
+ 401:
1666
+ description: Invalid Auth Token
1667
+ 404:
1668
+ description: Not found
1669
+ """
1670
+ parameters = json_parameters(parse_response)
1671
+ expires_at = param_get(parameters, 'expires_at', default=None)
1672
+ if expires_at:
1673
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1674
+
1675
+ try:
1676
+ not_declared_files = add_bad_dids(
1677
+ dids=param_get(parameters, 'dids', default=[]),
1678
+ rse=param_get(parameters, 'rse', default=None),
1679
+ issuer=request.environ.get('issuer'),
1680
+ state=BadFilesStatus.BAD,
1681
+ reason=param_get(parameters, 'reason', default=None),
1682
+ expires_at=expires_at,
1683
+ vo=request.environ.get('vo'),
1684
+ )
1685
+ except (ValueError, InvalidType) as error:
1686
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1687
+ except AccessDenied as error:
1688
+ return generate_http_error_flask(401, error)
1689
+ except ReplicaNotFound as error:
1690
+ return generate_http_error_flask(404, error)
1691
+ except Duplicate as error:
1692
+ return generate_http_error_flask(409, error)
1693
+
1694
+ return Response(dumps(not_declared_files), status=201, content_type='application/json')
1695
+
1696
+
1697
+ class BadPFNs(ErrorHandlingMethodView):
1698
+
1699
+ def post(self):
1700
+ """
1701
+ ---
1702
+ summary: Declare Bad PFNs
1703
+ description: Declare a list of bad PFNs.
1704
+ tags:
1705
+ - Replicas
1706
+ requestBody:
1707
+ content:
1708
+ application/json:
1709
+ schema:
1710
+ type: object
1711
+ properties:
1712
+ expires_at:
1713
+ description: The expires at value. Only apply to TEMPORARY_UNAVAILABLE.
1714
+ type: string
1715
+ format: date-time
1716
+ pfns:
1717
+ description: The list of pfns associated with the bad PFNs.
1718
+ type: array
1719
+ items:
1720
+ type: string
1721
+ state:
1722
+ description: The state to set the PFNs to.
1723
+ type: string
1724
+ enum: ["BAD", "SUSPICIOUS", "TEMPORARY_UNAVAILABLE"]
1725
+ reason:
1726
+ description: The reason for the change.
1727
+ type: string
1728
+ responses:
1729
+ 201:
1730
+ description: Created
1731
+ 400:
1732
+ description: Cannot decode json parameter list.
1733
+ 401:
1734
+ description: Invalid Auth Token
1735
+ 404:
1736
+ description: Replica not found
1737
+ 409:
1738
+ description: Duplicate
1739
+ """
1740
+ parameters = json_parameters(parse_response)
1741
+ expires_at = param_get(parameters, 'expires_at', default=None)
1742
+ if expires_at:
1743
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1744
+
1745
+ try:
1746
+ add_bad_pfns(
1747
+ pfns=param_get(parameters, 'pfns', default=[]),
1748
+ issuer=request.environ.get('issuer'),
1749
+ state=param_get(parameters, 'state', default=None),
1750
+ reason=param_get(parameters, 'reason', default=None),
1751
+ expires_at=expires_at,
1752
+ vo=request.environ.get('vo'),
1753
+ )
1754
+ except (ValueError, InvalidType) as error:
1755
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1756
+ except AccessDenied as error:
1757
+ return generate_http_error_flask(401, error)
1758
+ except ReplicaNotFound as error:
1759
+ return generate_http_error_flask(404, error)
1760
+ except Duplicate as error:
1761
+ return generate_http_error_flask(409, error)
1762
+
1763
+ return 'Created', 201
1764
+
1765
+
1766
+ class Tombstone(ErrorHandlingMethodView):
1767
+
1768
+ def post(self):
1769
+ """
1770
+ ---
1771
+ summary: Set Tombstone
1772
+ description: Set a tombstone on a list of replicas.
1773
+ tags:
1774
+ - Replicas
1775
+ requestBody:
1776
+ content:
1777
+ application/json:
1778
+ schema:
1779
+ type: object
1780
+ properties:
1781
+ replicas:
1782
+ description: The replicas to set the tombstone to.
1783
+ type: array
1784
+ items:
1785
+ type: object
1786
+ required:
1787
+ - rse
1788
+ - scope
1789
+ - name
1790
+ properties:
1791
+ rse:
1792
+ description: The rse associated with the tombstone.
1793
+ type: string
1794
+ scope:
1795
+ description: The scope of the replica
1796
+ type: string
1797
+ name:
1798
+ description: The name of the replica.
1799
+ type: string
1800
+ responses:
1801
+ 201:
1802
+ description: Created
1803
+ 401:
1804
+ description: Invalid Auth Token
1805
+ 404:
1806
+ description: Not found
1807
+ 423:
1808
+ description: Replica is locked.
1809
+ """
1810
+ parameters = json_parameters(parse_response)
1811
+ replicas = param_get(parameters, 'replicas', default=[])
1812
+
1813
+ try:
1814
+ for replica in replicas:
1815
+ set_tombstone(
1816
+ rse=replica['rse'],
1817
+ scope=replica['scope'],
1818
+ name=replica['name'],
1819
+ issuer=request.environ.get('issuer'),
1820
+ vo=request.environ.get('vo'),
1821
+ )
1822
+ except ReplicaNotFound as error:
1823
+ return generate_http_error_flask(404, error)
1824
+ except ReplicaIsLocked as error:
1825
+ return generate_http_error_flask(423, error)
1826
+
1827
+ return 'Created', 201
1828
+
1829
+
1830
+ def blueprint(with_doc=False):
1831
+ bp = AuthenticatedBlueprint('replicas', __name__, url_prefix='/replicas')
1832
+
1833
+ list_replicas_view = ListReplicas.as_view('list_replicas')
1834
+ bp.add_url_rule('/list', view_func=list_replicas_view, methods=['post', ])
1835
+ replicas_view = Replicas.as_view('replicas')
1836
+ if not with_doc:
1837
+ # rule without trailing slash needs to be added before rule with trailing slash
1838
+ bp.add_url_rule('', view_func=replicas_view, methods=['post', 'put', 'delete'])
1839
+ bp.add_url_rule('/', view_func=replicas_view, methods=['post', 'put', 'delete'])
1840
+ suspicious_replicas_view = SuspiciousReplicas.as_view('suspicious_replicas')
1841
+ bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['get', 'post'])
1842
+ bad_replicas_states_view = BadReplicasStates.as_view('bad_replicas_states')
1843
+ bp.add_url_rule('/bad/states', view_func=bad_replicas_states_view, methods=['get', ])
1844
+ bad_replicas_summary_view = BadReplicasSummary.as_view('bad_replicas_summary')
1845
+ bp.add_url_rule('/bad/summary', view_func=bad_replicas_summary_view, methods=['get', ])
1846
+ bad_replicas_pfn_view = BadPFNs.as_view('add_bad_pfns')
1847
+ bp.add_url_rule('/bad/pfns', view_func=bad_replicas_pfn_view, methods=['post', ])
1848
+ bad_replicas_dids_view = BadDIDs.as_view('add_bad_dids')
1849
+ bp.add_url_rule('/bad/dids', view_func=bad_replicas_dids_view, methods=['post', ])
1850
+ replicas_rse_view = ReplicasRSE.as_view('replicas_rse')
1851
+ bp.add_url_rule('/rse/<rse>', view_func=replicas_rse_view, methods=['get', ])
1852
+
1853
+ bad_replicas_view = BadReplicas.as_view('bad_replicas')
1854
+ bp.add_url_rule('/bad', view_func=bad_replicas_view, methods=['post', ])
1855
+
1856
+ quarantine_replicas_view = QuarantineReplicas.as_view('quarantine_replicas')
1857
+ bp.add_url_rule('/quarantine', view_func=quarantine_replicas_view, methods=['post', ])
1858
+
1859
+ replicas_dids_view = ReplicasDIDs.as_view('replicas_dids')
1860
+ bp.add_url_rule('/dids', view_func=replicas_dids_view, methods=['post', ])
1861
+ dataset_replicas_view = DatasetReplicas.as_view('dataset_replicas')
1862
+ bp.add_url_rule('/<path:scope_name>/datasets', view_func=dataset_replicas_view, methods=['get', ])
1863
+ dataset_replicas_bulk_view = DatasetReplicasBulk.as_view('dataset_replicas_bulk')
1864
+ bp.add_url_rule('/datasets_bulk', view_func=dataset_replicas_bulk_view, methods=['post', ])
1865
+ dataset_replicas_vp_view = DatasetReplicasVP.as_view('dataset_replicas_vp')
1866
+ bp.add_url_rule('/<path:scope_name>', view_func=replicas_view, methods=['get', ])
1867
+ set_tombstone_view = Tombstone.as_view('set_tombstone')
1868
+ bp.add_url_rule('/tombstone', view_func=set_tombstone_view, methods=['post', ])
1869
+
1870
+ if not with_doc:
1871
+ bp.add_url_rule('/list/', view_func=list_replicas_view, methods=['post', ])
1872
+ bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['get', 'post'])
1873
+ bp.add_url_rule('/bad/states/', view_func=bad_replicas_states_view, methods=['get', ])
1874
+ bp.add_url_rule('/bad/summary/', view_func=bad_replicas_summary_view, methods=['get', ])
1875
+ bp.add_url_rule('/bad/pfns/', view_func=bad_replicas_pfn_view, methods=['post', ])
1876
+ bp.add_url_rule('/bad/dids/', view_func=bad_replicas_dids_view, methods=['post', ])
1877
+ bp.add_url_rule('/rse/<rse>/', view_func=replicas_rse_view, methods=['get', ])
1878
+ bp.add_url_rule('/bad/', view_func=bad_replicas_view, methods=['post', ])
1879
+ bp.add_url_rule('/dids/', view_func=replicas_dids_view, methods=['post', ])
1880
+ bp.add_url_rule('/datasets_bulk/', view_func=dataset_replicas_bulk_view, methods=['post', ])
1881
+ bp.add_url_rule('/<path:scope_name>/datasets_vp', view_func=dataset_replicas_vp_view, methods=['get', ])
1882
+ bp.add_url_rule('/<path:scope_name>/', view_func=replicas_view, methods=['get', ])
1883
+ bp.add_url_rule('/tombstone/', view_func=set_tombstone_view, methods=['post', ])
1884
+ bp.add_url_rule('/quarantine/', view_func=quarantine_replicas_view, methods=['post', ])
1885
+
1886
+ bp.after_request(response_headers)
1887
+ return bp
1888
+
1889
+
1890
+ def make_doc():
1891
+ """ Only used for sphinx documentation """
1892
+ doc_app = Flask(__name__)
1893
+ doc_app.register_blueprint(blueprint(with_doc=True))
1894
+ return doc_app