rucio 35.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (493) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/client/__init__.py +15 -0
  4. rucio/client/accountclient.py +433 -0
  5. rucio/client/accountlimitclient.py +183 -0
  6. rucio/client/baseclient.py +974 -0
  7. rucio/client/client.py +76 -0
  8. rucio/client/configclient.py +126 -0
  9. rucio/client/credentialclient.py +59 -0
  10. rucio/client/didclient.py +866 -0
  11. rucio/client/diracclient.py +56 -0
  12. rucio/client/downloadclient.py +1785 -0
  13. rucio/client/exportclient.py +44 -0
  14. rucio/client/fileclient.py +50 -0
  15. rucio/client/importclient.py +42 -0
  16. rucio/client/lifetimeclient.py +90 -0
  17. rucio/client/lockclient.py +109 -0
  18. rucio/client/metaconventionsclient.py +140 -0
  19. rucio/client/pingclient.py +44 -0
  20. rucio/client/replicaclient.py +454 -0
  21. rucio/client/requestclient.py +125 -0
  22. rucio/client/rseclient.py +746 -0
  23. rucio/client/ruleclient.py +294 -0
  24. rucio/client/scopeclient.py +90 -0
  25. rucio/client/subscriptionclient.py +173 -0
  26. rucio/client/touchclient.py +82 -0
  27. rucio/client/uploadclient.py +955 -0
  28. rucio/common/__init__.py +13 -0
  29. rucio/common/cache.py +74 -0
  30. rucio/common/config.py +801 -0
  31. rucio/common/constants.py +159 -0
  32. rucio/common/constraints.py +17 -0
  33. rucio/common/didtype.py +189 -0
  34. rucio/common/dumper/__init__.py +335 -0
  35. rucio/common/dumper/consistency.py +452 -0
  36. rucio/common/dumper/data_models.py +318 -0
  37. rucio/common/dumper/path_parsing.py +64 -0
  38. rucio/common/exception.py +1151 -0
  39. rucio/common/extra.py +36 -0
  40. rucio/common/logging.py +420 -0
  41. rucio/common/pcache.py +1408 -0
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +84 -0
  44. rucio/common/schema/__init__.py +150 -0
  45. rucio/common/schema/atlas.py +413 -0
  46. rucio/common/schema/belleii.py +408 -0
  47. rucio/common/schema/domatpc.py +401 -0
  48. rucio/common/schema/escape.py +426 -0
  49. rucio/common/schema/generic.py +433 -0
  50. rucio/common/schema/generic_multi_vo.py +412 -0
  51. rucio/common/schema/icecube.py +406 -0
  52. rucio/common/stomp_utils.py +159 -0
  53. rucio/common/stopwatch.py +55 -0
  54. rucio/common/test_rucio_server.py +148 -0
  55. rucio/common/types.py +403 -0
  56. rucio/common/utils.py +2238 -0
  57. rucio/core/__init__.py +13 -0
  58. rucio/core/account.py +496 -0
  59. rucio/core/account_counter.py +236 -0
  60. rucio/core/account_limit.py +423 -0
  61. rucio/core/authentication.py +620 -0
  62. rucio/core/config.py +456 -0
  63. rucio/core/credential.py +225 -0
  64. rucio/core/did.py +3000 -0
  65. rucio/core/did_meta_plugins/__init__.py +252 -0
  66. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  68. rucio/core/did_meta_plugins/filter_engine.py +613 -0
  69. rucio/core/did_meta_plugins/json_meta.py +240 -0
  70. rucio/core/did_meta_plugins/mongo_meta.py +216 -0
  71. rucio/core/did_meta_plugins/postgres_meta.py +316 -0
  72. rucio/core/dirac.py +237 -0
  73. rucio/core/distance.py +187 -0
  74. rucio/core/exporter.py +59 -0
  75. rucio/core/heartbeat.py +363 -0
  76. rucio/core/identity.py +300 -0
  77. rucio/core/importer.py +259 -0
  78. rucio/core/lifetime_exception.py +377 -0
  79. rucio/core/lock.py +576 -0
  80. rucio/core/message.py +282 -0
  81. rucio/core/meta_conventions.py +203 -0
  82. rucio/core/monitor.py +447 -0
  83. rucio/core/naming_convention.py +195 -0
  84. rucio/core/nongrid_trace.py +136 -0
  85. rucio/core/oidc.py +1461 -0
  86. rucio/core/permission/__init__.py +119 -0
  87. rucio/core/permission/atlas.py +1348 -0
  88. rucio/core/permission/belleii.py +1077 -0
  89. rucio/core/permission/escape.py +1078 -0
  90. rucio/core/permission/generic.py +1130 -0
  91. rucio/core/permission/generic_multi_vo.py +1150 -0
  92. rucio/core/quarantined_replica.py +223 -0
  93. rucio/core/replica.py +4158 -0
  94. rucio/core/replica_sorter.py +366 -0
  95. rucio/core/request.py +3089 -0
  96. rucio/core/rse.py +1875 -0
  97. rucio/core/rse_counter.py +186 -0
  98. rucio/core/rse_expression_parser.py +459 -0
  99. rucio/core/rse_selector.py +302 -0
  100. rucio/core/rule.py +4483 -0
  101. rucio/core/rule_grouping.py +1618 -0
  102. rucio/core/scope.py +180 -0
  103. rucio/core/subscription.py +364 -0
  104. rucio/core/topology.py +490 -0
  105. rucio/core/trace.py +375 -0
  106. rucio/core/transfer.py +1517 -0
  107. rucio/core/vo.py +169 -0
  108. rucio/core/volatile_replica.py +150 -0
  109. rucio/daemons/__init__.py +13 -0
  110. rucio/daemons/abacus/__init__.py +13 -0
  111. rucio/daemons/abacus/account.py +116 -0
  112. rucio/daemons/abacus/collection_replica.py +124 -0
  113. rucio/daemons/abacus/rse.py +117 -0
  114. rucio/daemons/atropos/__init__.py +13 -0
  115. rucio/daemons/atropos/atropos.py +242 -0
  116. rucio/daemons/auditor/__init__.py +289 -0
  117. rucio/daemons/auditor/hdfs.py +97 -0
  118. rucio/daemons/auditor/srmdumps.py +355 -0
  119. rucio/daemons/automatix/__init__.py +13 -0
  120. rucio/daemons/automatix/automatix.py +293 -0
  121. rucio/daemons/badreplicas/__init__.py +13 -0
  122. rucio/daemons/badreplicas/minos.py +322 -0
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  124. rucio/daemons/badreplicas/necromancer.py +196 -0
  125. rucio/daemons/bb8/__init__.py +13 -0
  126. rucio/daemons/bb8/bb8.py +353 -0
  127. rucio/daemons/bb8/common.py +759 -0
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  129. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  130. rucio/daemons/c3po/__init__.py +13 -0
  131. rucio/daemons/c3po/algorithms/__init__.py +13 -0
  132. rucio/daemons/c3po/algorithms/simple.py +134 -0
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +128 -0
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +130 -0
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +294 -0
  136. rucio/daemons/c3po/c3po.py +371 -0
  137. rucio/daemons/c3po/collectors/__init__.py +13 -0
  138. rucio/daemons/c3po/collectors/agis.py +108 -0
  139. rucio/daemons/c3po/collectors/free_space.py +81 -0
  140. rucio/daemons/c3po/collectors/jedi_did.py +57 -0
  141. rucio/daemons/c3po/collectors/mock_did.py +51 -0
  142. rucio/daemons/c3po/collectors/network_metrics.py +71 -0
  143. rucio/daemons/c3po/collectors/workload.py +112 -0
  144. rucio/daemons/c3po/utils/__init__.py +13 -0
  145. rucio/daemons/c3po/utils/dataset_cache.py +50 -0
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +56 -0
  147. rucio/daemons/c3po/utils/expiring_list.py +62 -0
  148. rucio/daemons/c3po/utils/popularity.py +85 -0
  149. rucio/daemons/c3po/utils/timeseries.py +89 -0
  150. rucio/daemons/cache/__init__.py +13 -0
  151. rucio/daemons/cache/consumer.py +197 -0
  152. rucio/daemons/common.py +415 -0
  153. rucio/daemons/conveyor/__init__.py +13 -0
  154. rucio/daemons/conveyor/common.py +562 -0
  155. rucio/daemons/conveyor/finisher.py +529 -0
  156. rucio/daemons/conveyor/poller.py +404 -0
  157. rucio/daemons/conveyor/preparer.py +205 -0
  158. rucio/daemons/conveyor/receiver.py +249 -0
  159. rucio/daemons/conveyor/stager.py +132 -0
  160. rucio/daemons/conveyor/submitter.py +403 -0
  161. rucio/daemons/conveyor/throttler.py +532 -0
  162. rucio/daemons/follower/__init__.py +13 -0
  163. rucio/daemons/follower/follower.py +101 -0
  164. rucio/daemons/hermes/__init__.py +13 -0
  165. rucio/daemons/hermes/hermes.py +774 -0
  166. rucio/daemons/judge/__init__.py +13 -0
  167. rucio/daemons/judge/cleaner.py +159 -0
  168. rucio/daemons/judge/evaluator.py +185 -0
  169. rucio/daemons/judge/injector.py +162 -0
  170. rucio/daemons/judge/repairer.py +154 -0
  171. rucio/daemons/oauthmanager/__init__.py +13 -0
  172. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  173. rucio/daemons/reaper/__init__.py +13 -0
  174. rucio/daemons/reaper/dark_reaper.py +278 -0
  175. rucio/daemons/reaper/reaper.py +743 -0
  176. rucio/daemons/replicarecoverer/__init__.py +13 -0
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  178. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +13 -0
  186. rucio/daemons/storage/consistency/__init__.py +13 -0
  187. rucio/daemons/storage/consistency/actions.py +846 -0
  188. rucio/daemons/tracer/__init__.py +13 -0
  189. rucio/daemons/tracer/kronos.py +536 -0
  190. rucio/daemons/transmogrifier/__init__.py +13 -0
  191. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  192. rucio/daemons/undertaker/__init__.py +13 -0
  193. rucio/daemons/undertaker/undertaker.py +137 -0
  194. rucio/db/__init__.py +13 -0
  195. rucio/db/sqla/__init__.py +52 -0
  196. rucio/db/sqla/constants.py +201 -0
  197. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  198. rucio/db/sqla/migrate_repo/env.py +110 -0
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  327. rucio/db/sqla/models.py +1740 -0
  328. rucio/db/sqla/sautils.py +55 -0
  329. rucio/db/sqla/session.py +498 -0
  330. rucio/db/sqla/types.py +206 -0
  331. rucio/db/sqla/util.py +543 -0
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/gateway/account.py +339 -0
  334. rucio/gateway/account_limit.py +286 -0
  335. rucio/gateway/authentication.py +375 -0
  336. rucio/gateway/config.py +217 -0
  337. rucio/gateway/credential.py +71 -0
  338. rucio/gateway/did.py +970 -0
  339. rucio/gateway/dirac.py +81 -0
  340. rucio/gateway/exporter.py +59 -0
  341. rucio/gateway/heartbeat.py +74 -0
  342. rucio/gateway/identity.py +204 -0
  343. rucio/gateway/importer.py +45 -0
  344. rucio/gateway/lifetime_exception.py +120 -0
  345. rucio/gateway/lock.py +153 -0
  346. rucio/gateway/meta_conventions.py +87 -0
  347. rucio/gateway/permission.py +71 -0
  348. rucio/gateway/quarantined_replica.py +78 -0
  349. rucio/gateway/replica.py +529 -0
  350. rucio/gateway/request.py +321 -0
  351. rucio/gateway/rse.py +600 -0
  352. rucio/gateway/rule.py +417 -0
  353. rucio/gateway/scope.py +99 -0
  354. rucio/gateway/subscription.py +277 -0
  355. rucio/gateway/vo.py +122 -0
  356. rucio/rse/__init__.py +96 -0
  357. rucio/rse/protocols/__init__.py +13 -0
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +122 -0
  360. rucio/rse/protocols/dummy.py +111 -0
  361. rucio/rse/protocols/gfal.py +703 -0
  362. rucio/rse/protocols/globus.py +243 -0
  363. rucio/rse/protocols/gsiftp.py +92 -0
  364. rucio/rse/protocols/http_cache.py +82 -0
  365. rucio/rse/protocols/mock.py +123 -0
  366. rucio/rse/protocols/ngarc.py +209 -0
  367. rucio/rse/protocols/posix.py +250 -0
  368. rucio/rse/protocols/protocol.py +594 -0
  369. rucio/rse/protocols/rclone.py +364 -0
  370. rucio/rse/protocols/rfio.py +136 -0
  371. rucio/rse/protocols/srm.py +338 -0
  372. rucio/rse/protocols/ssh.py +413 -0
  373. rucio/rse/protocols/storm.py +206 -0
  374. rucio/rse/protocols/webdav.py +550 -0
  375. rucio/rse/protocols/xrootd.py +301 -0
  376. rucio/rse/rsemanager.py +764 -0
  377. rucio/tests/__init__.py +13 -0
  378. rucio/tests/common.py +270 -0
  379. rucio/tests/common_server.py +132 -0
  380. rucio/transfertool/__init__.py +13 -0
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +1596 -0
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +201 -0
  387. rucio/transfertool/globus_library.py +181 -0
  388. rucio/transfertool/mock.py +90 -0
  389. rucio/transfertool/transfertool.py +221 -0
  390. rucio/vcsversion.py +11 -0
  391. rucio/version.py +38 -0
  392. rucio/web/__init__.py +13 -0
  393. rucio/web/rest/__init__.py +13 -0
  394. rucio/web/rest/flaskapi/__init__.py +13 -0
  395. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  396. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  398. rucio/web/rest/flaskapi/v1/accounts.py +1089 -0
  399. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  400. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  401. rucio/web/rest/flaskapi/v1/common.py +426 -0
  402. rucio/web/rest/flaskapi/v1/config.py +304 -0
  403. rucio/web/rest/flaskapi/v1/credentials.py +212 -0
  404. rucio/web/rest/flaskapi/v1/dids.py +2334 -0
  405. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  406. rucio/web/rest/flaskapi/v1/export.py +75 -0
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  408. rucio/web/rest/flaskapi/v1/identities.py +261 -0
  409. rucio/web/rest/flaskapi/v1/import.py +132 -0
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  411. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  412. rucio/web/rest/flaskapi/v1/main.py +91 -0
  413. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  414. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  416. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  417. rucio/web/rest/flaskapi/v1/redirect.py +365 -0
  418. rucio/web/rest/flaskapi/v1/replicas.py +1890 -0
  419. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  420. rucio/web/rest/flaskapi/v1/rses.py +2239 -0
  421. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  422. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  424. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  425. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  426. rucio/web/rest/flaskapi/v1/traces.py +100 -0
  427. rucio/web/rest/flaskapi/v1/types.py +20 -0
  428. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  429. rucio/web/rest/main.py +18 -0
  430. rucio/web/rest/metrics.py +27 -0
  431. rucio/web/rest/ping.py +27 -0
  432. rucio-35.7.0.data/data/rucio/etc/alembic.ini.template +71 -0
  433. rucio-35.7.0.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  434. rucio-35.7.0.data/data/rucio/etc/globus-config.yml.template +5 -0
  435. rucio-35.7.0.data/data/rucio/etc/ldap.cfg.template +30 -0
  436. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  437. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  438. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  439. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  440. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  441. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  442. rucio-35.7.0.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  443. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.atlas.client.template +42 -0
  444. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.template +257 -0
  445. rucio-35.7.0.data/data/rucio/etc/rucio_multi_vo.cfg.template +234 -0
  446. rucio-35.7.0.data/data/rucio/requirements.server.txt +268 -0
  447. rucio-35.7.0.data/data/rucio/tools/bootstrap.py +34 -0
  448. rucio-35.7.0.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  449. rucio-35.7.0.data/data/rucio/tools/reset_database.py +40 -0
  450. rucio-35.7.0.data/scripts/rucio +2542 -0
  451. rucio-35.7.0.data/scripts/rucio-abacus-account +74 -0
  452. rucio-35.7.0.data/scripts/rucio-abacus-collection-replica +46 -0
  453. rucio-35.7.0.data/scripts/rucio-abacus-rse +78 -0
  454. rucio-35.7.0.data/scripts/rucio-admin +2447 -0
  455. rucio-35.7.0.data/scripts/rucio-atropos +60 -0
  456. rucio-35.7.0.data/scripts/rucio-auditor +205 -0
  457. rucio-35.7.0.data/scripts/rucio-automatix +50 -0
  458. rucio-35.7.0.data/scripts/rucio-bb8 +57 -0
  459. rucio-35.7.0.data/scripts/rucio-c3po +85 -0
  460. rucio-35.7.0.data/scripts/rucio-cache-client +134 -0
  461. rucio-35.7.0.data/scripts/rucio-cache-consumer +42 -0
  462. rucio-35.7.0.data/scripts/rucio-conveyor-finisher +58 -0
  463. rucio-35.7.0.data/scripts/rucio-conveyor-poller +66 -0
  464. rucio-35.7.0.data/scripts/rucio-conveyor-preparer +37 -0
  465. rucio-35.7.0.data/scripts/rucio-conveyor-receiver +43 -0
  466. rucio-35.7.0.data/scripts/rucio-conveyor-stager +76 -0
  467. rucio-35.7.0.data/scripts/rucio-conveyor-submitter +139 -0
  468. rucio-35.7.0.data/scripts/rucio-conveyor-throttler +104 -0
  469. rucio-35.7.0.data/scripts/rucio-dark-reaper +53 -0
  470. rucio-35.7.0.data/scripts/rucio-dumper +160 -0
  471. rucio-35.7.0.data/scripts/rucio-follower +44 -0
  472. rucio-35.7.0.data/scripts/rucio-hermes +54 -0
  473. rucio-35.7.0.data/scripts/rucio-judge-cleaner +89 -0
  474. rucio-35.7.0.data/scripts/rucio-judge-evaluator +137 -0
  475. rucio-35.7.0.data/scripts/rucio-judge-injector +44 -0
  476. rucio-35.7.0.data/scripts/rucio-judge-repairer +44 -0
  477. rucio-35.7.0.data/scripts/rucio-kronos +43 -0
  478. rucio-35.7.0.data/scripts/rucio-minos +53 -0
  479. rucio-35.7.0.data/scripts/rucio-minos-temporary-expiration +50 -0
  480. rucio-35.7.0.data/scripts/rucio-necromancer +120 -0
  481. rucio-35.7.0.data/scripts/rucio-oauth-manager +63 -0
  482. rucio-35.7.0.data/scripts/rucio-reaper +83 -0
  483. rucio-35.7.0.data/scripts/rucio-replica-recoverer +248 -0
  484. rucio-35.7.0.data/scripts/rucio-rse-decommissioner +66 -0
  485. rucio-35.7.0.data/scripts/rucio-storage-consistency-actions +74 -0
  486. rucio-35.7.0.data/scripts/rucio-transmogrifier +77 -0
  487. rucio-35.7.0.data/scripts/rucio-undertaker +76 -0
  488. rucio-35.7.0.dist-info/METADATA +72 -0
  489. rucio-35.7.0.dist-info/RECORD +493 -0
  490. rucio-35.7.0.dist-info/WHEEL +5 -0
  491. rucio-35.7.0.dist-info/licenses/AUTHORS.rst +97 -0
  492. rucio-35.7.0.dist-info/licenses/LICENSE +201 -0
  493. rucio-35.7.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1890 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from datetime import datetime
16
+ from itertools import chain
17
+ from json import dumps, loads
18
+ from urllib.parse import parse_qs, unquote
19
+ from xml.sax.saxutils import escape
20
+
21
+ from flask import Flask, Response, request
22
+
23
+ from rucio.common.config import config_get, config_get_int
24
+ from rucio.common.constants import SUPPORTED_PROTOCOLS
25
+ from rucio.common.exception import (
26
+ AccessDenied,
27
+ DataIdentifierAlreadyExists,
28
+ DataIdentifierNotFound,
29
+ Duplicate,
30
+ InvalidObject,
31
+ InvalidPath,
32
+ InvalidType,
33
+ ReplicaIsLocked,
34
+ ReplicaNotFound,
35
+ ResourceTemporaryUnavailable,
36
+ RSENotFound,
37
+ ScopeNotFound,
38
+ SortingAlgorithmNotSupported,
39
+ )
40
+ from rucio.common.utils import APIEncoder, parse_response, render_json
41
+ from rucio.core.replica_sorter import sort_replicas
42
+ from rucio.db.sqla.constants import BadFilesStatus
43
+ from rucio.gateway.quarantined_replica import quarantine_file_replicas
44
+ from rucio.gateway.replica import (
45
+ add_bad_dids,
46
+ add_bad_pfns,
47
+ add_replicas,
48
+ declare_bad_file_replicas,
49
+ declare_suspicious_file_replicas,
50
+ delete_replicas,
51
+ get_bad_replicas_summary,
52
+ get_did_from_pfns,
53
+ get_suspicious_files,
54
+ list_bad_replicas_status,
55
+ list_dataset_replicas,
56
+ list_dataset_replicas_bulk,
57
+ list_dataset_replicas_vp,
58
+ list_datasets_per_rse,
59
+ list_replicas,
60
+ set_tombstone,
61
+ update_replicas_states,
62
+ )
63
+ from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
64
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
65
+
66
+
67
+ def _sorted_with_priorities(replicas, sorted_pfns, limit=None):
68
+ """
69
+ Pick up to "limit" replicas from "replicas" in the order given by sorted_pfns.
70
+ Sets the corresponding priority in returned replicas.
71
+
72
+ :param replicas: Dictionary {pfn: replica_definition}.
73
+ :param sorted_pfns: Sorted list of pfns.
74
+ :param limit: only return this many replicas
75
+ :yields: index and corresponding pfn
76
+ """
77
+ for idx, pfn in enumerate(sorted_pfns, start=1):
78
+ if limit is None or idx <= limit:
79
+ replica = replicas[pfn]
80
+ replica['priority'] = idx
81
+ yield pfn, replica
82
+
83
+
84
+ def _generate_one_metalink_file(rfile, policy_schema, detailed_url=True):
85
+ yield ' <file name="' + rfile['name'] + '">\n'
86
+
87
+ if 'parents' in rfile and rfile['parents']:
88
+ yield ' <parents>\n'
89
+ for parent in rfile['parents']:
90
+ yield ' <did>' + parent + '</did>\n'
91
+ yield ' </parents>\n'
92
+
93
+ yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
94
+ if rfile['adler32'] is not None:
95
+ yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
96
+ if rfile['md5'] is not None:
97
+ yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
98
+
99
+ yield ' <size>' + str(rfile['bytes']) + '</size>\n'
100
+
101
+ yield f' <glfn name="/{policy_schema}/rucio/{rfile["scope"]}:{rfile["name"]}"></glfn>\n'
102
+
103
+ for pfn, replica in rfile['pfns'].items():
104
+ if detailed_url:
105
+ yield (
106
+ ' '
107
+ f'<url location="{replica["rse"]}"'
108
+ f' domain="{replica["domain"]}"'
109
+ f' priority="{replica["priority"]}"'
110
+ f' client_extract="{str(replica["client_extract"]).lower()}"'
111
+ f'>{escape(pfn)}</url>\n'
112
+ )
113
+ else:
114
+ yield (
115
+ ' '
116
+ f'<url location="{replica["rse"]}"'
117
+ f' priority="{replica["priority"]}"'
118
+ f'>{escape(pfn)}</url>\n'
119
+ )
120
+ yield ' </file>\n'
121
+
122
+
123
+ def _generate_metalink_response(rfiles, policy_schema, detailed_url=True):
124
+ first = True
125
+ for rfile in rfiles:
126
+ if first:
127
+ # first, set the appropriate content type, and stream the header
128
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
129
+ first = False
130
+
131
+ yield from _generate_one_metalink_file(rfile, policy_schema=policy_schema, detailed_url=detailed_url)
132
+
133
+ if first:
134
+ # if still first output, i.e. there were no replicas
135
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
136
+ else:
137
+ # don't forget to send the metalink footer
138
+ yield '</metalink>\n'
139
+
140
+
141
+ def _generate_json_response(rfiles):
142
+ for rfile in rfiles:
143
+ yield dumps(rfile) + '\n'
144
+
145
+
146
+ class Replicas(ErrorHandlingMethodView):
147
+
148
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
149
+ def get(self, scope_name):
150
+ """
151
+ ---
152
+ summary: Get Replicas
153
+ description: List all replicas for data identifiers.
154
+ tags:
155
+ - Replicas
156
+ parameters:
157
+ - name: scope_name
158
+ in: path
159
+ description: The DID associated with the replicas.
160
+ schema:
161
+ type: string
162
+ style: simple
163
+ - name: X-Forwarded-For
164
+ in: header
165
+ description: The client ip
166
+ schema:
167
+ type: string
168
+ - name: schemes
169
+ in: query
170
+ description: The schemes of the replicas.
171
+ schema:
172
+ type: string
173
+ - name: select
174
+ in: query
175
+ description: The sorting algorithm.
176
+ schema:
177
+ type: string
178
+ enum: ["geoip", "closeness", "dynamic", "ranking", "random"]
179
+ - name: limit
180
+ in: query
181
+ description: The maximum number of replicas returned.
182
+ schema:
183
+ type: integer
184
+ responses:
185
+ 200:
186
+ description: OK
187
+ content:
188
+ application/x-json-stream:
189
+ schema:
190
+ description: A list with all replicas.
191
+ type: array
192
+ items:
193
+ description: A replica. Possibly contains more information.
194
+ type: object
195
+ properties:
196
+ scope:
197
+ description: The scope of the replica.
198
+ type: string
199
+ name:
200
+ description: The name of the replica.
201
+ type: string
202
+ bytes:
203
+ description: The size of the replica in bytes.
204
+ type: integer
205
+ md5:
206
+ description: The md5 checksum of the replica.
207
+ type: string
208
+ adler32:
209
+ description: The adler32 checksum of the replica.
210
+ type: string
211
+ pfns:
212
+ description: The pfns associated with the replica.
213
+ type: array
214
+ rses:
215
+ description: The rse associated with the replica.
216
+ type: string
217
+ 401:
218
+ description: Invalid Auth Token
219
+ 404:
220
+ description: Did not found
221
+ 406:
222
+ description: Not acceptable
223
+ """
224
+ try:
225
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
226
+ except ValueError as error:
227
+ return generate_http_error_flask(400, error)
228
+
229
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
230
+ metalink = (content_type == 'application/metalink4+xml')
231
+ dids = [{'scope': scope, 'name': name}]
232
+ select, limit = None, None
233
+
234
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
235
+ client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
236
+
237
+ schemes = request.args.get('schemes', default=None)
238
+ select = request.args.get('select', default=None)
239
+ limit = request.args.get('limit', default=None)
240
+ if limit:
241
+ limit = int(limit)
242
+
243
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
244
+ if metalink and schemes is None:
245
+ schemes = SUPPORTED_PROTOCOLS
246
+
247
+ try:
248
+ def _list_and_sort_replicas(vo):
249
+ # we need to call list_replicas before starting to reply
250
+ # otherwise the exceptions won't be propagated correctly
251
+ for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
252
+ replicas = []
253
+ dictreplica = {}
254
+ for rse in rfile['rses']:
255
+ for replica in rfile['rses'][rse]:
256
+ replicas.append(replica)
257
+ dictreplica[replica] = rse
258
+
259
+ replicas = sort_replicas(dictreplica, client_location, selection=select)
260
+ rfile['pfns'] = dict(_sorted_with_priorities(rfile['pfns'], replicas, limit=limit))
261
+ yield rfile
262
+
263
+ rfiles = _list_and_sort_replicas(vo=request.environ.get('vo'))
264
+ if metalink:
265
+ response_generator = _generate_metalink_response(rfiles, 'atlas', detailed_url=False)
266
+ else:
267
+ response_generator = _generate_json_response(rfiles)
268
+ return try_stream(response_generator, content_type=content_type)
269
+ except (DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
270
+ return generate_http_error_flask(404, error)
271
+
272
+ def post(self):
273
+ """
274
+ ---
275
+ summary: Create File Replicas
276
+ description: Create file replicas at a given RSE.
277
+ tags:
278
+ - Replicas
279
+ requestBody:
280
+ content:
281
+ application/json:
282
+ schema:
283
+ type: object
284
+ required:
285
+ - rse
286
+ - files
287
+ properties:
288
+ rse:
289
+ description: The rse for the replication
290
+ type: string
291
+ files:
292
+ description: The files to replicate
293
+ type: array
294
+ items:
295
+ type: object
296
+ required:
297
+ - pfn
298
+ - bytes
299
+ - name
300
+ properties:
301
+ pfn:
302
+ description: The pfn of the replica.
303
+ type: string
304
+ name:
305
+ description: The DID name.
306
+ type: string
307
+ bytes:
308
+ description: The size of the replica in bytes.
309
+ type: integer
310
+ state:
311
+ description: The state of the replica.
312
+ type: string
313
+ path:
314
+ description: The path of the new replica.
315
+ type: string
316
+ md5:
317
+ description: The md5 checksum.
318
+ type: string
319
+ adler32:
320
+ description: The adler32 checksum.
321
+ type: string
322
+ lcok_cnt:
323
+ description: The lock count.
324
+ type: integer
325
+ tombstone:
326
+ description: The tombstone.
327
+ type: string
328
+ ignore_availability:
329
+ description: The ignore availability.
330
+ type: boolean
331
+ responses:
332
+ 201:
333
+ description: OK
334
+ content:
335
+ application/json:
336
+ schema:
337
+ type: string
338
+ enum: ["Created"]
339
+ 400:
340
+ description: Invalid Path
341
+ 401:
342
+ description: Invalid Auth Token
343
+ 404:
344
+ description: Rse or scope not found
345
+ 409:
346
+ description: Replica or Did already exists
347
+ 503:
348
+ description: Resource temporary unavailable
349
+ """
350
+ parameters = json_parameters(parse_response)
351
+ rse = param_get(parameters, 'rse')
352
+ files = param_get(parameters, 'files')
353
+
354
+ try:
355
+ add_replicas(
356
+ rse=rse,
357
+ files=files,
358
+ issuer=request.environ.get('issuer'),
359
+ vo=request.environ.get('vo'),
360
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
361
+ )
362
+ except InvalidPath as error:
363
+ return generate_http_error_flask(400, error)
364
+ except AccessDenied as error:
365
+ return generate_http_error_flask(401, error)
366
+ except (Duplicate, DataIdentifierAlreadyExists) as error:
367
+ return generate_http_error_flask(409, error)
368
+ except (RSENotFound, ScopeNotFound) as error:
369
+ return generate_http_error_flask(404, error)
370
+ except ResourceTemporaryUnavailable as error:
371
+ return generate_http_error_flask(503, error)
372
+
373
+ return 'Created', 201
374
+
375
+ def put(self):
376
+ """
377
+ ---
378
+ summary: Update File Replicas
379
+ description: Update file replicas state at a given RSE.
380
+ tags:
381
+ - Replicas
382
+ requestBody:
383
+ content:
384
+ application/json:
385
+ schema:
386
+ type: object
387
+ required:
388
+ - rse
389
+ - files
390
+ properties:
391
+ rse:
392
+ description: The rse for the replication
393
+ type: string
394
+ files:
395
+ description: The files to replicate
396
+ type: array
397
+ items:
398
+ type: object
399
+ properties:
400
+ name:
401
+ description: The pfn of the replica.
402
+ type: string
403
+ state:
404
+ description: The pfn of the replica.
405
+ type: string
406
+ path:
407
+ description: The pfn of the replica.
408
+ type: string
409
+ error_message:
410
+ description: The error message if an error occurred.
411
+ type: string
412
+ broken_rule_id:
413
+ description: The id of the broken rule if one was found.
414
+ type: string
415
+ broken_message:
416
+ description: The message of the broken rule.
417
+ type: string
418
+ responses:
419
+ 200:
420
+ description: OK
421
+ 400:
422
+ description: Cannot decode json parameter list
423
+ 401:
424
+ description: Invalid Auth Token
425
+ """
426
+ parameters = json_parameters(parse_response)
427
+ rse = param_get(parameters, 'rse')
428
+ files = param_get(parameters, 'files')
429
+
430
+ try:
431
+ update_replicas_states(rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
432
+ except AccessDenied as error:
433
+ return generate_http_error_flask(401, error)
434
+
435
+ return '', 200
436
+
437
+ def delete(self):
438
+ """
439
+ ---
440
+ summary: Delete File Replicas
441
+ description: Delete file replicas at a given RSE.
442
+ tags:
443
+ - Replicas
444
+ requestBody:
445
+ content:
446
+ application/json:
447
+ schema:
448
+ type: object
449
+ required:
450
+ - rse
451
+ - files
452
+ properties:
453
+ rse:
454
+ description: The rse name.
455
+ type: string
456
+ files:
457
+ description: The files to delete.
458
+ type: array
459
+ items:
460
+ type: object
461
+ required:
462
+ - name
463
+ properties:
464
+ name:
465
+ description: The name of the replica.
466
+ type: string
467
+ responses:
468
+ 200:
469
+ description: OK
470
+ 400:
471
+ description: Cannot decode json parameter list.
472
+ 401:
473
+ description: Invalid Auth Token
474
+ 404:
475
+ description: Rse or Replica not found
476
+ """
477
+ parameters = json_parameters(parse_response)
478
+ rse = param_get(parameters, 'rse')
479
+ files = param_get(parameters, 'files')
480
+
481
+ try:
482
+ delete_replicas(
483
+ rse=rse,
484
+ files=files,
485
+ issuer=request.environ.get('issuer'),
486
+ vo=request.environ.get('vo'),
487
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
488
+ )
489
+ except AccessDenied as error:
490
+ return generate_http_error_flask(401, error)
491
+ except (RSENotFound, ReplicaNotFound) as error:
492
+ return generate_http_error_flask(404, error)
493
+ except ResourceTemporaryUnavailable as error:
494
+ return generate_http_error_flask(503, error)
495
+
496
+ return '', 200
497
+
498
+
499
+ class ListReplicas(ErrorHandlingMethodView):
500
+
501
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
502
+ def post(self):
503
+ """
504
+ ---
505
+ summary: List Replicas
506
+ description: List all replicas for a DID.
507
+ tags:
508
+ - Replicas
509
+ parameters:
510
+ - name: X-Forwarded-For
511
+ in: header
512
+ description: The client ip address.
513
+ schema:
514
+ type: string
515
+ - name: limit
516
+ in: query
517
+ description: The maximum number pfns per replica to return.
518
+ schema:
519
+ type: integer
520
+ - name: select
521
+ in: query
522
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
523
+ schema:
524
+ type: string
525
+ - name: sort
526
+ in: query
527
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
528
+ schema:
529
+ type: string
530
+ requestBody:
531
+ content:
532
+ application/json:
533
+ schema:
534
+ type: object
535
+ properties:
536
+ client_location:
537
+ description: The clients location.
538
+ type: string
539
+ dids:
540
+ description: List of Dids.
541
+ type: array
542
+ items:
543
+ type: object
544
+ properties:
545
+ scope:
546
+ description: The scope of the did.
547
+ type: string
548
+ name:
549
+ description: The name of the did.
550
+ type: string
551
+ schemes:
552
+ description: A list of schemes to filter the replicas.
553
+ type: array
554
+ items:
555
+ type: string
556
+ sort:
557
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
558
+ type: string
559
+ unavailable:
560
+ description: If unavailable rse should be considered.
561
+ type: boolean
562
+ deprecated: true
563
+ ignore_availability:
564
+ description: If the availability should be ignored.
565
+ type: boolean
566
+ rse_expression:
567
+ description: The RSE expression to restrict on a list of RSEs.
568
+ type: string
569
+ all_states:
570
+ description: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
571
+ type: boolean
572
+ domain:
573
+ description: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
574
+ type: string
575
+ signature_lifetime:
576
+ description: If supported, in seconds, restrict the lifetime of the signed PFN.
577
+ type: integer
578
+ resolve_archives:
579
+ description: When set to True, find archives which contain the replicas.
580
+ type: boolean
581
+ resolve_parents:
582
+ description: When set to True, find all parent datasets which contain the replicas.
583
+ type: boolean
584
+ updated_after:
585
+ description: datetime object (UTC time), only return replicas updated after this time
586
+ type: string
587
+ nrandom:
588
+ description: The maximum number of replicas to return.
589
+ type: integer
590
+ responses:
591
+ 200:
592
+ description: OK
593
+ content:
594
+ application/json:
595
+ schema:
596
+ type: array
597
+ items:
598
+ type: object
599
+ properties:
600
+ scope:
601
+ description: The scope of the replica.
602
+ type: string
603
+ name:
604
+ description: The name of the replica.
605
+ type: string
606
+ bytes:
607
+ description: The size of the replica in bytes.
608
+ type: integer
609
+ md5:
610
+ description: The md5 checksum.
611
+ type: string
612
+ adler32:
613
+ description: The adler32 checksum.
614
+ type: string
615
+ pfns:
616
+ description: The pfns.
617
+ type: array
618
+ rses:
619
+ description: The RSESs.
620
+ type: array
621
+ application/metalink4+xml:
622
+ schema:
623
+ type: object
624
+ properties:
625
+ scope:
626
+ description: The scope of the replica.
627
+ type: string
628
+ name:
629
+ description: The name of the replica.
630
+ type: string
631
+ bytes:
632
+ description: The size of the replica in bytes.
633
+ type: integer
634
+ md5:
635
+ description: The md5 checksum.
636
+ type: string
637
+ adler32:
638
+ description: The adler32 checksum.
639
+ type: string
640
+ pfns:
641
+ description: The pfns.
642
+ type: array
643
+ rses:
644
+ description: The RSESs.
645
+ type: array
646
+ 400:
647
+ description: Cannot decode json parameter list.
648
+ 401:
649
+ description: Invalid Auth Token
650
+ 404:
651
+ description: Did not found.
652
+ 406:
653
+ description: Not acceptable
654
+ """
655
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
656
+ metalink = (content_type == 'application/metalink4+xml')
657
+
658
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
659
+
660
+ parameters = json_parameters(parse_response)
661
+
662
+ client_location = {'ip': client_ip,
663
+ 'fqdn': None,
664
+ 'site': None}
665
+ client_location.update(param_get(parameters, 'client_location', default={}))
666
+
667
+ # making sure IP address is not overwritten
668
+ client_location['ip'] = client_ip
669
+
670
+ dids = param_get(parameters, 'dids', default=[])
671
+ schemes = param_get(parameters, 'schemes', default=None)
672
+ select = param_get(parameters, 'sort', default=None)
673
+ unavailable = param_get(parameters, 'unavailable', default=False)
674
+ ignore_availability = param_get(parameters, 'ignore_availability', default='unavailable' in parameters)
675
+ rse_expression = param_get(parameters, 'rse_expression', default=None)
676
+ all_states = param_get(parameters, 'all_states', default=False)
677
+ domain = param_get(parameters, 'domain', default=None)
678
+ if 'signature_lifetime' in parameters:
679
+ signature_lifetime = param_get(parameters, 'signature_lifetime')
680
+ else:
681
+ # hardcoded default of 10 minutes if config is not parseable
682
+ signature_lifetime = config_get_int('credentials', 'signature_lifetime', raise_exception=False, default=600)
683
+ resolve_archives = param_get(parameters, 'resolve_archives', default=True)
684
+ resolve_parents = param_get(parameters, 'resolve_parents', default=False)
685
+ updated_after = param_get(parameters, 'updated_after', default=None)
686
+ if updated_after is not None:
687
+ if isinstance(updated_after, (int, float)):
688
+ # convert from epoch time stamp to datetime object
689
+ updated_after = datetime.utcfromtimestamp(updated_after)
690
+ else:
691
+ # attempt UTC format '%Y-%m-%dT%H:%M:%S' conversion
692
+ updated_after = datetime.strptime(updated_after, '%Y-%m-%dT%H:%M:%S')
693
+ nrandom = param_get(parameters, 'nrandom', default=None)
694
+ if nrandom:
695
+ nrandom = int(nrandom)
696
+
697
+ limit = request.args.get('limit', default=None)
698
+ select = request.args.get('select', default=select)
699
+ select = request.args.get('sort', default=select)
700
+
701
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
702
+ if metalink and schemes is None:
703
+ schemes = SUPPORTED_PROTOCOLS
704
+
705
+ content_type = 'application/metalink4+xml' if metalink else 'application/x-json-stream'
706
+
707
+ try:
708
+ def _list_and_sort_replicas(request_id, issuer, vo):
709
+ # we need to call list_replicas before starting to reply
710
+ # otherwise the exceptions won't be propagated correctly
711
+ for rfile in list_replicas(dids=dids, schemes=schemes,
712
+ unavailable=unavailable,
713
+ request_id=request_id,
714
+ ignore_availability=ignore_availability,
715
+ all_states=all_states,
716
+ rse_expression=rse_expression,
717
+ client_location=client_location,
718
+ domain=domain,
719
+ signature_lifetime=signature_lifetime,
720
+ resolve_archives=resolve_archives,
721
+ resolve_parents=resolve_parents,
722
+ nrandom=nrandom,
723
+ updated_after=updated_after,
724
+ issuer=issuer,
725
+ vo=vo):
726
+
727
+ # Sort rfile['pfns'] and limit its size according to "limit" parameter
728
+ lanreplicas = {}
729
+ wanreplicas = {}
730
+ for pfn, replica in rfile['pfns'].items():
731
+ replica_tuple = (replica['domain'], replica['priority'], replica['rse'], replica['client_extract'])
732
+ if replica_tuple[0] == 'lan':
733
+ lanreplicas[pfn] = replica_tuple
734
+ else:
735
+ wanreplicas[pfn] = replica_tuple
736
+
737
+ rfile['pfns'] = dict(_sorted_with_priorities(replicas=rfile['pfns'],
738
+ # Lan replicas sorted by priority; followed by wan replicas sorted by selection criteria
739
+ sorted_pfns=chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
740
+ sort_replicas(wanreplicas, client_location, selection=select)),
741
+ limit=limit))
742
+ yield rfile
743
+
744
+ rfiles = _list_and_sort_replicas(request_id=request.environ.get('request_id'),
745
+ issuer=request.environ.get('issuer'),
746
+ vo=request.environ.get('vo'))
747
+ if metalink:
748
+ policy_schema = config_get('policy', 'schema', raise_exception=False, default='generic')
749
+ response_generator = _generate_metalink_response(rfiles, policy_schema)
750
+ else:
751
+ response_generator = _generate_json_response(rfiles)
752
+ return try_stream(response_generator, content_type=content_type)
753
+ except (InvalidObject, DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
754
+ return generate_http_error_flask(400, error)
755
+
756
+
757
+ class ReplicasDIDs(ErrorHandlingMethodView):
758
+
759
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
760
+ def post(self):
761
+ """
762
+ ---
763
+ summary: List Replicas Dids
764
+ description: List the DIDs associated to a list of replicas.
765
+ tags:
766
+ - Replicas
767
+ requestBody:
768
+ content:
769
+ application/json:
770
+ schema:
771
+ type: object
772
+ required:
773
+ - rse
774
+ properties:
775
+ pfns:
776
+ description: The list of pfns.
777
+ type: array
778
+ items:
779
+ type: string
780
+ rse:
781
+ description: The RSE name.
782
+ type: string
783
+ responses:
784
+ 200:
785
+ description: OK
786
+ content:
787
+ application/x-json-stream:
788
+ schema:
789
+ type: array
790
+ items:
791
+ type: object
792
+ additionalProperties:
793
+ x-additionalPropertiesName: mapped PFNs to DIDs
794
+ description: A mapping from a pfn to a did.
795
+ type: object
796
+ properties:
797
+ scope:
798
+ description: The scope of the DID.
799
+ type: string
800
+ name:
801
+ description: The name of the DID.
802
+ type: string
803
+ 400:
804
+ description: Cannot decode json parameter list.
805
+ 401:
806
+ description: Invalid Auth Token
807
+ 404:
808
+ description: Not found
809
+ 406:
810
+ description: Not acceptable
811
+ """
812
+ parameters = json_parameters()
813
+ pfns = param_get(parameters, 'pfns', default=[])
814
+ rse = param_get(parameters, 'rse')
815
+
816
+ try:
817
+ def generate(vo):
818
+ for pfn in get_did_from_pfns(pfns, rse, vo=vo):
819
+ yield dumps(pfn) + '\n'
820
+
821
+ return try_stream(generate(vo=request.environ.get('vo')))
822
+ except AccessDenied as error:
823
+ return generate_http_error_flask(401, error)
824
+
825
+
826
+ class BadReplicas(ErrorHandlingMethodView):
827
+
828
+ @check_accept_header_wrapper_flask(['application/json'])
829
+ def post(self):
830
+ """
831
+ ---
832
+ summary: Declare Bad Replicas
833
+ description: Declares a list of bad replicas.
834
+ tags:
835
+ - Replicas
836
+ requestBody:
837
+ content:
838
+ application/json:
839
+ schema:
840
+ type: object
841
+ properties:
842
+ replicas:
843
+ description: The list of pfns or list of dicts with "scope", "name", "rse_id"/"rse"
844
+ type: array
845
+ items:
846
+ type: string
847
+ pfns:
848
+ deprecated: true
849
+ description: The list of pfns, for backward compatibility with older versions of the ReplicaClient
850
+ type: array
851
+ items:
852
+ type: string
853
+ reason:
854
+ description: The reason for the declaration.
855
+ type: string
856
+ force:
857
+ description: If true, ignore existing replica status in the bad_replicas table.
858
+ type: boolean
859
+ responses:
860
+ 201:
861
+ description: OK
862
+ content:
863
+ application/json:
864
+ schema:
865
+ description: Returns the not declared files.
866
+ type: array
867
+ 400:
868
+ description: Can not decode json parameter list.
869
+ 404:
870
+ description: Not found
871
+ 406:
872
+ description: Not acceptable
873
+ """
874
+ parameters = json_parameters()
875
+ replicas = param_get(parameters, 'replicas', default=[]) or param_get(parameters, 'pfns', default=[])
876
+ reason = param_get(parameters, 'reason', default=None)
877
+ force = param_get(parameters, 'force', default=False)
878
+
879
+ try:
880
+ not_declared_files = declare_bad_file_replicas(replicas, reason=reason,
881
+ issuer=request.environ.get('issuer'), vo=request.environ.get('vo'),
882
+ force=force)
883
+ return not_declared_files, 201
884
+ except AccessDenied as error:
885
+ return generate_http_error_flask(401, error)
886
+ except (RSENotFound, ReplicaNotFound) as error:
887
+ return generate_http_error_flask(404, error)
888
+
889
+
890
+ class QuarantineReplicas(ErrorHandlingMethodView):
891
+
892
+ def post(self):
893
+ """
894
+ ---
895
+ summary: Quarantine replicas
896
+ description: Quarantine replicas.
897
+ tags:
898
+ - Replicas
899
+ requestBody:
900
+ content:
901
+ application/json:
902
+ schema:
903
+ type: object
904
+ required:
905
+ - replicas
906
+ properties:
907
+ replicas:
908
+ description: replicas
909
+ type: array
910
+ items:
911
+ type: object
912
+ required:
913
+ - path
914
+ properties:
915
+ path:
916
+ description: path
917
+ type: string
918
+ scope:
919
+ description: scope
920
+ type: string
921
+ name:
922
+ description: name
923
+ type: string
924
+ rse:
925
+ description: RSE name
926
+ type: string
927
+ rse_id:
928
+ description: RSE id
929
+ type: string
930
+ responses:
931
+ 200:
932
+ description: OK
933
+ 403:
934
+ description: Forbidden.
935
+ 404:
936
+ description: Not found
937
+ """
938
+
939
+ parameters = json_parameters()
940
+ replicas = param_get(parameters, 'replicas', default=[])
941
+ rse = param_get(parameters, 'rse', default=None)
942
+ rse_id = param_get(parameters, 'rse_id', default=None)
943
+ vo = request.environ.get('vo')
944
+ issuer = request.environ.get('issuer')
945
+
946
+ if replicas:
947
+ try:
948
+ quarantine_file_replicas(replicas, issuer, rse=rse, rse_id=rse_id, vo=vo)
949
+ except AccessDenied as error:
950
+ return generate_http_error_flask(403, error)
951
+ except (RSENotFound, ReplicaNotFound) as error:
952
+ return generate_http_error_flask(404, error)
953
+
954
+ return '', 200
955
+
956
+
957
+ class SuspiciousReplicas(ErrorHandlingMethodView):
958
+
959
+ @check_accept_header_wrapper_flask(['application/json'])
960
+ def post(self):
961
+ """
962
+ ---
963
+ summary: Declare Suspicious Replicas
964
+ description: Declare a list of suspicious replicas.
965
+ tags:
966
+ - Replicas
967
+ requestBody:
968
+ content:
969
+ application/json:
970
+ schema:
971
+ type: object
972
+ properties:
973
+ pfns:
974
+ description: The list of pfns.
975
+ type: array
976
+ items:
977
+ type: string
978
+ reason:
979
+ description: The reason for the declaration.
980
+ type: string
981
+ responses:
982
+ 201:
983
+ description: OK
984
+ content:
985
+ application/json:
986
+ schema:
987
+ description: Returns the not declared files.
988
+ type: array
989
+ 400:
990
+ description: Can not decode json parameter list.
991
+ 404:
992
+ description: Not found
993
+ 406:
994
+ description: Not acceptable
995
+ """
996
+ parameters = json_parameters(parse_response)
997
+ pfns = param_get(parameters, 'pfns', default=[])
998
+ reason = param_get(parameters, 'reason', default=None)
999
+
1000
+ try:
1001
+ not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
1002
+ return not_declared_files, 201
1003
+ except AccessDenied as error:
1004
+ return generate_http_error_flask(401, error)
1005
+
1006
+ @check_accept_header_wrapper_flask(['application/json'])
1007
+ def get(self):
1008
+ """
1009
+ ---
1010
+ summary: List Suspicious Replicas
1011
+ description: List the suspicious replicas on a list of RSEs.
1012
+ tags:
1013
+ - Replicas
1014
+ parameters:
1015
+ - name: rse_expression
1016
+ in: query
1017
+ description: The RSE expression to filter for.
1018
+ schema:
1019
+ type: string
1020
+ - name: younger_than
1021
+ in: query
1022
+ description: Date to filter for.
1023
+ schema:
1024
+ type: string
1025
+ - name: nattempts
1026
+ in: query
1027
+ description: The maximum number of attempts to make.
1028
+ schema:
1029
+ type: integer
1030
+ responses:
1031
+ 200:
1032
+ description: OK
1033
+ content:
1034
+ application/json:
1035
+ schema:
1036
+ type: array
1037
+ items:
1038
+ type: object
1039
+ properties:
1040
+ scope:
1041
+ description: The scope of the Replica.
1042
+ type: string
1043
+ name:
1044
+ description: The name of the Replica.
1045
+ type: string
1046
+ rse:
1047
+ description: The rse name.
1048
+ type: string
1049
+ rse_id:
1050
+ description: The id of the rse.
1051
+ type: string
1052
+ cnt:
1053
+ description: The number of replicas.
1054
+ type: integer
1055
+ created_at:
1056
+ description: The time when the replica was created.
1057
+ type: string
1058
+ 401:
1059
+ description: Invalid Auth Token
1060
+ 404:
1061
+ description: Not found
1062
+ 406:
1063
+ description: Not acceptable
1064
+ """
1065
+ rse_expression, younger_than, nattempts = None, None, None
1066
+ if request.query_string:
1067
+ query_string = request.query_string.decode(encoding='utf-8')
1068
+ try:
1069
+ params = loads(unquote(query_string))
1070
+ except ValueError:
1071
+ params = parse_qs(query_string)
1072
+
1073
+ if 'rse_expression' in params:
1074
+ rse_expression = params['rse_expression'][0]
1075
+ if 'younger_than' in params and params['younger_than'][0]:
1076
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S")
1077
+ if 'nattempts' in params:
1078
+ nattempts = int(params['nattempts'][0])
1079
+
1080
+ result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
1081
+ return Response(render_json(result), 200, content_type='application/json')
1082
+
1083
+
1084
+ class BadReplicasStates(ErrorHandlingMethodView):
1085
+
1086
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1087
+ def get(self):
1088
+ """
1089
+ ---
1090
+ summary: List Bad Replicas By States
1091
+ description: List the bad or suspicious replicas by states.
1092
+ tags:
1093
+ - Replicas
1094
+ parameters:
1095
+ - name: state
1096
+ in: query
1097
+ description: The state of the file.
1098
+ schema:
1099
+ type: string
1100
+ enum: [SUSPICIOUS, BAD]
1101
+ - name: rse
1102
+ in: query
1103
+ description: The rse name.
1104
+ schema:
1105
+ type: string
1106
+ - name: younger_than
1107
+ in: query
1108
+ description: Date to select bad replicas younger than this date.
1109
+ schema:
1110
+ type: string
1111
+ format: date-time
1112
+ - name: older_than
1113
+ in: query
1114
+ description: Date to select bad replicas older than this date.
1115
+ schema:
1116
+ type: string
1117
+ format: date-time
1118
+ - name: limit
1119
+ in: query
1120
+ description: The maximum number of replicas returned.
1121
+ schema:
1122
+ type: integer
1123
+ - name: list_pfns
1124
+ in: query
1125
+ description: Flag to include pfns.
1126
+ schema:
1127
+ type: boolean
1128
+ responses:
1129
+ 200:
1130
+ description: OK
1131
+ content:
1132
+ application/x-json-stream:
1133
+ schema:
1134
+ description: A list of all result replicas.
1135
+ type: array
1136
+ items:
1137
+ oneOf:
1138
+ - type: object
1139
+ properties:
1140
+ scope:
1141
+ description: The scope of the replica.
1142
+ type: string
1143
+ name:
1144
+ description: The name of the replica.
1145
+ type: string
1146
+ type:
1147
+ description: The type of the replica.
1148
+ type: string
1149
+ - type: object
1150
+ properties:
1151
+ scope:
1152
+ description: The scope of the replica.
1153
+ type: string
1154
+ name:
1155
+ description: The name of the replica.
1156
+ type: string
1157
+ rse:
1158
+ description: The name of the associated rse.
1159
+ type: string
1160
+ rse_id:
1161
+ description: The id of the associated rse.
1162
+ type: string
1163
+ state:
1164
+ description: The state of the replica.
1165
+ type: string
1166
+ created_at:
1167
+ description: The date-time the replica was created.
1168
+ type: string
1169
+ format: date-time
1170
+ updated_at:
1171
+ description: The date-time the replica was updated.
1172
+ type: string
1173
+ format: date-time
1174
+ 401:
1175
+ description: Invalid Auth Token
1176
+ 406:
1177
+ description: Not acceptable
1178
+ """
1179
+ state, rse, younger_than, older_than, limit, list_pfns = None, None, None, None, None, None
1180
+ if request.query_string:
1181
+ query_string = request.query_string.decode(encoding='utf-8')
1182
+ try:
1183
+ params = loads(unquote(query_string))
1184
+ except ValueError:
1185
+ params = parse_qs(query_string)
1186
+ if 'state' in params:
1187
+ state = params['state'][0]
1188
+ if isinstance(state, str):
1189
+ state = BadFilesStatus(state)
1190
+ if 'rse' in params:
1191
+ rse = params['rse'][0]
1192
+ if 'younger_than' in params:
1193
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1194
+ if 'older_than' in params and params['older_than']:
1195
+ older_than = datetime.strptime(params['older_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1196
+ if 'limit' in params:
1197
+ limit = int(params['limit'][0])
1198
+ if 'list_pfns' in params:
1199
+ list_pfns = bool(params['list_pfns'][0])
1200
+
1201
+ def generate(vo):
1202
+ for row in list_bad_replicas_status(state=state, rse=rse, younger_than=younger_than,
1203
+ older_than=older_than, limit=limit, list_pfns=list_pfns,
1204
+ vo=vo):
1205
+ yield dumps(row, cls=APIEncoder) + '\n'
1206
+
1207
+ return try_stream(generate(vo=request.environ.get('vo')))
1208
+
1209
+
1210
+ class BadReplicasSummary(ErrorHandlingMethodView):
1211
+
1212
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1213
+ def get(self):
1214
+ """
1215
+ ---
1216
+ summary: Bad Replicas Summary
1217
+ description: Return a summary of the bad replicas by incident.
1218
+ tags:
1219
+ - Replicas
1220
+ parameters:
1221
+ - name: rse_expression
1222
+ in: query
1223
+ description: The RSE expression.
1224
+ schema:
1225
+ type: string
1226
+ - name: from_date
1227
+ in: query
1228
+ description: The start date.
1229
+ schema:
1230
+ type: string
1231
+ format: date-time
1232
+ - name: to_date
1233
+ in: query
1234
+ description: The end date.
1235
+ schema:
1236
+ type: string
1237
+ format: date-time
1238
+ responses:
1239
+ 200:
1240
+ description: OK
1241
+ content:
1242
+ application/x-json-stream:
1243
+ schema:
1244
+ description: A list of summaries.
1245
+ type: array
1246
+ items:
1247
+ type: object
1248
+ properties:
1249
+ rse:
1250
+ description: The name of the associated RSE.
1251
+ type: string
1252
+ rse_id:
1253
+ description: The id of the associated RSE.
1254
+ type: string
1255
+ created_at:
1256
+ description: The creation date-time.
1257
+ type: string
1258
+ format: date-time
1259
+ reason:
1260
+ description: The reason for the incident.
1261
+ type: string
1262
+ 401:
1263
+ description: Invalid Auth Token
1264
+ 406:
1265
+ description: Not acceptable
1266
+ """
1267
+ rse_expression, from_date, to_date = None, None, None
1268
+ if request.query_string:
1269
+ query_string = request.query_string.decode(encoding='utf-8')
1270
+ try:
1271
+ params = loads(unquote(query_string))
1272
+ except ValueError:
1273
+ params = parse_qs(query_string)
1274
+ if 'rse_expression' in params:
1275
+ rse_expression = params['rse_expression'][0]
1276
+ if 'from_date' in params and params['from_date'][0]:
1277
+ from_date = datetime.strptime(params['from_date'][0], "%Y-%m-%d")
1278
+ if 'to_date' in params:
1279
+ to_date = datetime.strptime(params['to_date'][0], "%Y-%m-%d")
1280
+
1281
+ def generate(vo):
1282
+ for row in get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date,
1283
+ to_date=to_date, vo=vo):
1284
+ yield dumps(row, cls=APIEncoder) + '\n'
1285
+
1286
+ return try_stream(generate(vo=request.environ.get('vo')))
1287
+
1288
+
1289
+ class DatasetReplicas(ErrorHandlingMethodView):
1290
+
1291
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1292
+ def get(self, scope_name):
1293
+ """
1294
+ ---
1295
+ summary: List Dataset Replicas
1296
+ description: List dataset replicas.
1297
+ tags:
1298
+ - Replicas
1299
+ parameters:
1300
+ - name: scope_name
1301
+ in: path
1302
+ description: data identifier (scope)/(name).
1303
+ schema:
1304
+ type: string
1305
+ style: simple
1306
+ - name: deep
1307
+ in: query
1308
+ description: Flag to ennable lookup at the file level.
1309
+ schema:
1310
+ type: boolean
1311
+ responses:
1312
+ 200:
1313
+ description: OK
1314
+ content:
1315
+ application/x-json-stream:
1316
+ schema:
1317
+ description: A list of dataset replicas.
1318
+ type: array
1319
+ items:
1320
+ type: object
1321
+ properties:
1322
+ scope:
1323
+ description: The scope of the replica.
1324
+ type: string
1325
+ name:
1326
+ description: The name of the replica.
1327
+ type: string
1328
+ rse:
1329
+ description: The name of the associated RSE.
1330
+ type: string
1331
+ rse_id:
1332
+ description: The id of the associated RSE.
1333
+ type: string
1334
+ bytes:
1335
+ description: The size of the replica.
1336
+ type: integer
1337
+ length:
1338
+ description: The length of the replica.
1339
+ type: integer
1340
+ available_bytes:
1341
+ description: The number of available bytes of the replica.
1342
+ type: integer
1343
+ available_length:
1344
+ description: The available length of the replica.
1345
+ type: integer
1346
+ state:
1347
+ description: The state of the replica.
1348
+ type: string
1349
+ created_at:
1350
+ description: The date-time the replica was created.
1351
+ type: string
1352
+ format: date-time
1353
+ updated_at:
1354
+ description: The date-time the replica was updated.
1355
+ type: string
1356
+ format: date-time
1357
+ accessed_at:
1358
+ description: The date-time the replica was accessed.
1359
+ type: string
1360
+ format: date-time
1361
+ 401:
1362
+ description: Invalid Auth Token
1363
+ 404:
1364
+ description: Not found
1365
+ 406:
1366
+ description: Not acceptable
1367
+ """
1368
+ try:
1369
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1370
+
1371
+ def generate(_deep, vo):
1372
+ for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
1373
+ yield dumps(row, cls=APIEncoder) + '\n'
1374
+
1375
+ deep = request.args.get('deep', default=False)
1376
+
1377
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1378
+ except ValueError as error:
1379
+ return generate_http_error_flask(400, error)
1380
+
1381
+
1382
+ class DatasetReplicasBulk(ErrorHandlingMethodView):
1383
+
1384
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1385
+ def post(self):
1386
+ """
1387
+ ---
1388
+ summary: List Dataset Replicas for Multiple DIDs
1389
+ description: List dataset replicas for multiple dids.
1390
+ tags:
1391
+ - Replicas
1392
+ requestBody:
1393
+ content:
1394
+ application/json:
1395
+ schema:
1396
+ type: object
1397
+ required:
1398
+ - dids
1399
+ properties:
1400
+ dids:
1401
+ description: A list of dids.
1402
+ type: array
1403
+ items:
1404
+ description: A did.
1405
+ type: object
1406
+ properties:
1407
+ scope:
1408
+ description: The scope of the did.
1409
+ type: string
1410
+ name:
1411
+ description: The name of the did.
1412
+ type: string
1413
+ responses:
1414
+ 200:
1415
+ description: OK
1416
+ content:
1417
+ application/x-json-stream:
1418
+ schema:
1419
+ description: A list of dataset replicas.
1420
+ type: array
1421
+ items:
1422
+ type: object
1423
+ properties:
1424
+ scope:
1425
+ description: The scope of the replica.
1426
+ type: string
1427
+ name:
1428
+ description: The name of the replica.
1429
+ type: string
1430
+ rse:
1431
+ description: The name of the associated RSE.
1432
+ type: string
1433
+ rse_id:
1434
+ description: The id of the associated RSE.
1435
+ type: string
1436
+ bytes:
1437
+ description: The size of the replica.
1438
+ type: integer
1439
+ length:
1440
+ description: The length of the replica.
1441
+ type: integer
1442
+ available_bytes:
1443
+ description: The number of available bytes of the replica.
1444
+ type: integer
1445
+ available_length:
1446
+ description: The available length of the replica.
1447
+ type: integer
1448
+ state:
1449
+ description: The state of the replica.
1450
+ type: string
1451
+ created_at:
1452
+ description: The date-time the replica was created.
1453
+ type: string
1454
+ format: date-time
1455
+ updated_at:
1456
+ description: The date-time the replica was updated.
1457
+ type: string
1458
+ format: date-time
1459
+ accessed_at:
1460
+ description: The date-time the replica was accessed.
1461
+ type: string
1462
+ format: date-time
1463
+ 400:
1464
+ description: Bad Request.
1465
+ 401:
1466
+ description: Invalid Auth Token
1467
+ 404:
1468
+ description: Not found
1469
+ 406:
1470
+ description: Not acceptable
1471
+ """
1472
+ parameters = json_parameters(parse_response)
1473
+ dids = param_get(parameters, 'dids')
1474
+ if len(dids) == 0:
1475
+ return generate_http_error_flask(400, ValueError.__name__, 'List of DIDs is empty')
1476
+
1477
+ try:
1478
+ def generate(vo):
1479
+ for row in list_dataset_replicas_bulk(dids=dids, vo=vo):
1480
+ yield dumps(row, cls=APIEncoder) + '\n'
1481
+
1482
+ return try_stream(generate(vo=request.environ.get('vo')))
1483
+ except InvalidObject as error:
1484
+ return generate_http_error_flask(400, error, f'Cannot validate DIDs: {error}')
1485
+
1486
+
1487
+ class DatasetReplicasVP(ErrorHandlingMethodView):
1488
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1489
+ def get(self, scope_name):
1490
+ """
1491
+ ---
1492
+ summary: List Dataset Replicas VP
1493
+ description: |
1494
+ List dataset replicas using the Virtual Placement service.
1495
+ This is an RnD function and might change or go away at any time.
1496
+ tags:
1497
+ - Replicas
1498
+ parameters:
1499
+ - name: scope_name
1500
+ in: path
1501
+ description: data identifier (scope)/(name).
1502
+ schema:
1503
+ type: string
1504
+ style: simple
1505
+ - name: deep
1506
+ in: query
1507
+ description: Flag to ennable lookup at the file level.
1508
+ schema:
1509
+ type: boolean
1510
+ responses:
1511
+ 200:
1512
+ description: OK. This needs documentation!
1513
+ 401:
1514
+ description: Invalid Auth Token
1515
+ 406:
1516
+ description: Not acceptable
1517
+ """
1518
+ try:
1519
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1520
+
1521
+ def generate(_deep, vo):
1522
+ for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
1523
+ yield dumps(row, cls=APIEncoder) + '\n'
1524
+
1525
+ deep = request.args.get('deep', default=False)
1526
+
1527
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1528
+ except ValueError as error:
1529
+ return generate_http_error_flask(400, error)
1530
+
1531
+
1532
+ class ReplicasRSE(ErrorHandlingMethodView):
1533
+
1534
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1535
+ def get(self, rse):
1536
+ """
1537
+ ---
1538
+ summary: List Dataset Replicas per RSE
1539
+ description: List dataset replicas per RSE.
1540
+ tags:
1541
+ - Replicas
1542
+ parameters:
1543
+ - name: rse
1544
+ in: path
1545
+ description: The rse to filter for.
1546
+ schema:
1547
+ type: string
1548
+ style: simple
1549
+ responses:
1550
+ 200:
1551
+ description: OK
1552
+ content:
1553
+ application/x-json-stream:
1554
+ schema:
1555
+ description: A list of dataset replicas.
1556
+ type: array
1557
+ items:
1558
+ type: object
1559
+ properties:
1560
+ scope:
1561
+ description: The scope of the replica.
1562
+ type: string
1563
+ name:
1564
+ description: The name of the replica.
1565
+ type: string
1566
+ rse:
1567
+ description: The name of the associated RSE.
1568
+ type: string
1569
+ rse_id:
1570
+ description: The id of the associated RSE.
1571
+ type: string
1572
+ bytes:
1573
+ description: The size of the replica.
1574
+ type: integer
1575
+ length:
1576
+ description: The length of the replica.
1577
+ type: integer
1578
+ available_bytes:
1579
+ description: The number of available bytes of the replica.
1580
+ type: integer
1581
+ available_length:
1582
+ description: The available length of the replica.
1583
+ type: integer
1584
+ state:
1585
+ description: The state of the replica.
1586
+ type: string
1587
+ created_at:
1588
+ description: The date-time the replica was created.
1589
+ type: string
1590
+ format: date-time
1591
+ updated_at:
1592
+ description: The date-time the replica was updated.
1593
+ type: string
1594
+ format: date-time
1595
+ accessed_at:
1596
+ description: The date-time the replica was accessed.
1597
+ type: string
1598
+ format: date-time
1599
+ 401:
1600
+ description: Invalid Auth Token
1601
+ 406:
1602
+ description: Not acceptable
1603
+ """
1604
+
1605
+ def generate(vo):
1606
+ for row in list_datasets_per_rse(rse=rse, vo=vo):
1607
+ yield dumps(row, cls=APIEncoder) + '\n'
1608
+
1609
+ return try_stream(generate(vo=request.environ.get('vo')))
1610
+
1611
+
1612
+ class BadDIDs(ErrorHandlingMethodView):
1613
+
1614
+ def post(self):
1615
+ """
1616
+ ---
1617
+ summary: Mark Bad by DID
1618
+ description: Declare a list of bad replicas by DID.
1619
+ tags:
1620
+ - Replicas
1621
+ requestBody:
1622
+ content:
1623
+ application/json:
1624
+ schema:
1625
+ type: object
1626
+ properties:
1627
+ expires_at:
1628
+ description: The expires at value.
1629
+ type: string
1630
+ format: date-time
1631
+ dids:
1632
+ description: The list of dids associated with the bad replicas.
1633
+ type: array
1634
+ items:
1635
+ type: object
1636
+ properties:
1637
+ scope:
1638
+ description: The scope of the did.
1639
+ type: string
1640
+ name:
1641
+ description: The name of the did.
1642
+ type: string
1643
+ rse:
1644
+ description: The name of the rse.
1645
+ type: string
1646
+ reason:
1647
+ description: The reason for the change.
1648
+ type: string
1649
+ responses:
1650
+ 201:
1651
+ description: OK
1652
+ content:
1653
+ application/json:
1654
+ schema:
1655
+ description: All files not declared as bad.
1656
+ type: array
1657
+ items:
1658
+ type: string
1659
+ 400:
1660
+ description: Cannot decode json parameter list.
1661
+ 401:
1662
+ description: Invalid Auth Token
1663
+ 404:
1664
+ description: Not found
1665
+ """
1666
+ parameters = json_parameters(parse_response)
1667
+ expires_at = param_get(parameters, 'expires_at', default=None)
1668
+ if expires_at:
1669
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1670
+
1671
+ try:
1672
+ not_declared_files = add_bad_dids(
1673
+ dids=param_get(parameters, 'dids', default=[]),
1674
+ rse=param_get(parameters, 'rse', default=None),
1675
+ issuer=request.environ.get('issuer'),
1676
+ state=BadFilesStatus.BAD,
1677
+ reason=param_get(parameters, 'reason', default=None),
1678
+ expires_at=expires_at,
1679
+ vo=request.environ.get('vo'),
1680
+ )
1681
+ except (ValueError, InvalidType) as error:
1682
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1683
+ except AccessDenied as error:
1684
+ return generate_http_error_flask(401, error)
1685
+ except ReplicaNotFound as error:
1686
+ return generate_http_error_flask(404, error)
1687
+ except Duplicate as error:
1688
+ return generate_http_error_flask(409, error)
1689
+
1690
+ return Response(dumps(not_declared_files), status=201, content_type='application/json')
1691
+
1692
+
1693
+ class BadPFNs(ErrorHandlingMethodView):
1694
+
1695
+ def post(self):
1696
+ """
1697
+ ---
1698
+ summary: Declare Bad PFNs
1699
+ description: Declare a list of bad PFNs.
1700
+ tags:
1701
+ - Replicas
1702
+ requestBody:
1703
+ content:
1704
+ application/json:
1705
+ schema:
1706
+ type: object
1707
+ properties:
1708
+ expires_at:
1709
+ description: The expires at value. Only apply to TEMPORARY_UNAVAILABLE.
1710
+ type: string
1711
+ format: date-time
1712
+ pfns:
1713
+ description: The list of pfns associated with the bad PFNs.
1714
+ type: array
1715
+ items:
1716
+ type: string
1717
+ state:
1718
+ description: The state to set the PFNs to.
1719
+ type: string
1720
+ enum: ["BAD", "SUSPICIOUS", "TEMPORARY_UNAVAILABLE"]
1721
+ reason:
1722
+ description: The reason for the change.
1723
+ type: string
1724
+ responses:
1725
+ 201:
1726
+ description: Created
1727
+ 400:
1728
+ description: Cannot decode json parameter list.
1729
+ 401:
1730
+ description: Invalid Auth Token
1731
+ 404:
1732
+ description: Replica not found
1733
+ 409:
1734
+ description: Duplicate
1735
+ """
1736
+ parameters = json_parameters(parse_response)
1737
+ expires_at = param_get(parameters, 'expires_at', default=None)
1738
+ if expires_at:
1739
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1740
+
1741
+ try:
1742
+ add_bad_pfns(
1743
+ pfns=param_get(parameters, 'pfns', default=[]),
1744
+ issuer=request.environ.get('issuer'),
1745
+ state=param_get(parameters, 'state', default=None),
1746
+ reason=param_get(parameters, 'reason', default=None),
1747
+ expires_at=expires_at,
1748
+ vo=request.environ.get('vo'),
1749
+ )
1750
+ except (ValueError, InvalidType) as error:
1751
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1752
+ except AccessDenied as error:
1753
+ return generate_http_error_flask(401, error)
1754
+ except ReplicaNotFound as error:
1755
+ return generate_http_error_flask(404, error)
1756
+ except Duplicate as error:
1757
+ return generate_http_error_flask(409, error)
1758
+
1759
+ return 'Created', 201
1760
+
1761
+
1762
+ class Tombstone(ErrorHandlingMethodView):
1763
+
1764
+ def post(self):
1765
+ """
1766
+ ---
1767
+ summary: Set Tombstone
1768
+ description: Set a tombstone on a list of replicas.
1769
+ tags:
1770
+ - Replicas
1771
+ requestBody:
1772
+ content:
1773
+ application/json:
1774
+ schema:
1775
+ type: object
1776
+ properties:
1777
+ replicas:
1778
+ description: The replicas to set the tombstone to.
1779
+ type: array
1780
+ items:
1781
+ type: object
1782
+ required:
1783
+ - rse
1784
+ - scope
1785
+ - name
1786
+ properties:
1787
+ rse:
1788
+ description: The rse associated with the tombstone.
1789
+ type: string
1790
+ scope:
1791
+ description: The scope of the replica
1792
+ type: string
1793
+ name:
1794
+ description: The name of the replica.
1795
+ type: string
1796
+ responses:
1797
+ 201:
1798
+ description: Created
1799
+ 401:
1800
+ description: Invalid Auth Token
1801
+ 404:
1802
+ description: Not found
1803
+ 423:
1804
+ description: Replica is locked.
1805
+ """
1806
+ parameters = json_parameters(parse_response)
1807
+ replicas = param_get(parameters, 'replicas', default=[])
1808
+
1809
+ try:
1810
+ for replica in replicas:
1811
+ set_tombstone(
1812
+ rse=replica['rse'],
1813
+ scope=replica['scope'],
1814
+ name=replica['name'],
1815
+ issuer=request.environ.get('issuer'),
1816
+ vo=request.environ.get('vo'),
1817
+ )
1818
+ except ReplicaNotFound as error:
1819
+ return generate_http_error_flask(404, error)
1820
+ except ReplicaIsLocked as error:
1821
+ return generate_http_error_flask(423, error)
1822
+
1823
+ return 'Created', 201
1824
+
1825
+
1826
+ def blueprint(with_doc=False):
1827
+ bp = AuthenticatedBlueprint('replicas', __name__, url_prefix='/replicas')
1828
+
1829
+ list_replicas_view = ListReplicas.as_view('list_replicas')
1830
+ bp.add_url_rule('/list', view_func=list_replicas_view, methods=['post', ])
1831
+ replicas_view = Replicas.as_view('replicas')
1832
+ if not with_doc:
1833
+ # rule without trailing slash needs to be added before rule with trailing slash
1834
+ bp.add_url_rule('', view_func=replicas_view, methods=['post', 'put', 'delete'])
1835
+ bp.add_url_rule('/', view_func=replicas_view, methods=['post', 'put', 'delete'])
1836
+ suspicious_replicas_view = SuspiciousReplicas.as_view('suspicious_replicas')
1837
+ bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['get', 'post'])
1838
+ bad_replicas_states_view = BadReplicasStates.as_view('bad_replicas_states')
1839
+ bp.add_url_rule('/bad/states', view_func=bad_replicas_states_view, methods=['get', ])
1840
+ bad_replicas_summary_view = BadReplicasSummary.as_view('bad_replicas_summary')
1841
+ bp.add_url_rule('/bad/summary', view_func=bad_replicas_summary_view, methods=['get', ])
1842
+ bad_replicas_pfn_view = BadPFNs.as_view('add_bad_pfns')
1843
+ bp.add_url_rule('/bad/pfns', view_func=bad_replicas_pfn_view, methods=['post', ])
1844
+ bad_replicas_dids_view = BadDIDs.as_view('add_bad_dids')
1845
+ bp.add_url_rule('/bad/dids', view_func=bad_replicas_dids_view, methods=['post', ])
1846
+ replicas_rse_view = ReplicasRSE.as_view('replicas_rse')
1847
+ bp.add_url_rule('/rse/<rse>', view_func=replicas_rse_view, methods=['get', ])
1848
+
1849
+ bad_replicas_view = BadReplicas.as_view('bad_replicas')
1850
+ bp.add_url_rule('/bad', view_func=bad_replicas_view, methods=['post', ])
1851
+
1852
+ quarantine_replicas_view = QuarantineReplicas.as_view('quarantine_replicas')
1853
+ bp.add_url_rule('/quarantine', view_func=quarantine_replicas_view, methods=['post', ])
1854
+
1855
+ replicas_dids_view = ReplicasDIDs.as_view('replicas_dids')
1856
+ bp.add_url_rule('/dids', view_func=replicas_dids_view, methods=['post', ])
1857
+ dataset_replicas_view = DatasetReplicas.as_view('dataset_replicas')
1858
+ bp.add_url_rule('/<path:scope_name>/datasets', view_func=dataset_replicas_view, methods=['get', ])
1859
+ dataset_replicas_bulk_view = DatasetReplicasBulk.as_view('dataset_replicas_bulk')
1860
+ bp.add_url_rule('/datasets_bulk', view_func=dataset_replicas_bulk_view, methods=['post', ])
1861
+ dataset_replicas_vp_view = DatasetReplicasVP.as_view('dataset_replicas_vp')
1862
+ bp.add_url_rule('/<path:scope_name>', view_func=replicas_view, methods=['get', ])
1863
+ set_tombstone_view = Tombstone.as_view('set_tombstone')
1864
+ bp.add_url_rule('/tombstone', view_func=set_tombstone_view, methods=['post', ])
1865
+
1866
+ if not with_doc:
1867
+ bp.add_url_rule('/list/', view_func=list_replicas_view, methods=['post', ])
1868
+ bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['get', 'post'])
1869
+ bp.add_url_rule('/bad/states/', view_func=bad_replicas_states_view, methods=['get', ])
1870
+ bp.add_url_rule('/bad/summary/', view_func=bad_replicas_summary_view, methods=['get', ])
1871
+ bp.add_url_rule('/bad/pfns/', view_func=bad_replicas_pfn_view, methods=['post', ])
1872
+ bp.add_url_rule('/bad/dids/', view_func=bad_replicas_dids_view, methods=['post', ])
1873
+ bp.add_url_rule('/rse/<rse>/', view_func=replicas_rse_view, methods=['get', ])
1874
+ bp.add_url_rule('/bad/', view_func=bad_replicas_view, methods=['post', ])
1875
+ bp.add_url_rule('/dids/', view_func=replicas_dids_view, methods=['post', ])
1876
+ bp.add_url_rule('/datasets_bulk/', view_func=dataset_replicas_bulk_view, methods=['post', ])
1877
+ bp.add_url_rule('/<path:scope_name>/datasets_vp', view_func=dataset_replicas_vp_view, methods=['get', ])
1878
+ bp.add_url_rule('/<path:scope_name>/', view_func=replicas_view, methods=['get', ])
1879
+ bp.add_url_rule('/tombstone/', view_func=set_tombstone_view, methods=['post', ])
1880
+ bp.add_url_rule('/quarantine/', view_func=quarantine_replicas_view, methods=['post', ])
1881
+
1882
+ bp.after_request(response_headers)
1883
+ return bp
1884
+
1885
+
1886
+ def make_doc():
1887
+ """ Only used for sphinx documentation """
1888
+ doc_app = Flask(__name__)
1889
+ doc_app.register_blueprint(blueprint(with_doc=True))
1890
+ return doc_app