rucio 32.8.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (481) hide show
  1. rucio/__init__.py +18 -0
  2. rucio/alembicrevision.py +16 -0
  3. rucio/api/__init__.py +14 -0
  4. rucio/api/account.py +266 -0
  5. rucio/api/account_limit.py +287 -0
  6. rucio/api/authentication.py +302 -0
  7. rucio/api/config.py +218 -0
  8. rucio/api/credential.py +60 -0
  9. rucio/api/did.py +726 -0
  10. rucio/api/dirac.py +71 -0
  11. rucio/api/exporter.py +60 -0
  12. rucio/api/heartbeat.py +62 -0
  13. rucio/api/identity.py +160 -0
  14. rucio/api/importer.py +46 -0
  15. rucio/api/lifetime_exception.py +95 -0
  16. rucio/api/lock.py +131 -0
  17. rucio/api/meta.py +85 -0
  18. rucio/api/permission.py +72 -0
  19. rucio/api/quarantined_replica.py +69 -0
  20. rucio/api/replica.py +528 -0
  21. rucio/api/request.py +220 -0
  22. rucio/api/rse.py +601 -0
  23. rucio/api/rule.py +335 -0
  24. rucio/api/scope.py +89 -0
  25. rucio/api/subscription.py +255 -0
  26. rucio/api/temporary_did.py +49 -0
  27. rucio/api/vo.py +112 -0
  28. rucio/client/__init__.py +16 -0
  29. rucio/client/accountclient.py +413 -0
  30. rucio/client/accountlimitclient.py +155 -0
  31. rucio/client/baseclient.py +929 -0
  32. rucio/client/client.py +77 -0
  33. rucio/client/configclient.py +113 -0
  34. rucio/client/credentialclient.py +54 -0
  35. rucio/client/didclient.py +691 -0
  36. rucio/client/diracclient.py +48 -0
  37. rucio/client/downloadclient.py +1674 -0
  38. rucio/client/exportclient.py +44 -0
  39. rucio/client/fileclient.py +51 -0
  40. rucio/client/importclient.py +42 -0
  41. rucio/client/lifetimeclient.py +74 -0
  42. rucio/client/lockclient.py +99 -0
  43. rucio/client/metaclient.py +137 -0
  44. rucio/client/pingclient.py +45 -0
  45. rucio/client/replicaclient.py +444 -0
  46. rucio/client/requestclient.py +109 -0
  47. rucio/client/rseclient.py +664 -0
  48. rucio/client/ruleclient.py +287 -0
  49. rucio/client/scopeclient.py +88 -0
  50. rucio/client/subscriptionclient.py +161 -0
  51. rucio/client/touchclient.py +78 -0
  52. rucio/client/uploadclient.py +871 -0
  53. rucio/common/__init__.py +14 -0
  54. rucio/common/cache.py +74 -0
  55. rucio/common/config.py +796 -0
  56. rucio/common/constants.py +92 -0
  57. rucio/common/constraints.py +18 -0
  58. rucio/common/didtype.py +187 -0
  59. rucio/common/dumper/__init__.py +306 -0
  60. rucio/common/dumper/consistency.py +449 -0
  61. rucio/common/dumper/data_models.py +325 -0
  62. rucio/common/dumper/path_parsing.py +65 -0
  63. rucio/common/exception.py +1092 -0
  64. rucio/common/extra.py +37 -0
  65. rucio/common/logging.py +404 -0
  66. rucio/common/pcache.py +1387 -0
  67. rucio/common/policy.py +84 -0
  68. rucio/common/schema/__init__.py +143 -0
  69. rucio/common/schema/atlas.py +411 -0
  70. rucio/common/schema/belleii.py +406 -0
  71. rucio/common/schema/cms.py +478 -0
  72. rucio/common/schema/domatpc.py +399 -0
  73. rucio/common/schema/escape.py +424 -0
  74. rucio/common/schema/generic.py +431 -0
  75. rucio/common/schema/generic_multi_vo.py +410 -0
  76. rucio/common/schema/icecube.py +404 -0
  77. rucio/common/schema/lsst.py +423 -0
  78. rucio/common/stomp_utils.py +160 -0
  79. rucio/common/stopwatch.py +56 -0
  80. rucio/common/test_rucio_server.py +148 -0
  81. rucio/common/types.py +158 -0
  82. rucio/common/utils.py +1946 -0
  83. rucio/core/__init__.py +14 -0
  84. rucio/core/account.py +426 -0
  85. rucio/core/account_counter.py +171 -0
  86. rucio/core/account_limit.py +357 -0
  87. rucio/core/authentication.py +563 -0
  88. rucio/core/config.py +386 -0
  89. rucio/core/credential.py +218 -0
  90. rucio/core/did.py +3102 -0
  91. rucio/core/did_meta_plugins/__init__.py +250 -0
  92. rucio/core/did_meta_plugins/did_column_meta.py +326 -0
  93. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +116 -0
  94. rucio/core/did_meta_plugins/filter_engine.py +573 -0
  95. rucio/core/did_meta_plugins/json_meta.py +215 -0
  96. rucio/core/did_meta_plugins/mongo_meta.py +199 -0
  97. rucio/core/did_meta_plugins/postgres_meta.py +317 -0
  98. rucio/core/dirac.py +208 -0
  99. rucio/core/distance.py +164 -0
  100. rucio/core/exporter.py +59 -0
  101. rucio/core/heartbeat.py +263 -0
  102. rucio/core/identity.py +290 -0
  103. rucio/core/importer.py +248 -0
  104. rucio/core/lifetime_exception.py +377 -0
  105. rucio/core/lock.py +474 -0
  106. rucio/core/message.py +241 -0
  107. rucio/core/meta.py +190 -0
  108. rucio/core/monitor.py +441 -0
  109. rucio/core/naming_convention.py +154 -0
  110. rucio/core/nongrid_trace.py +124 -0
  111. rucio/core/oidc.py +1339 -0
  112. rucio/core/permission/__init__.py +107 -0
  113. rucio/core/permission/atlas.py +1333 -0
  114. rucio/core/permission/belleii.py +1076 -0
  115. rucio/core/permission/cms.py +1166 -0
  116. rucio/core/permission/escape.py +1076 -0
  117. rucio/core/permission/generic.py +1128 -0
  118. rucio/core/permission/generic_multi_vo.py +1148 -0
  119. rucio/core/quarantined_replica.py +190 -0
  120. rucio/core/replica.py +3627 -0
  121. rucio/core/replica_sorter.py +368 -0
  122. rucio/core/request.py +2241 -0
  123. rucio/core/rse.py +1835 -0
  124. rucio/core/rse_counter.py +155 -0
  125. rucio/core/rse_expression_parser.py +460 -0
  126. rucio/core/rse_selector.py +277 -0
  127. rucio/core/rule.py +3419 -0
  128. rucio/core/rule_grouping.py +1473 -0
  129. rucio/core/scope.py +152 -0
  130. rucio/core/subscription.py +316 -0
  131. rucio/core/temporary_did.py +188 -0
  132. rucio/core/topology.py +448 -0
  133. rucio/core/trace.py +361 -0
  134. rucio/core/transfer.py +1233 -0
  135. rucio/core/vo.py +151 -0
  136. rucio/core/volatile_replica.py +123 -0
  137. rucio/daemons/__init__.py +14 -0
  138. rucio/daemons/abacus/__init__.py +14 -0
  139. rucio/daemons/abacus/account.py +106 -0
  140. rucio/daemons/abacus/collection_replica.py +113 -0
  141. rucio/daemons/abacus/rse.py +107 -0
  142. rucio/daemons/atropos/__init__.py +14 -0
  143. rucio/daemons/atropos/atropos.py +243 -0
  144. rucio/daemons/auditor/__init__.py +261 -0
  145. rucio/daemons/auditor/hdfs.py +86 -0
  146. rucio/daemons/auditor/srmdumps.py +284 -0
  147. rucio/daemons/automatix/__init__.py +14 -0
  148. rucio/daemons/automatix/automatix.py +281 -0
  149. rucio/daemons/badreplicas/__init__.py +14 -0
  150. rucio/daemons/badreplicas/minos.py +311 -0
  151. rucio/daemons/badreplicas/minos_temporary_expiration.py +173 -0
  152. rucio/daemons/badreplicas/necromancer.py +200 -0
  153. rucio/daemons/bb8/__init__.py +14 -0
  154. rucio/daemons/bb8/bb8.py +356 -0
  155. rucio/daemons/bb8/common.py +762 -0
  156. rucio/daemons/bb8/nuclei_background_rebalance.py +147 -0
  157. rucio/daemons/bb8/t2_background_rebalance.py +146 -0
  158. rucio/daemons/c3po/__init__.py +14 -0
  159. rucio/daemons/c3po/algorithms/__init__.py +14 -0
  160. rucio/daemons/c3po/algorithms/simple.py +131 -0
  161. rucio/daemons/c3po/algorithms/t2_free_space.py +125 -0
  162. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +127 -0
  163. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +279 -0
  164. rucio/daemons/c3po/c3po.py +342 -0
  165. rucio/daemons/c3po/collectors/__init__.py +14 -0
  166. rucio/daemons/c3po/collectors/agis.py +108 -0
  167. rucio/daemons/c3po/collectors/free_space.py +62 -0
  168. rucio/daemons/c3po/collectors/jedi_did.py +48 -0
  169. rucio/daemons/c3po/collectors/mock_did.py +46 -0
  170. rucio/daemons/c3po/collectors/network_metrics.py +63 -0
  171. rucio/daemons/c3po/collectors/workload.py +110 -0
  172. rucio/daemons/c3po/utils/__init__.py +14 -0
  173. rucio/daemons/c3po/utils/dataset_cache.py +40 -0
  174. rucio/daemons/c3po/utils/expiring_dataset_cache.py +45 -0
  175. rucio/daemons/c3po/utils/expiring_list.py +63 -0
  176. rucio/daemons/c3po/utils/popularity.py +82 -0
  177. rucio/daemons/c3po/utils/timeseries.py +76 -0
  178. rucio/daemons/cache/__init__.py +14 -0
  179. rucio/daemons/cache/consumer.py +191 -0
  180. rucio/daemons/common.py +391 -0
  181. rucio/daemons/conveyor/__init__.py +14 -0
  182. rucio/daemons/conveyor/common.py +530 -0
  183. rucio/daemons/conveyor/finisher.py +492 -0
  184. rucio/daemons/conveyor/poller.py +372 -0
  185. rucio/daemons/conveyor/preparer.py +198 -0
  186. rucio/daemons/conveyor/receiver.py +206 -0
  187. rucio/daemons/conveyor/stager.py +127 -0
  188. rucio/daemons/conveyor/submitter.py +379 -0
  189. rucio/daemons/conveyor/throttler.py +468 -0
  190. rucio/daemons/follower/__init__.py +14 -0
  191. rucio/daemons/follower/follower.py +97 -0
  192. rucio/daemons/hermes/__init__.py +14 -0
  193. rucio/daemons/hermes/hermes.py +738 -0
  194. rucio/daemons/judge/__init__.py +14 -0
  195. rucio/daemons/judge/cleaner.py +149 -0
  196. rucio/daemons/judge/evaluator.py +172 -0
  197. rucio/daemons/judge/injector.py +154 -0
  198. rucio/daemons/judge/repairer.py +144 -0
  199. rucio/daemons/oauthmanager/__init__.py +14 -0
  200. rucio/daemons/oauthmanager/oauthmanager.py +199 -0
  201. rucio/daemons/reaper/__init__.py +14 -0
  202. rucio/daemons/reaper/dark_reaper.py +272 -0
  203. rucio/daemons/reaper/light_reaper.py +255 -0
  204. rucio/daemons/reaper/reaper.py +701 -0
  205. rucio/daemons/replicarecoverer/__init__.py +14 -0
  206. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +487 -0
  207. rucio/daemons/storage/__init__.py +14 -0
  208. rucio/daemons/storage/consistency/__init__.py +14 -0
  209. rucio/daemons/storage/consistency/actions.py +753 -0
  210. rucio/daemons/tracer/__init__.py +14 -0
  211. rucio/daemons/tracer/kronos.py +513 -0
  212. rucio/daemons/transmogrifier/__init__.py +14 -0
  213. rucio/daemons/transmogrifier/transmogrifier.py +753 -0
  214. rucio/daemons/undertaker/__init__.py +14 -0
  215. rucio/daemons/undertaker/undertaker.py +137 -0
  216. rucio/db/__init__.py +14 -0
  217. rucio/db/sqla/__init__.py +38 -0
  218. rucio/db/sqla/constants.py +192 -0
  219. rucio/db/sqla/migrate_repo/__init__.py +14 -0
  220. rucio/db/sqla/migrate_repo/env.py +111 -0
  221. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +71 -0
  222. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +50 -0
  223. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +61 -0
  224. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +46 -0
  225. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +93 -0
  226. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +78 -0
  227. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +46 -0
  228. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +53 -0
  229. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +69 -0
  230. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +42 -0
  231. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +46 -0
  232. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +61 -0
  233. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +42 -0
  234. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +141 -0
  235. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +75 -0
  236. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +75 -0
  237. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +46 -0
  238. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +51 -0
  239. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +135 -0
  240. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +65 -0
  241. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +42 -0
  242. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +66 -0
  243. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +54 -0
  244. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +43 -0
  245. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +46 -0
  246. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +47 -0
  247. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +54 -0
  248. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +39 -0
  249. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +48 -0
  250. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +47 -0
  251. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +48 -0
  252. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +59 -0
  253. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +47 -0
  254. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +72 -0
  255. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +46 -0
  256. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +45 -0
  257. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +48 -0
  258. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +48 -0
  259. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +42 -0
  260. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +69 -0
  261. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +46 -0
  262. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +78 -0
  263. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +62 -0
  264. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +74 -0
  265. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +44 -0
  266. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +67 -0
  267. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +134 -0
  268. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +58 -0
  269. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +79 -0
  270. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +61 -0
  271. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +45 -0
  272. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +46 -0
  273. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +65 -0
  274. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +42 -0
  275. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +46 -0
  276. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +46 -0
  277. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +80 -0
  278. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +43 -0
  279. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +61 -0
  280. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +47 -0
  281. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +46 -0
  282. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +52 -0
  283. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +42 -0
  284. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +65 -0
  285. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +46 -0
  286. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +47 -0
  287. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +45 -0
  288. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +46 -0
  289. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +48 -0
  290. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +50 -0
  291. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +59 -0
  292. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +48 -0
  293. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +108 -0
  294. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +57 -0
  295. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +51 -0
  296. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +50 -0
  297. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +46 -0
  298. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +42 -0
  299. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +93 -0
  300. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +73 -0
  301. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +52 -0
  302. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +45 -0
  303. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +46 -0
  304. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +54 -0
  305. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +48 -0
  306. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +70 -0
  307. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +48 -0
  308. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +95 -0
  309. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +55 -0
  310. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +74 -0
  311. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +78 -0
  312. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +49 -0
  313. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +124 -0
  314. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +60 -0
  315. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +53 -0
  316. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +56 -0
  317. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +67 -0
  318. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +50 -0
  319. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +46 -0
  320. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +92 -0
  321. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +42 -0
  322. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +46 -0
  323. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +147 -0
  324. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +78 -0
  325. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +53 -0
  326. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +74 -0
  327. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +56 -0
  328. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +46 -0
  329. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +68 -0
  330. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +48 -0
  331. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +149 -0
  332. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +106 -0
  333. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +47 -0
  334. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +45 -0
  335. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +105 -0
  336. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +52 -0
  337. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +106 -0
  338. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +30 -0
  339. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +75 -0
  340. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +49 -0
  341. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +45 -0
  342. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +38 -0
  343. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +44 -0
  344. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +46 -0
  345. rucio/db/sqla/models.py +1834 -0
  346. rucio/db/sqla/sautils.py +48 -0
  347. rucio/db/sqla/session.py +470 -0
  348. rucio/db/sqla/types.py +207 -0
  349. rucio/db/sqla/util.py +521 -0
  350. rucio/rse/__init__.py +97 -0
  351. rucio/rse/protocols/__init__.py +14 -0
  352. rucio/rse/protocols/cache.py +123 -0
  353. rucio/rse/protocols/dummy.py +112 -0
  354. rucio/rse/protocols/gfal.py +701 -0
  355. rucio/rse/protocols/globus.py +243 -0
  356. rucio/rse/protocols/gsiftp.py +93 -0
  357. rucio/rse/protocols/http_cache.py +83 -0
  358. rucio/rse/protocols/mock.py +124 -0
  359. rucio/rse/protocols/ngarc.py +210 -0
  360. rucio/rse/protocols/posix.py +251 -0
  361. rucio/rse/protocols/protocol.py +530 -0
  362. rucio/rse/protocols/rclone.py +365 -0
  363. rucio/rse/protocols/rfio.py +137 -0
  364. rucio/rse/protocols/srm.py +339 -0
  365. rucio/rse/protocols/ssh.py +414 -0
  366. rucio/rse/protocols/storm.py +207 -0
  367. rucio/rse/protocols/webdav.py +547 -0
  368. rucio/rse/protocols/xrootd.py +295 -0
  369. rucio/rse/rsemanager.py +752 -0
  370. rucio/tests/__init__.py +14 -0
  371. rucio/tests/common.py +244 -0
  372. rucio/tests/common_server.py +132 -0
  373. rucio/transfertool/__init__.py +14 -0
  374. rucio/transfertool/fts3.py +1484 -0
  375. rucio/transfertool/globus.py +200 -0
  376. rucio/transfertool/globus_library.py +182 -0
  377. rucio/transfertool/mock.py +81 -0
  378. rucio/transfertool/transfertool.py +212 -0
  379. rucio/vcsversion.py +11 -0
  380. rucio/version.py +46 -0
  381. rucio/web/__init__.py +14 -0
  382. rucio/web/rest/__init__.py +14 -0
  383. rucio/web/rest/flaskapi/__init__.py +14 -0
  384. rucio/web/rest/flaskapi/authenticated_bp.py +28 -0
  385. rucio/web/rest/flaskapi/v1/__init__.py +14 -0
  386. rucio/web/rest/flaskapi/v1/accountlimits.py +234 -0
  387. rucio/web/rest/flaskapi/v1/accounts.py +1088 -0
  388. rucio/web/rest/flaskapi/v1/archives.py +100 -0
  389. rucio/web/rest/flaskapi/v1/auth.py +1642 -0
  390. rucio/web/rest/flaskapi/v1/common.py +385 -0
  391. rucio/web/rest/flaskapi/v1/config.py +305 -0
  392. rucio/web/rest/flaskapi/v1/credentials.py +213 -0
  393. rucio/web/rest/flaskapi/v1/dids.py +2204 -0
  394. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  395. rucio/web/rest/flaskapi/v1/export.py +77 -0
  396. rucio/web/rest/flaskapi/v1/heartbeats.py +129 -0
  397. rucio/web/rest/flaskapi/v1/identities.py +263 -0
  398. rucio/web/rest/flaskapi/v1/import.py +133 -0
  399. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +315 -0
  400. rucio/web/rest/flaskapi/v1/locks.py +360 -0
  401. rucio/web/rest/flaskapi/v1/main.py +83 -0
  402. rucio/web/rest/flaskapi/v1/meta.py +226 -0
  403. rucio/web/rest/flaskapi/v1/metrics.py +37 -0
  404. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  405. rucio/web/rest/flaskapi/v1/ping.py +89 -0
  406. rucio/web/rest/flaskapi/v1/redirect.py +366 -0
  407. rucio/web/rest/flaskapi/v1/replicas.py +1866 -0
  408. rucio/web/rest/flaskapi/v1/requests.py +841 -0
  409. rucio/web/rest/flaskapi/v1/rses.py +2204 -0
  410. rucio/web/rest/flaskapi/v1/rules.py +824 -0
  411. rucio/web/rest/flaskapi/v1/scopes.py +161 -0
  412. rucio/web/rest/flaskapi/v1/subscriptions.py +646 -0
  413. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  414. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  415. rucio/web/rest/flaskapi/v1/tmp_dids.py +115 -0
  416. rucio/web/rest/flaskapi/v1/traces.py +100 -0
  417. rucio/web/rest/flaskapi/v1/vos.py +280 -0
  418. rucio/web/rest/main.py +19 -0
  419. rucio/web/rest/metrics.py +28 -0
  420. rucio-32.8.6.data/data/rucio/etc/alembic.ini.template +71 -0
  421. rucio-32.8.6.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  422. rucio-32.8.6.data/data/rucio/etc/globus-config.yml.template +5 -0
  423. rucio-32.8.6.data/data/rucio/etc/ldap.cfg.template +30 -0
  424. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  425. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  426. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  427. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  428. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  429. rucio-32.8.6.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  430. rucio-32.8.6.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  431. rucio-32.8.6.data/data/rucio/etc/rucio.cfg.atlas.client.template +42 -0
  432. rucio-32.8.6.data/data/rucio/etc/rucio.cfg.template +257 -0
  433. rucio-32.8.6.data/data/rucio/etc/rucio_multi_vo.cfg.template +234 -0
  434. rucio-32.8.6.data/data/rucio/requirements.txt +55 -0
  435. rucio-32.8.6.data/data/rucio/tools/bootstrap.py +34 -0
  436. rucio-32.8.6.data/data/rucio/tools/merge_rucio_configs.py +147 -0
  437. rucio-32.8.6.data/data/rucio/tools/reset_database.py +40 -0
  438. rucio-32.8.6.data/scripts/rucio +2540 -0
  439. rucio-32.8.6.data/scripts/rucio-abacus-account +75 -0
  440. rucio-32.8.6.data/scripts/rucio-abacus-collection-replica +47 -0
  441. rucio-32.8.6.data/scripts/rucio-abacus-rse +79 -0
  442. rucio-32.8.6.data/scripts/rucio-admin +2434 -0
  443. rucio-32.8.6.data/scripts/rucio-atropos +61 -0
  444. rucio-32.8.6.data/scripts/rucio-auditor +199 -0
  445. rucio-32.8.6.data/scripts/rucio-automatix +51 -0
  446. rucio-32.8.6.data/scripts/rucio-bb8 +58 -0
  447. rucio-32.8.6.data/scripts/rucio-c3po +86 -0
  448. rucio-32.8.6.data/scripts/rucio-cache-client +135 -0
  449. rucio-32.8.6.data/scripts/rucio-cache-consumer +43 -0
  450. rucio-32.8.6.data/scripts/rucio-conveyor-finisher +59 -0
  451. rucio-32.8.6.data/scripts/rucio-conveyor-poller +67 -0
  452. rucio-32.8.6.data/scripts/rucio-conveyor-preparer +38 -0
  453. rucio-32.8.6.data/scripts/rucio-conveyor-receiver +44 -0
  454. rucio-32.8.6.data/scripts/rucio-conveyor-stager +77 -0
  455. rucio-32.8.6.data/scripts/rucio-conveyor-submitter +140 -0
  456. rucio-32.8.6.data/scripts/rucio-conveyor-throttler +105 -0
  457. rucio-32.8.6.data/scripts/rucio-dark-reaper +54 -0
  458. rucio-32.8.6.data/scripts/rucio-dumper +159 -0
  459. rucio-32.8.6.data/scripts/rucio-follower +45 -0
  460. rucio-32.8.6.data/scripts/rucio-hermes +55 -0
  461. rucio-32.8.6.data/scripts/rucio-judge-cleaner +90 -0
  462. rucio-32.8.6.data/scripts/rucio-judge-evaluator +138 -0
  463. rucio-32.8.6.data/scripts/rucio-judge-injector +45 -0
  464. rucio-32.8.6.data/scripts/rucio-judge-repairer +45 -0
  465. rucio-32.8.6.data/scripts/rucio-kronos +45 -0
  466. rucio-32.8.6.data/scripts/rucio-light-reaper +53 -0
  467. rucio-32.8.6.data/scripts/rucio-minos +54 -0
  468. rucio-32.8.6.data/scripts/rucio-minos-temporary-expiration +51 -0
  469. rucio-32.8.6.data/scripts/rucio-necromancer +121 -0
  470. rucio-32.8.6.data/scripts/rucio-oauth-manager +64 -0
  471. rucio-32.8.6.data/scripts/rucio-reaper +84 -0
  472. rucio-32.8.6.data/scripts/rucio-replica-recoverer +249 -0
  473. rucio-32.8.6.data/scripts/rucio-storage-consistency-actions +75 -0
  474. rucio-32.8.6.data/scripts/rucio-transmogrifier +78 -0
  475. rucio-32.8.6.data/scripts/rucio-undertaker +77 -0
  476. rucio-32.8.6.dist-info/METADATA +83 -0
  477. rucio-32.8.6.dist-info/RECORD +481 -0
  478. rucio-32.8.6.dist-info/WHEEL +5 -0
  479. rucio-32.8.6.dist-info/licenses/AUTHORS.rst +94 -0
  480. rucio-32.8.6.dist-info/licenses/LICENSE +201 -0
  481. rucio-32.8.6.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1866 @@
1
+ # -*- coding: utf-8 -*-
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ from datetime import datetime
17
+ from itertools import chain
18
+ from json import dumps, loads
19
+ from urllib.parse import parse_qs, unquote
20
+ from xml.sax.saxutils import escape
21
+
22
+ from flask import Flask, Response, request
23
+
24
+ from rucio.api.quarantined_replica import quarantine_file_replicas
25
+ from rucio.api.replica import add_replicas, list_replicas, list_dataset_replicas, list_dataset_replicas_bulk, \
26
+ delete_replicas, get_did_from_pfns, update_replicas_states, declare_bad_file_replicas, add_bad_dids, add_bad_pfns, \
27
+ get_suspicious_files, declare_suspicious_file_replicas, list_bad_replicas_status, get_bad_replicas_summary, \
28
+ list_datasets_per_rse, set_tombstone, list_dataset_replicas_vp
29
+ from rucio.common.config import config_get, config_get_int
30
+ from rucio.common.constants import SUPPORTED_PROTOCOLS
31
+ from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, InvalidType, DataIdentifierNotFound, \
32
+ Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, ReplicaNotFound, InvalidObject, ScopeNotFound, ReplicaIsLocked
33
+ from rucio.common.utils import parse_response, APIEncoder, render_json_list
34
+ from rucio.core.replica_sorter import sort_replicas
35
+ from rucio.db.sqla.constants import BadFilesStatus
36
+ from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
37
+ from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, try_stream, parse_scope_name, \
38
+ response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
39
+
40
+
41
+ def _sorted_with_priorities(replicas, sorted_pfns, limit=None):
42
+ """
43
+ Pick up to "limit" replicas from "replicas" in the order given by sorted_pfns.
44
+ Sets the corresponding priority in returned replicas.
45
+
46
+ :param replicas: Dictionary {pfn: replica_definition}.
47
+ :param sorted_pfns: Sorted list of pfns.
48
+ :param limit: only return this many replicas
49
+ :yields: index and corresponding pfn
50
+ """
51
+ for idx, pfn in enumerate(sorted_pfns, start=1):
52
+ if limit is None or idx <= limit:
53
+ replica = replicas[pfn]
54
+ replica['priority'] = idx
55
+ yield pfn, replica
56
+
57
+
58
+ def _generate_one_metalink_file(rfile, policy_schema, detailed_url=True):
59
+ yield ' <file name="' + rfile['name'] + '">\n'
60
+
61
+ if 'parents' in rfile and rfile['parents']:
62
+ yield ' <parents>\n'
63
+ for parent in rfile['parents']:
64
+ yield ' <did>' + parent + '</did>\n'
65
+ yield ' </parents>\n'
66
+
67
+ yield ' <identity>' + rfile['scope'] + ':' + rfile['name'] + '</identity>\n'
68
+ if rfile['adler32'] is not None:
69
+ yield ' <hash type="adler32">' + rfile['adler32'] + '</hash>\n'
70
+ if rfile['md5'] is not None:
71
+ yield ' <hash type="md5">' + rfile['md5'] + '</hash>\n'
72
+
73
+ yield ' <size>' + str(rfile['bytes']) + '</size>\n'
74
+
75
+ yield f' <glfn name="/{policy_schema}/rucio/{rfile["scope"]}:{rfile["name"]}"></glfn>\n'
76
+
77
+ for pfn, replica in rfile['pfns'].items():
78
+ if detailed_url:
79
+ yield (
80
+ ' '
81
+ f'<url location="{replica["rse"]}"'
82
+ f' domain="{replica["domain"]}"'
83
+ f' priority="{replica["priority"]}"'
84
+ f' client_extract="{str(replica["client_extract"]).lower()}"'
85
+ f'>{escape(pfn)}</url>\n'
86
+ )
87
+ else:
88
+ yield (
89
+ ' '
90
+ f'<url location="{replica["rse"]}"'
91
+ f' priority="{replica["priority"]}"'
92
+ f'>{escape(pfn)}</url>\n'
93
+ )
94
+ yield ' </file>\n'
95
+
96
+
97
+ def _generate_metalink_response(rfiles, policy_schema, detailed_url=True):
98
+ first = True
99
+ for rfile in rfiles:
100
+ if first:
101
+ # first, set the appropriate content type, and stream the header
102
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n'
103
+ first = False
104
+
105
+ yield from _generate_one_metalink_file(rfile, policy_schema=policy_schema, detailed_url=detailed_url)
106
+
107
+ if first:
108
+ # if still first output, i.e. there were no replicas
109
+ yield '<?xml version="1.0" encoding="UTF-8"?>\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\n</metalink>\n'
110
+ else:
111
+ # don't forget to send the metalink footer
112
+ yield '</metalink>\n'
113
+
114
+
115
+ def _generate_json_response(rfiles):
116
+ for rfile in rfiles:
117
+ yield dumps(rfile) + '\n'
118
+
119
+
120
+ class Replicas(ErrorHandlingMethodView):
121
+
122
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
123
+ def get(self, scope_name):
124
+ """
125
+ ---
126
+ summary: Get Replicas
127
+ description: List all replicas for data identifiers.
128
+ tags:
129
+ - Replicas
130
+ parameters:
131
+ - name: scope_name
132
+ in: path
133
+ description: The DID associated with the replicas.
134
+ schema:
135
+ type: string
136
+ style: simple
137
+ - name: X-Forwarded-For
138
+ in: header
139
+ description: The client ip
140
+ schema:
141
+ type: string
142
+ - name: schemes
143
+ in: query
144
+ description: The schemes of the replicas.
145
+ schema:
146
+ type: string
147
+ - name: select
148
+ in: query
149
+ description: The sorting algorithm.
150
+ schema:
151
+ type: string
152
+ enum: ["geoip", "closeness", "dynamic", "ranking", "random"]
153
+ - name: limit
154
+ in: query
155
+ description: The maximum number of replicas returned.
156
+ schema:
157
+ type: integer
158
+ responses:
159
+ 200:
160
+ description: OK
161
+ content:
162
+ application/x-json-stream:
163
+ schema:
164
+ description: A list with all replicas.
165
+ type: array
166
+ items:
167
+ description: A replica. Possibly contains more information.
168
+ type: object
169
+ properties:
170
+ scope:
171
+ description: The scope of the replica.
172
+ type: string
173
+ name:
174
+ description: The name of the replica.
175
+ type: string
176
+ bytes:
177
+ description: The size of the replica in bytes.
178
+ type: integer
179
+ md5:
180
+ description: The md5 checksum of the replica.
181
+ type: string
182
+ adler32:
183
+ description: The adler32 checksum of the replica.
184
+ type: string
185
+ pfns:
186
+ description: The pfns associated with the replica.
187
+ type: array
188
+ rses:
189
+ description: The rse associated with the replica.
190
+ type: string
191
+ 401:
192
+ description: Invalid Auth Token
193
+ 404:
194
+ description: Did not found
195
+ 406:
196
+ description: Not acceptable
197
+ """
198
+ try:
199
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
200
+ except ValueError as error:
201
+ return generate_http_error_flask(400, error)
202
+
203
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
204
+ metalink = (content_type == 'application/metalink4+xml')
205
+ dids = [{'scope': scope, 'name': name}]
206
+ select, limit = None, None
207
+
208
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
209
+ client_location = {'ip': client_ip, 'fqdn': None, 'site': None}
210
+
211
+ schemes = request.args.get('schemes', default=None)
212
+ select = request.args.get('select', default=None)
213
+ limit = request.args.get('limit', default=None)
214
+ if limit:
215
+ limit = int(limit)
216
+
217
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
218
+ if metalink and schemes is None:
219
+ schemes = SUPPORTED_PROTOCOLS
220
+
221
+ try:
222
+ def _list_and_sort_replicas(vo):
223
+ # we need to call list_replicas before starting to reply
224
+ # otherwise the exceptions won't be propagated correctly
225
+ for rfile in list_replicas(dids=dids, schemes=schemes, vo=vo):
226
+ replicas = []
227
+ dictreplica = {}
228
+ for rse in rfile['rses']:
229
+ for replica in rfile['rses'][rse]:
230
+ replicas.append(replica)
231
+ dictreplica[replica] = rse
232
+
233
+ replicas = sort_replicas(dictreplica, client_location, selection=select)
234
+ rfile['pfns'] = dict(_sorted_with_priorities(rfile['pfns'], replicas, limit=limit))
235
+ yield rfile
236
+
237
+ rfiles = _list_and_sort_replicas(vo=request.environ.get('vo'))
238
+ if metalink:
239
+ response_generator = _generate_metalink_response(rfiles, 'atlas', detailed_url=False)
240
+ else:
241
+ response_generator = _generate_json_response(rfiles)
242
+ return try_stream(response_generator, content_type=content_type)
243
+ except DataIdentifierNotFound as error:
244
+ return generate_http_error_flask(404, error)
245
+
246
+ def post(self):
247
+ """
248
+ ---
249
+ summary: Create File Replicas
250
+ description: Create file replicas at a given RSE.
251
+ tags:
252
+ - Replicas
253
+ requestBody:
254
+ content:
255
+ application/json:
256
+ schema:
257
+ type: object
258
+ required:
259
+ - rse
260
+ - files
261
+ properties:
262
+ rse:
263
+ description: The rse for the replication
264
+ type: string
265
+ files:
266
+ description: The files to replicate
267
+ type: array
268
+ items:
269
+ type: object
270
+ required:
271
+ - pfn
272
+ - bytes
273
+ - name
274
+ properties:
275
+ pfn:
276
+ description: The pfn of the replica.
277
+ type: string
278
+ name:
279
+ description: The DID name.
280
+ type: string
281
+ bytes:
282
+ description: The size of the replica in bytes.
283
+ type: integer
284
+ state:
285
+ description: The state of the replica.
286
+ type: string
287
+ path:
288
+ description: The path of the new replica.
289
+ type: string
290
+ md5:
291
+ description: The md5 checksum.
292
+ type: string
293
+ adler32:
294
+ description: The adler32 checksum.
295
+ type: string
296
+ lcok_cnt:
297
+ description: The lock count.
298
+ type: integer
299
+ tombstone:
300
+ description: The tombstone.
301
+ type: string
302
+ ignore_availability:
303
+ description: The ignore availability.
304
+ type: boolean
305
+ responses:
306
+ 201:
307
+ description: OK
308
+ content:
309
+ application/json:
310
+ schema:
311
+ type: string
312
+ enum: ["Created"]
313
+ 400:
314
+ description: Invalid Path
315
+ 401:
316
+ description: Invalid Auth Token
317
+ 404:
318
+ description: Rse or scope not found
319
+ 409:
320
+ description: Replica or Did already exists
321
+ 503:
322
+ description: Resource temporary unavailable
323
+ """
324
+ parameters = json_parameters(parse_response)
325
+ rse = param_get(parameters, 'rse')
326
+ files = param_get(parameters, 'files')
327
+
328
+ try:
329
+ add_replicas(
330
+ rse=rse,
331
+ files=files,
332
+ issuer=request.environ.get('issuer'),
333
+ vo=request.environ.get('vo'),
334
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
335
+ )
336
+ except InvalidPath as error:
337
+ return generate_http_error_flask(400, error)
338
+ except AccessDenied as error:
339
+ return generate_http_error_flask(401, error)
340
+ except (Duplicate, DataIdentifierAlreadyExists) as error:
341
+ return generate_http_error_flask(409, error)
342
+ except (RSENotFound, ScopeNotFound) as error:
343
+ return generate_http_error_flask(404, error)
344
+ except ResourceTemporaryUnavailable as error:
345
+ return generate_http_error_flask(503, error)
346
+
347
+ return 'Created', 201
348
+
349
+ def put(self):
350
+ """
351
+ ---
352
+ summary: Update File Replicas
353
+ description: Update file replicas state at a given RSE.
354
+ tags:
355
+ - Replicas
356
+ requestBody:
357
+ content:
358
+ application/json:
359
+ schema:
360
+ type: object
361
+ required:
362
+ - rse
363
+ - files
364
+ properties:
365
+ rse:
366
+ description: The rse for the replication
367
+ type: string
368
+ files:
369
+ description: The files to replicate
370
+ type: array
371
+ items:
372
+ type: object
373
+ properties:
374
+ name:
375
+ description: The pfn of the replica.
376
+ type: string
377
+ state:
378
+ description: The pfn of the replica.
379
+ type: string
380
+ path:
381
+ description: The pfn of the replica.
382
+ type: string
383
+ error_message:
384
+ description: The error message if an error occured.
385
+ type: string
386
+ broken_rule_id:
387
+ description: The id of the broken rule if one was found.
388
+ type: string
389
+ broken_message:
390
+ description: The message of the broken rule.
391
+ type: string
392
+ responses:
393
+ 200:
394
+ description: OK
395
+ 400:
396
+ description: Cannot decode json parameter list
397
+ 401:
398
+ description: Invalid Auth Token
399
+ """
400
+ parameters = json_parameters(parse_response)
401
+ rse = param_get(parameters, 'rse')
402
+ files = param_get(parameters, 'files')
403
+
404
+ try:
405
+ update_replicas_states(rse=rse, files=files, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
406
+ except AccessDenied as error:
407
+ return generate_http_error_flask(401, error)
408
+
409
+ return '', 200
410
+
411
+ def delete(self):
412
+ """
413
+ ---
414
+ summary: Delete File Replicas
415
+ description: Delete file replicas at a given RSE.
416
+ tags:
417
+ - Replicas
418
+ requestBody:
419
+ content:
420
+ application/json:
421
+ schema:
422
+ type: object
423
+ required:
424
+ - rse
425
+ - files
426
+ properties:
427
+ rse:
428
+ description: The rse name.
429
+ type: string
430
+ files:
431
+ description: The files to delete.
432
+ type: array
433
+ items:
434
+ type: object
435
+ required:
436
+ - name
437
+ properties:
438
+ name:
439
+ description: The name of the replica.
440
+ type: string
441
+ responses:
442
+ 200:
443
+ description: OK
444
+ 400:
445
+ description: Cannot decode json parameter list.
446
+ 401:
447
+ description: Invalid Auth Token
448
+ 404:
449
+ description: Rse or Replica not found
450
+ """
451
+ parameters = json_parameters(parse_response)
452
+ rse = param_get(parameters, 'rse')
453
+ files = param_get(parameters, 'files')
454
+
455
+ try:
456
+ delete_replicas(
457
+ rse=rse,
458
+ files=files,
459
+ issuer=request.environ.get('issuer'),
460
+ vo=request.environ.get('vo'),
461
+ ignore_availability=param_get(parameters, 'ignore_availability', default=False),
462
+ )
463
+ except AccessDenied as error:
464
+ return generate_http_error_flask(401, error)
465
+ except (RSENotFound, ReplicaNotFound) as error:
466
+ return generate_http_error_flask(404, error)
467
+ except ResourceTemporaryUnavailable as error:
468
+ return generate_http_error_flask(503, error)
469
+
470
+ return '', 200
471
+
472
+
473
+ class ListReplicas(ErrorHandlingMethodView):
474
+
475
+ @check_accept_header_wrapper_flask(['application/x-json-stream', 'application/metalink4+xml'])
476
+ def post(self):
477
+ """
478
+ ---
479
+ summary: List Replicas
480
+ description: List all replicas for a DID.
481
+ tags:
482
+ - Replicas
483
+ parameters:
484
+ - name: X-Forwarded-For
485
+ in: header
486
+ description: The client ip address.
487
+ schema:
488
+ type: string
489
+ - name: limit
490
+ in: query
491
+ description: The maximum number pfns per replica to return.
492
+ schema:
493
+ type: integer
494
+ - name: select
495
+ in: query
496
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
497
+ schema:
498
+ type: string
499
+ - name: sort
500
+ in: query
501
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
502
+ schema:
503
+ type: string
504
+ requestBody:
505
+ content:
506
+ application/json:
507
+ schema:
508
+ type: object
509
+ properties:
510
+ client_location:
511
+ description: The clients location.
512
+ type: string
513
+ dids:
514
+ description: List of Dids.
515
+ type: array
516
+ items:
517
+ type: object
518
+ properties:
519
+ scope:
520
+ description: The scope of the did.
521
+ type: string
522
+ name:
523
+ description: The name of the did.
524
+ type: string
525
+ schemes:
526
+ description: A list of schemes to filter the replicas.
527
+ type: array
528
+ items:
529
+ type: string
530
+ sort:
531
+ description: Requested sorting of the result, e.g., 'geoip', 'closeness', 'dynamic', 'ranking', 'random'.
532
+ type: string
533
+ unavailable:
534
+ description: If unavailable rse should be considered.
535
+ type: boolean
536
+ deprecated: true
537
+ ignore_availability:
538
+ description: If the availability should be ignored.
539
+ type: boolean
540
+ rse_expression:
541
+ description: The RSE expression to restrict on a list of RSEs.
542
+ type: string
543
+ all_states:
544
+ description: Return all replicas whatever state they are in. Adds an extra 'states' entry in the result dictionary.
545
+ type: boolean
546
+ domain:
547
+ description: The network domain for the call, either None, 'wan' or 'lan'. None is fallback to 'wan', 'all' is both ['lan','wan']
548
+ type: string
549
+ signature_lifetime:
550
+ description: If supported, in seconds, restrict the lifetime of the signed PFN.
551
+ type: integer
552
+ resolve_archives:
553
+ description: When set to True, find archives which contain the replicas.
554
+ type: boolean
555
+ resolve_parents:
556
+ description: When set to True, find all parent datasets which contain the replicas.
557
+ type: boolean
558
+ updated_after:
559
+ description: datetime object (UTC time), only return replicas updated after this time
560
+ type: string
561
+ nrandom:
562
+ description: The maximum number of replicas to return.
563
+ type: integer
564
+ responses:
565
+ 200:
566
+ description: OK
567
+ content:
568
+ application/json:
569
+ schema:
570
+ type: array
571
+ items:
572
+ type: object
573
+ properties:
574
+ scope:
575
+ description: The scope of the replica.
576
+ type: string
577
+ name:
578
+ description: The name of the replica.
579
+ type: string
580
+ bytes:
581
+ description: The size of the replica in bytes.
582
+ type: integer
583
+ md5:
584
+ description: The md5 checksum.
585
+ type: string
586
+ adler32:
587
+ description: The adler32 checksum.
588
+ type: string
589
+ pfns:
590
+ description: The pfns.
591
+ type: array
592
+ rses:
593
+ description: The RSESs.
594
+ type: array
595
+ application/metalink4+xml:
596
+ schema:
597
+ type: object
598
+ properties:
599
+ scope:
600
+ description: The scope of the replica.
601
+ type: string
602
+ name:
603
+ description: The name of the replica.
604
+ type: string
605
+ bytes:
606
+ description: The size of the replica in bytes.
607
+ type: integer
608
+ md5:
609
+ description: The md5 checksum.
610
+ type: string
611
+ adler32:
612
+ description: The adler32 checksum.
613
+ type: string
614
+ pfns:
615
+ description: The pfns.
616
+ type: array
617
+ rses:
618
+ description: The RSESs.
619
+ type: array
620
+ 400:
621
+ description: Cannot decode json parameter list.
622
+ 401:
623
+ description: Invalid Auth Token
624
+ 404:
625
+ description: Did not found.
626
+ 406:
627
+ description: Not acceptable
628
+ """
629
+ content_type = request.accept_mimetypes.best_match(['application/x-json-stream', 'application/metalink4+xml'], 'application/x-json-stream')
630
+ metalink = (content_type == 'application/metalink4+xml')
631
+
632
+ client_ip = request.headers.get('X-Forwarded-For', default=request.remote_addr)
633
+
634
+ parameters = json_parameters(parse_response)
635
+
636
+ client_location = {'ip': client_ip,
637
+ 'fqdn': None,
638
+ 'site': None}
639
+ client_location.update(param_get(parameters, 'client_location', default={}))
640
+
641
+ # making sure IP address is not overwritten
642
+ client_location['ip'] = client_ip
643
+
644
+ dids = param_get(parameters, 'dids', default=[])
645
+ schemes = param_get(parameters, 'schemes', default=None)
646
+ select = param_get(parameters, 'sort', default=None)
647
+ unavailable = param_get(parameters, 'unavailable', default=False)
648
+ ignore_availability = param_get(parameters, 'ignore_availability', default='unavailable' in parameters)
649
+ rse_expression = param_get(parameters, 'rse_expression', default=None)
650
+ all_states = param_get(parameters, 'all_states', default=False)
651
+ domain = param_get(parameters, 'domain', default=None)
652
+ if 'signature_lifetime' in parameters:
653
+ signature_lifetime = param_get(parameters, 'signature_lifetime')
654
+ else:
655
+ # hardcoded default of 10 minutes if config is not parseable
656
+ signature_lifetime = config_get_int('credentials', 'signature_lifetime', raise_exception=False, default=600)
657
+ resolve_archives = param_get(parameters, 'resolve_archives', default=True)
658
+ resolve_parents = param_get(parameters, 'resolve_parents', default=False)
659
+ updated_after = param_get(parameters, 'updated_after', default=None)
660
+ if updated_after is not None:
661
+ if isinstance(updated_after, (int, float)):
662
+ # convert from epoch time stamp to datetime object
663
+ updated_after = datetime.utcfromtimestamp(updated_after)
664
+ else:
665
+ # attempt UTC format '%Y-%m-%dT%H:%M:%S' conversion
666
+ updated_after = datetime.strptime(updated_after, '%Y-%m-%dT%H:%M:%S')
667
+ nrandom = param_get(parameters, 'nrandom', default=None)
668
+ if nrandom:
669
+ nrandom = int(nrandom)
670
+
671
+ limit = request.args.get('limit', default=None)
672
+ select = request.args.get('select', default=select)
673
+ select = request.args.get('sort', default=select)
674
+
675
+ # Resolve all reasonable protocols when doing metalink for maximum access possibilities
676
+ if metalink and schemes is None:
677
+ schemes = SUPPORTED_PROTOCOLS
678
+
679
+ content_type = 'application/metalink4+xml' if metalink else 'application/x-json-stream'
680
+
681
+ try:
682
+ def _list_and_sort_replicas(request_id, issuer, vo):
683
+ # we need to call list_replicas before starting to reply
684
+ # otherwise the exceptions won't be propagated correctly
685
+ for rfile in list_replicas(dids=dids, schemes=schemes,
686
+ unavailable=unavailable,
687
+ request_id=request_id,
688
+ ignore_availability=ignore_availability,
689
+ all_states=all_states,
690
+ rse_expression=rse_expression,
691
+ client_location=client_location,
692
+ domain=domain,
693
+ signature_lifetime=signature_lifetime,
694
+ resolve_archives=resolve_archives,
695
+ resolve_parents=resolve_parents,
696
+ nrandom=nrandom,
697
+ updated_after=updated_after,
698
+ issuer=issuer,
699
+ vo=vo):
700
+
701
+ # Sort rfile['pfns'] and limit its size according to "limit" parameter
702
+ lanreplicas = {}
703
+ wanreplicas = {}
704
+ for pfn, replica in rfile['pfns'].items():
705
+ replica_tuple = (replica['domain'], replica['priority'], replica['rse'], replica['client_extract'])
706
+ if replica_tuple[0] == 'lan':
707
+ lanreplicas[pfn] = replica_tuple
708
+ else:
709
+ wanreplicas[pfn] = replica_tuple
710
+
711
+ rfile['pfns'] = dict(_sorted_with_priorities(replicas=rfile['pfns'],
712
+ # Lan replicas sorted by priority; followed by wan replicas sorted by selection criteria
713
+ sorted_pfns=chain(sorted(lanreplicas.keys(), key=lambda pfn: lanreplicas[pfn][1]),
714
+ sort_replicas(wanreplicas, client_location, selection=select)),
715
+ limit=limit))
716
+ yield rfile
717
+
718
+ rfiles = _list_and_sort_replicas(request_id=request.environ.get('request_id'),
719
+ issuer=request.environ.get('issuer'),
720
+ vo=request.environ.get('vo'))
721
+ if metalink:
722
+ policy_schema = config_get('policy', 'schema', raise_exception=False, default='generic')
723
+ response_generator = _generate_metalink_response(rfiles, policy_schema)
724
+ else:
725
+ response_generator = _generate_json_response(rfiles)
726
+ return try_stream(response_generator, content_type=content_type)
727
+ except InvalidObject as error:
728
+ return generate_http_error_flask(400, error)
729
+ except DataIdentifierNotFound as error:
730
+ return generate_http_error_flask(404, error)
731
+
732
+
733
+ class ReplicasDIDs(ErrorHandlingMethodView):
734
+
735
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
736
+ def post(self):
737
+ """
738
+ ---
739
+ summary: List Replicas Dids
740
+ description: List the DIDs associated to a list of replicas.
741
+ tags:
742
+ - Replicas
743
+ requestBody:
744
+ content:
745
+ application/json:
746
+ schema:
747
+ type: object
748
+ required:
749
+ - rse
750
+ properties:
751
+ pfns:
752
+ description: The list of pfns.
753
+ type: array
754
+ items:
755
+ type: string
756
+ rse:
757
+ description: The RSE name.
758
+ type: string
759
+ responses:
760
+ 200:
761
+ description: OK
762
+ content:
763
+ application/x-json-stream:
764
+ schema:
765
+ type: array
766
+ items:
767
+ type: object
768
+ additionalProperties:
769
+ x-additionalPropertiesName: mapped PFNs to DIDs
770
+ description: A mapping from a pfn to a did.
771
+ type: object
772
+ properties:
773
+ scope:
774
+ description: The scope of the DID.
775
+ type: string
776
+ name:
777
+ description: The name of the DID.
778
+ type: string
779
+ 400:
780
+ description: Cannot decode json parameter list.
781
+ 401:
782
+ description: Invalid Auth Token
783
+ 404:
784
+ description: Not found
785
+ 406:
786
+ description: Not acceptable
787
+ """
788
+ parameters = json_parameters()
789
+ pfns = param_get(parameters, 'pfns', default=[])
790
+ rse = param_get(parameters, 'rse')
791
+
792
+ try:
793
+ def generate(vo):
794
+ for pfn in get_did_from_pfns(pfns, rse, vo=vo):
795
+ yield dumps(pfn) + '\n'
796
+
797
+ return try_stream(generate(vo=request.environ.get('vo')))
798
+ except AccessDenied as error:
799
+ return generate_http_error_flask(401, error)
800
+
801
+
802
+ class BadReplicas(ErrorHandlingMethodView):
803
+
804
+ @check_accept_header_wrapper_flask(['application/json'])
805
+ def post(self):
806
+ """
807
+ ---
808
+ summary: Declare Bad Replicas
809
+ description: Declares a list of bad replicas.
810
+ tags:
811
+ - Replicas
812
+ requestBody:
813
+ content:
814
+ application/json:
815
+ schema:
816
+ type: object
817
+ properties:
818
+ replicas:
819
+ description: The list of pfns or list of dicts with "scope", "name", "rse_id"/"rse"
820
+ type: array
821
+ items:
822
+ type: string
823
+ pfns:
824
+ deprecated: true
825
+ description: The list of pfns, for backward compatibility with older versions of the ReplicaClient
826
+ type: array
827
+ items:
828
+ type: string
829
+ reason:
830
+ description: The reason for the declaration.
831
+ type: string
832
+ force:
833
+ description: If true, ignore existing replica status in the bad_replicas table.
834
+ type: boolean
835
+ responses:
836
+ 201:
837
+ description: OK
838
+ content:
839
+ application/json:
840
+ schema:
841
+ description: Returns the not declared files.
842
+ type: array
843
+ 400:
844
+ description: Can not decode json parameter list.
845
+ 404:
846
+ description: Not found
847
+ 406:
848
+ description: Not acceptable
849
+ """
850
+ parameters = json_parameters()
851
+ replicas = param_get(parameters, 'replicas', default=[]) or param_get(parameters, 'pfns', default=[])
852
+ reason = param_get(parameters, 'reason', default=None)
853
+ force = param_get(parameters, 'force', default=False)
854
+
855
+ try:
856
+ not_declared_files = declare_bad_file_replicas(replicas, reason=reason,
857
+ issuer=request.environ.get('issuer'), vo=request.environ.get('vo'),
858
+ force=force)
859
+ return not_declared_files, 201
860
+ except AccessDenied as error:
861
+ return generate_http_error_flask(401, error)
862
+ except (RSENotFound, ReplicaNotFound) as error:
863
+ return generate_http_error_flask(404, error)
864
+
865
+
866
+ class QuarantineReplicas(ErrorHandlingMethodView):
867
+
868
+ def post(self):
869
+ """
870
+ ---
871
+ summary: Quarantine replicas
872
+ description: Quarantine replicas.
873
+ tags:
874
+ - Replicas
875
+ requestBody:
876
+ content:
877
+ application/json:
878
+ schema:
879
+ type: object
880
+ required:
881
+ - replicas
882
+ properties:
883
+ replicas:
884
+ description: replicas
885
+ type: array
886
+ items:
887
+ type: object
888
+ required:
889
+ - path
890
+ properties:
891
+ path:
892
+ description: path
893
+ type: string
894
+ scope:
895
+ description: scope
896
+ type: string
897
+ name:
898
+ description: name
899
+ type: string
900
+ rse:
901
+ description: RSE name
902
+ type: string
903
+ rse_id:
904
+ description: RSE id
905
+ type: string
906
+ responses:
907
+ 200:
908
+ description: OK
909
+ 403:
910
+ description: Forbidden.
911
+ 404:
912
+ description: Not found
913
+ """
914
+
915
+ parameters = json_parameters()
916
+ replicas = param_get(parameters, 'replicas', default=[])
917
+ rse = param_get(parameters, 'rse', default=None)
918
+ rse_id = param_get(parameters, 'rse_id', default=None)
919
+ vo = request.environ.get('vo')
920
+ issuer = request.environ.get('issuer')
921
+
922
+ if replicas:
923
+ try:
924
+ quarantine_file_replicas(replicas, issuer, rse=rse, rse_id=rse_id, vo=vo)
925
+ except AccessDenied as error:
926
+ return generate_http_error_flask(403, error)
927
+ except (RSENotFound, ReplicaNotFound) as error:
928
+ return generate_http_error_flask(404, error)
929
+
930
+ return '', 200
931
+
932
+
933
+ class SuspiciousReplicas(ErrorHandlingMethodView):
934
+
935
+ @check_accept_header_wrapper_flask(['application/json'])
936
+ def post(self):
937
+ """
938
+ ---
939
+ summary: Declare Suspicious Replicas
940
+ description: Declare a list of suspicious replicas.
941
+ tags:
942
+ - Replicas
943
+ requestBody:
944
+ content:
945
+ application/json:
946
+ schema:
947
+ type: object
948
+ properties:
949
+ pfns:
950
+ description: The list of pfns.
951
+ type: array
952
+ items:
953
+ type: string
954
+ reason:
955
+ description: The reason for the declaration.
956
+ type: string
957
+ responses:
958
+ 201:
959
+ description: OK
960
+ content:
961
+ application/json:
962
+ schema:
963
+ description: Returns the not declared files.
964
+ type: array
965
+ 400:
966
+ description: Can not decode json parameter list.
967
+ 404:
968
+ description: Not found
969
+ 406:
970
+ description: Not acceptable
971
+ """
972
+ parameters = json_parameters(parse_response)
973
+ pfns = param_get(parameters, 'pfns', default=[])
974
+ reason = param_get(parameters, 'reason', default=None)
975
+
976
+ try:
977
+ not_declared_files = declare_suspicious_file_replicas(pfns=pfns, reason=reason, issuer=request.environ.get('issuer'), vo=request.environ.get('vo'))
978
+ return not_declared_files, 201
979
+ except AccessDenied as error:
980
+ return generate_http_error_flask(401, error)
981
+
982
+ @check_accept_header_wrapper_flask(['application/json'])
983
+ def get(self):
984
+ """
985
+ ---
986
+ summary: List Suspicious Replicas
987
+ description: List the suspicious replicas on a list of RSEs.
988
+ tags:
989
+ - Replicas
990
+ parameters:
991
+ - name: rse_expression
992
+ in: query
993
+ description: The RSE expression to filter for.
994
+ schema:
995
+ type: string
996
+ - name: younger_than
997
+ in: query
998
+ description: Date to filter for.
999
+ schema:
1000
+ type: string
1001
+ - name: nattempts
1002
+ in: query
1003
+ description: The maximum number of attempts to make.
1004
+ schema:
1005
+ type: integer
1006
+ responses:
1007
+ 200:
1008
+ description: OK
1009
+ content:
1010
+ application/json:
1011
+ schema:
1012
+ type: array
1013
+ items:
1014
+ type: object
1015
+ properties:
1016
+ scope:
1017
+ description: The scope of the Replica.
1018
+ type: string
1019
+ name:
1020
+ description: The name of the Replica.
1021
+ type: string
1022
+ rse:
1023
+ description: The rse name.
1024
+ type: string
1025
+ rse_id:
1026
+ description: The id of the rse.
1027
+ type: string
1028
+ cnt:
1029
+ description: The number of replicas.
1030
+ type: integer
1031
+ created_at:
1032
+ description: The time when the replica was created.
1033
+ type: string
1034
+ 401:
1035
+ description: Invalid Auth Token
1036
+ 404:
1037
+ description: Not found
1038
+ 406:
1039
+ description: Not acceptable
1040
+ """
1041
+ rse_expression, younger_than, nattempts = None, None, None
1042
+ if request.query_string:
1043
+ query_string = request.query_string.decode(encoding='utf-8')
1044
+ try:
1045
+ params = loads(unquote(query_string))
1046
+ except ValueError:
1047
+ params = parse_qs(query_string)
1048
+
1049
+ if 'rse_expression' in params:
1050
+ rse_expression = params['rse_expression'][0]
1051
+ if 'younger_than' in params and params['younger_than'][0]:
1052
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S")
1053
+ if 'nattempts' in params:
1054
+ nattempts = int(params['nattempts'][0])
1055
+
1056
+ result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
1057
+ return Response(render_json_list(result), 200, content_type='application/json')
1058
+
1059
+
1060
+ class BadReplicasStates(ErrorHandlingMethodView):
1061
+
1062
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1063
+ def get(self):
1064
+ """
1065
+ ---
1066
+ summary: List Bad Replicas By States
1067
+ description: List the bad or suspicious replicas by states.
1068
+ tags:
1069
+ - Replicas
1070
+ parameters:
1071
+ - name: state
1072
+ in: query
1073
+ description: The state of the file.
1074
+ schema:
1075
+ type: string
1076
+ enum: [SUSPICIOUS, BAD]
1077
+ - name: rse
1078
+ in: query
1079
+ description: The rse name.
1080
+ schema:
1081
+ type: string
1082
+ - name: younger_than
1083
+ in: query
1084
+ description: Date to select bad replicas younger than this date.
1085
+ schema:
1086
+ type: string
1087
+ format: date-time
1088
+ - name: older_than
1089
+ in: query
1090
+ description: Date to select bad replicas older than this date.
1091
+ schema:
1092
+ type: string
1093
+ format: date-time
1094
+ - name: limit
1095
+ in: query
1096
+ description: The maximum number of replicas returned.
1097
+ schema:
1098
+ type: integer
1099
+ - name: list_pfns
1100
+ in: query
1101
+ description: Flag to include pfns.
1102
+ schema:
1103
+ type: boolean
1104
+ responses:
1105
+ 200:
1106
+ description: OK
1107
+ content:
1108
+ application/x-json-stream:
1109
+ schema:
1110
+ description: A list of all result replicas.
1111
+ type: array
1112
+ items:
1113
+ oneOf:
1114
+ - type: object
1115
+ properties:
1116
+ scope:
1117
+ description: The scope fo the replica.
1118
+ type: string
1119
+ name:
1120
+ description: The name of the replica.
1121
+ type: string
1122
+ type:
1123
+ description: The type of the replica.
1124
+ type: string
1125
+ - type: object
1126
+ properties:
1127
+ scope:
1128
+ description: The scope fo the replica.
1129
+ type: string
1130
+ name:
1131
+ description: The name of the replica.
1132
+ type: string
1133
+ rse:
1134
+ description: The name of the associated rse.
1135
+ type: string
1136
+ rse_id:
1137
+ description: The id of the associated rse.
1138
+ type: string
1139
+ state:
1140
+ description: The state of the replica.
1141
+ type: string
1142
+ created_at:
1143
+ description: The date-time the replica was created.
1144
+ type: string
1145
+ format: date-time
1146
+ updated_at:
1147
+ description: The date-time the replica was updated.
1148
+ type: string
1149
+ format: date-time
1150
+ 401:
1151
+ description: Invalid Auth Token
1152
+ 406:
1153
+ description: Not acceptable
1154
+ """
1155
+ state, rse, younger_than, older_than, limit, list_pfns = None, None, None, None, None, None
1156
+ if request.query_string:
1157
+ query_string = request.query_string.decode(encoding='utf-8')
1158
+ try:
1159
+ params = loads(unquote(query_string))
1160
+ except ValueError:
1161
+ params = parse_qs(query_string)
1162
+ if 'state' in params:
1163
+ state = params['state'][0]
1164
+ if isinstance(state, str):
1165
+ state = BadFilesStatus(state)
1166
+ if 'rse' in params:
1167
+ rse = params['rse'][0]
1168
+ if 'younger_than' in params:
1169
+ younger_than = datetime.strptime(params['younger_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1170
+ if 'older_than' in params and params['older_than']:
1171
+ older_than = datetime.strptime(params['older_than'][0], "%Y-%m-%dT%H:%M:%S.%f")
1172
+ if 'limit' in params:
1173
+ limit = int(params['limit'][0])
1174
+ if 'list_pfns' in params:
1175
+ list_pfns = bool(params['list_pfns'][0])
1176
+
1177
+ def generate(vo):
1178
+ for row in list_bad_replicas_status(state=state, rse=rse, younger_than=younger_than,
1179
+ older_than=older_than, limit=limit, list_pfns=list_pfns,
1180
+ vo=vo):
1181
+ yield dumps(row, cls=APIEncoder) + '\n'
1182
+
1183
+ return try_stream(generate(vo=request.environ.get('vo')))
1184
+
1185
+
1186
+ class BadReplicasSummary(ErrorHandlingMethodView):
1187
+
1188
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1189
+ def get(self):
1190
+ """
1191
+ ---
1192
+ summary: Bad Replicas Summary
1193
+ description: Return a summary of the bad replicas by incident.
1194
+ tags:
1195
+ - Replicas
1196
+ parameters:
1197
+ - name: rse_expression
1198
+ in: query
1199
+ description: The RSE expression.
1200
+ schema:
1201
+ type: string
1202
+ - name: from_date
1203
+ in: query
1204
+ description: The start date.
1205
+ schema:
1206
+ type: string
1207
+ format: date-time
1208
+ - name: to_date
1209
+ in: query
1210
+ description: The end date.
1211
+ schema:
1212
+ type: string
1213
+ format: date-time
1214
+ responses:
1215
+ 200:
1216
+ description: OK
1217
+ content:
1218
+ application/x-json-stream:
1219
+ schema:
1220
+ description: A list of summaries.
1221
+ type: array
1222
+ items:
1223
+ type: object
1224
+ properties:
1225
+ rse:
1226
+ description: The name of the associated RSE.
1227
+ type: string
1228
+ rse_id:
1229
+ description: The id of the associated RSE.
1230
+ type: string
1231
+ created_at:
1232
+ description: The creation date-time.
1233
+ type: string
1234
+ format: date-time
1235
+ reason:
1236
+ description: The reason for the incident.
1237
+ type: string
1238
+ 401:
1239
+ description: Invalid Auth Token
1240
+ 406:
1241
+ description: Not acceptable
1242
+ """
1243
+ rse_expression, from_date, to_date = None, None, None
1244
+ if request.query_string:
1245
+ query_string = request.query_string.decode(encoding='utf-8')
1246
+ try:
1247
+ params = loads(unquote(query_string))
1248
+ except ValueError:
1249
+ params = parse_qs(query_string)
1250
+ if 'rse_expression' in params:
1251
+ rse_expression = params['rse_expression'][0]
1252
+ if 'from_date' in params and params['from_date'][0]:
1253
+ from_date = datetime.strptime(params['from_date'][0], "%Y-%m-%d")
1254
+ if 'to_date' in params:
1255
+ to_date = datetime.strptime(params['to_date'][0], "%Y-%m-%d")
1256
+
1257
+ def generate(vo):
1258
+ for row in get_bad_replicas_summary(rse_expression=rse_expression, from_date=from_date,
1259
+ to_date=to_date, vo=vo):
1260
+ yield dumps(row, cls=APIEncoder) + '\n'
1261
+
1262
+ return try_stream(generate(vo=request.environ.get('vo')))
1263
+
1264
+
1265
+ class DatasetReplicas(ErrorHandlingMethodView):
1266
+
1267
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1268
+ def get(self, scope_name):
1269
+ """
1270
+ ---
1271
+ summary: List Dataset Replicas
1272
+ description: List dataset replicas.
1273
+ tags:
1274
+ - Replicas
1275
+ parameters:
1276
+ - name: scope_name
1277
+ in: path
1278
+ description: data identifier (scope)/(name).
1279
+ schema:
1280
+ type: string
1281
+ style: simple
1282
+ - name: deep
1283
+ in: query
1284
+ description: Flag to ennable lookup at the file level.
1285
+ schema:
1286
+ type: boolean
1287
+ responses:
1288
+ 200:
1289
+ description: OK
1290
+ content:
1291
+ application/x-json-stream:
1292
+ schema:
1293
+ description: A list of dataset replicas.
1294
+ type: array
1295
+ items:
1296
+ type: object
1297
+ properties:
1298
+ scope:
1299
+ description: The scope of the replica.
1300
+ type: string
1301
+ name:
1302
+ description: The name of the replica.
1303
+ type: string
1304
+ rse:
1305
+ description: The name of the associated RSE.
1306
+ type: string
1307
+ rse_id:
1308
+ description: The id of the associated RSE.
1309
+ type: string
1310
+ bytes:
1311
+ description: The size of the replica.
1312
+ type: integer
1313
+ length:
1314
+ description: The length of the replica.
1315
+ type: integer
1316
+ available_bytes:
1317
+ description: The number of available bytes of the replica.
1318
+ type: integer
1319
+ available_length:
1320
+ description: The available length of the replica.
1321
+ type: integer
1322
+ state:
1323
+ description: The state of the replica.
1324
+ type: string
1325
+ created_at:
1326
+ description: The date-time the replica was created.
1327
+ type: string
1328
+ format: date-time
1329
+ updated_at:
1330
+ description: The date-time the replica was updated.
1331
+ type: string
1332
+ format: date-time
1333
+ accessed_at:
1334
+ description: The date-time the replica was accessed.
1335
+ type: string
1336
+ format: date-time
1337
+ 401:
1338
+ description: Invalid Auth Token
1339
+ 404:
1340
+ description: Not found
1341
+ 406:
1342
+ description: Not acceptable
1343
+ """
1344
+ try:
1345
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1346
+
1347
+ def generate(_deep, vo):
1348
+ for row in list_dataset_replicas(scope=scope, name=name, deep=_deep, vo=vo):
1349
+ yield dumps(row, cls=APIEncoder) + '\n'
1350
+
1351
+ deep = request.args.get('deep', default=False)
1352
+
1353
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1354
+ except ValueError as error:
1355
+ return generate_http_error_flask(400, error)
1356
+
1357
+
1358
+ class DatasetReplicasBulk(ErrorHandlingMethodView):
1359
+
1360
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1361
+ def post(self):
1362
+ """
1363
+ ---
1364
+ summary: List Dataset Replicas for Multiple DIDs
1365
+ description: List dataset replicas for multiple dids.
1366
+ tags:
1367
+ - Replicas
1368
+ requestBody:
1369
+ content:
1370
+ application/json:
1371
+ schema:
1372
+ type: object
1373
+ required:
1374
+ - dids
1375
+ properties:
1376
+ dids:
1377
+ description: A list of dids.
1378
+ type: array
1379
+ items:
1380
+ description: A did.
1381
+ type: object
1382
+ properties:
1383
+ scope:
1384
+ description: The scope of the did.
1385
+ type: string
1386
+ name:
1387
+ description: The name of the did.
1388
+ type: string
1389
+ responses:
1390
+ 200:
1391
+ description: OK
1392
+ content:
1393
+ application/x-json-stream:
1394
+ schema:
1395
+ description: A list of dataset replicas.
1396
+ type: array
1397
+ items:
1398
+ type: object
1399
+ properties:
1400
+ scope:
1401
+ description: The scope of the replica.
1402
+ type: string
1403
+ name:
1404
+ description: The name of the replica.
1405
+ type: string
1406
+ rse:
1407
+ description: The name of the associated RSE.
1408
+ type: string
1409
+ rse_id:
1410
+ description: The id of the associated RSE.
1411
+ type: string
1412
+ bytes:
1413
+ description: The size of the replica.
1414
+ type: integer
1415
+ length:
1416
+ description: The length of the replica.
1417
+ type: integer
1418
+ available_bytes:
1419
+ description: The number of available bytes of the replica.
1420
+ type: integer
1421
+ available_length:
1422
+ description: The available length of the replica.
1423
+ type: integer
1424
+ state:
1425
+ description: The state of the replica.
1426
+ type: string
1427
+ created_at:
1428
+ description: The date-time the replica was created.
1429
+ type: string
1430
+ format: date-time
1431
+ updated_at:
1432
+ description: The date-time the replica was updated.
1433
+ type: string
1434
+ format: date-time
1435
+ accessed_at:
1436
+ description: The date-time the replica was accessed.
1437
+ type: string
1438
+ format: date-time
1439
+ 400:
1440
+ description: Bad Request.
1441
+ 401:
1442
+ description: Invalid Auth Token
1443
+ 404:
1444
+ description: Not found
1445
+ 406:
1446
+ description: Not acceptable
1447
+ """
1448
+ parameters = json_parameters(parse_response)
1449
+ dids = param_get(parameters, 'dids')
1450
+ if len(dids) == 0:
1451
+ return generate_http_error_flask(400, ValueError.__name__, 'List of DIDs is empty')
1452
+
1453
+ try:
1454
+ def generate(vo):
1455
+ for row in list_dataset_replicas_bulk(dids=dids, vo=vo):
1456
+ yield dumps(row, cls=APIEncoder) + '\n'
1457
+
1458
+ return try_stream(generate(vo=request.environ.get('vo')))
1459
+ except InvalidObject as error:
1460
+ return generate_http_error_flask(400, error, f'Cannot validate DIDs: {error}')
1461
+
1462
+
1463
+ class DatasetReplicasVP(ErrorHandlingMethodView):
1464
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1465
+ def get(self, scope_name):
1466
+ """
1467
+ ---
1468
+ summary: List Dataset Replicas VP
1469
+ description: |
1470
+ List dataset replicas using the Virtual Placement service.
1471
+ This is an RnD function and might change or go away at any time.
1472
+ tags:
1473
+ - Replicas
1474
+ parameters:
1475
+ - name: scope_name
1476
+ in: path
1477
+ description: data identifier (scope)/(name).
1478
+ schema:
1479
+ type: string
1480
+ style: simple
1481
+ - name: deep
1482
+ in: query
1483
+ description: Flag to ennable lookup at the file level.
1484
+ schema:
1485
+ type: boolean
1486
+ responses:
1487
+ 200:
1488
+ description: OK. This needs documentation!
1489
+ 401:
1490
+ description: Invalid Auth Token
1491
+ 406:
1492
+ description: Not acceptable
1493
+ """
1494
+ try:
1495
+ scope, name = parse_scope_name(scope_name, request.environ.get('vo'))
1496
+
1497
+ def generate(_deep, vo):
1498
+ for row in list_dataset_replicas_vp(scope=scope, name=name, deep=_deep, vo=vo):
1499
+ yield dumps(row, cls=APIEncoder) + '\n'
1500
+
1501
+ deep = request.args.get('deep', default=False)
1502
+
1503
+ return try_stream(generate(_deep=deep, vo=request.environ.get('vo')))
1504
+ except ValueError as error:
1505
+ return generate_http_error_flask(400, error)
1506
+
1507
+
1508
+ class ReplicasRSE(ErrorHandlingMethodView):
1509
+
1510
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
1511
+ def get(self, rse):
1512
+ """
1513
+ ---
1514
+ summary: List Dataset Replicas per RSE
1515
+ description: List dataset replicas per RSE.
1516
+ tags:
1517
+ - Replicas
1518
+ parameters:
1519
+ - name: rse
1520
+ in: path
1521
+ description: The rse to filter for.
1522
+ schema:
1523
+ type: string
1524
+ style: simple
1525
+ responses:
1526
+ 200:
1527
+ description: OK
1528
+ content:
1529
+ application/x-json-stream:
1530
+ schema:
1531
+ description: A list of dataset replicas.
1532
+ type: array
1533
+ items:
1534
+ type: object
1535
+ properties:
1536
+ scope:
1537
+ description: The scope of the replica.
1538
+ type: string
1539
+ name:
1540
+ description: The name of the replica.
1541
+ type: string
1542
+ rse:
1543
+ description: The name of the associated RSE.
1544
+ type: string
1545
+ rse_id:
1546
+ description: The id of the associated RSE.
1547
+ type: string
1548
+ bytes:
1549
+ description: The size of the replica.
1550
+ type: integer
1551
+ length:
1552
+ description: The length of the replica.
1553
+ type: integer
1554
+ available_bytes:
1555
+ description: The number of available bytes of the replica.
1556
+ type: integer
1557
+ available_length:
1558
+ description: The available length of the replica.
1559
+ type: integer
1560
+ state:
1561
+ description: The state of the replica.
1562
+ type: string
1563
+ created_at:
1564
+ description: The date-time the replica was created.
1565
+ type: string
1566
+ format: date-time
1567
+ updated_at:
1568
+ description: The date-time the replica was updated.
1569
+ type: string
1570
+ format: date-time
1571
+ accessed_at:
1572
+ description: The date-time the replica was accessed.
1573
+ type: string
1574
+ format: date-time
1575
+ 401:
1576
+ description: Invalid Auth Token
1577
+ 406:
1578
+ description: Not acceptable
1579
+ """
1580
+
1581
+ def generate(vo):
1582
+ for row in list_datasets_per_rse(rse=rse, vo=vo):
1583
+ yield dumps(row, cls=APIEncoder) + '\n'
1584
+
1585
+ return try_stream(generate(vo=request.environ.get('vo')))
1586
+
1587
+
1588
+ class BadDIDs(ErrorHandlingMethodView):
1589
+
1590
+ def post(self):
1591
+ """
1592
+ ---
1593
+ summary: Mark Bad by DID
1594
+ description: Declare a list of bad replicas by DID.
1595
+ tags:
1596
+ - Replicas
1597
+ requestBody:
1598
+ content:
1599
+ application/json:
1600
+ schema:
1601
+ type: object
1602
+ properties:
1603
+ expires_at:
1604
+ description: The expires at value.
1605
+ type: string
1606
+ format: date-time
1607
+ dids:
1608
+ description: The list of dids associated with the bad replicas.
1609
+ type: array
1610
+ items:
1611
+ type: object
1612
+ properties:
1613
+ scope:
1614
+ description: The scope of the did.
1615
+ type: string
1616
+ name:
1617
+ description: The name of the did.
1618
+ type: string
1619
+ rse:
1620
+ description: The name of the rse.
1621
+ type: string
1622
+ reason:
1623
+ description: The reason for the change.
1624
+ type: string
1625
+ responses:
1626
+ 201:
1627
+ description: OK
1628
+ content:
1629
+ application/json:
1630
+ schema:
1631
+ description: All files not declared as bad.
1632
+ type: array
1633
+ items:
1634
+ type: string
1635
+ 400:
1636
+ description: Cannot decode json parameter list.
1637
+ 401:
1638
+ description: Invalid Auth Token
1639
+ 404:
1640
+ description: Not found
1641
+ """
1642
+ parameters = json_parameters(parse_response)
1643
+ expires_at = param_get(parameters, 'expires_at', default=None)
1644
+ if expires_at:
1645
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1646
+
1647
+ try:
1648
+ not_declared_files = add_bad_dids(
1649
+ dids=param_get(parameters, 'dids', default=[]),
1650
+ rse=param_get(parameters, 'rse', default=None),
1651
+ issuer=request.environ.get('issuer'),
1652
+ state=BadFilesStatus.BAD,
1653
+ reason=param_get(parameters, 'reason', default=None),
1654
+ expires_at=expires_at,
1655
+ vo=request.environ.get('vo'),
1656
+ )
1657
+ except (ValueError, InvalidType) as error:
1658
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1659
+ except AccessDenied as error:
1660
+ return generate_http_error_flask(401, error)
1661
+ except ReplicaNotFound as error:
1662
+ return generate_http_error_flask(404, error)
1663
+ except Duplicate as error:
1664
+ return generate_http_error_flask(409, error)
1665
+
1666
+ return Response(dumps(not_declared_files), status=201, content_type='application/json')
1667
+
1668
+
1669
+ class BadPFNs(ErrorHandlingMethodView):
1670
+
1671
+ def post(self):
1672
+ """
1673
+ ---
1674
+ summary: Declare Bad PFNs
1675
+ description: Declare a list of bad PFNs.
1676
+ tags:
1677
+ - Replicas
1678
+ requestBody:
1679
+ content:
1680
+ application/json:
1681
+ schema:
1682
+ type: object
1683
+ properties:
1684
+ expires_at:
1685
+ description: The expires at value. Only apply to TEMPORARY_UNAVAILABLE.
1686
+ type: string
1687
+ format: date-time
1688
+ pfns:
1689
+ description: The list of pfns associated with the bad PFNs.
1690
+ type: array
1691
+ items:
1692
+ type: string
1693
+ state:
1694
+ description: The state to set the PFNs to.
1695
+ type: string
1696
+ enum: ["BAD", "SUSPICIOUS", "TEMPORARY_UNAVAILABLE"]
1697
+ reason:
1698
+ description: The reason for the change.
1699
+ type: string
1700
+ responses:
1701
+ 201:
1702
+ description: Created
1703
+ 400:
1704
+ description: Cannot decode json parameter list.
1705
+ 401:
1706
+ description: Invalid Auth Token
1707
+ 404:
1708
+ description: Replica not found
1709
+ 409:
1710
+ description: Duplicate
1711
+ """
1712
+ parameters = json_parameters(parse_response)
1713
+ expires_at = param_get(parameters, 'expires_at', default=None)
1714
+ if expires_at:
1715
+ expires_at = datetime.strptime(expires_at, "%Y-%m-%dT%H:%M:%S.%f")
1716
+
1717
+ try:
1718
+ add_bad_pfns(
1719
+ pfns=param_get(parameters, 'pfns', default=[]),
1720
+ issuer=request.environ.get('issuer'),
1721
+ state=param_get(parameters, 'state', default=None),
1722
+ reason=param_get(parameters, 'reason', default=None),
1723
+ expires_at=expires_at,
1724
+ vo=request.environ.get('vo'),
1725
+ )
1726
+ except (ValueError, InvalidType) as error:
1727
+ return generate_http_error_flask(400, ValueError.__name__, error.args[0])
1728
+ except AccessDenied as error:
1729
+ return generate_http_error_flask(401, error)
1730
+ except ReplicaNotFound as error:
1731
+ return generate_http_error_flask(404, error)
1732
+ except Duplicate as error:
1733
+ return generate_http_error_flask(409, error)
1734
+
1735
+ return 'Created', 201
1736
+
1737
+
1738
+ class Tombstone(ErrorHandlingMethodView):
1739
+
1740
+ def post(self):
1741
+ """
1742
+ ---
1743
+ summary: Set Tombstone
1744
+ description: Set a tombstone on a list of replicas.
1745
+ tags:
1746
+ - Replicas
1747
+ requestBody:
1748
+ content:
1749
+ application/json:
1750
+ schema:
1751
+ type: object
1752
+ properties:
1753
+ replicas:
1754
+ description: The replicas to set the tombstone to.
1755
+ type: array
1756
+ items:
1757
+ type: object
1758
+ required:
1759
+ - rse
1760
+ - scope
1761
+ - name
1762
+ properties:
1763
+ rse:
1764
+ description: The rse associated with the tombstone.
1765
+ type: string
1766
+ scope:
1767
+ description: The scope of the replica
1768
+ type: string
1769
+ name:
1770
+ description: The name of the replica.
1771
+ type: string
1772
+ responses:
1773
+ 201:
1774
+ description: Created
1775
+ 401:
1776
+ description: Invalid Auth Token
1777
+ 404:
1778
+ description: Not found
1779
+ 423:
1780
+ description: Replica is locked.
1781
+ """
1782
+ parameters = json_parameters(parse_response)
1783
+ replicas = param_get(parameters, 'replicas', default=[])
1784
+
1785
+ try:
1786
+ for replica in replicas:
1787
+ set_tombstone(
1788
+ rse=replica['rse'],
1789
+ scope=replica['scope'],
1790
+ name=replica['name'],
1791
+ issuer=request.environ.get('issuer'),
1792
+ vo=request.environ.get('vo'),
1793
+ )
1794
+ except ReplicaNotFound as error:
1795
+ return generate_http_error_flask(404, error)
1796
+ except ReplicaIsLocked as error:
1797
+ return generate_http_error_flask(423, error)
1798
+
1799
+ return 'Created', 201
1800
+
1801
+
1802
+ def blueprint(with_doc=False):
1803
+ bp = AuthenticatedBlueprint('replicas', __name__, url_prefix='/replicas')
1804
+
1805
+ list_replicas_view = ListReplicas.as_view('list_replicas')
1806
+ bp.add_url_rule('/list', view_func=list_replicas_view, methods=['post', ])
1807
+ replicas_view = Replicas.as_view('replicas')
1808
+ if not with_doc:
1809
+ # rule without trailing slash needs to be added before rule with trailing slash
1810
+ bp.add_url_rule('', view_func=replicas_view, methods=['post', 'put', 'delete'])
1811
+ bp.add_url_rule('/', view_func=replicas_view, methods=['post', 'put', 'delete'])
1812
+ suspicious_replicas_view = SuspiciousReplicas.as_view('suspicious_replicas')
1813
+ bp.add_url_rule('/suspicious', view_func=suspicious_replicas_view, methods=['get', 'post'])
1814
+ bad_replicas_states_view = BadReplicasStates.as_view('bad_replicas_states')
1815
+ bp.add_url_rule('/bad/states', view_func=bad_replicas_states_view, methods=['get', ])
1816
+ bad_replicas_summary_view = BadReplicasSummary.as_view('bad_replicas_summary')
1817
+ bp.add_url_rule('/bad/summary', view_func=bad_replicas_summary_view, methods=['get', ])
1818
+ bad_replicas_pfn_view = BadPFNs.as_view('add_bad_pfns')
1819
+ bp.add_url_rule('/bad/pfns', view_func=bad_replicas_pfn_view, methods=['post', ])
1820
+ bad_replicas_dids_view = BadDIDs.as_view('add_bad_dids')
1821
+ bp.add_url_rule('/bad/dids', view_func=bad_replicas_dids_view, methods=['post', ])
1822
+ replicas_rse_view = ReplicasRSE.as_view('replicas_rse')
1823
+ bp.add_url_rule('/rse/<rse>', view_func=replicas_rse_view, methods=['get', ])
1824
+
1825
+ bad_replicas_view = BadReplicas.as_view('bad_replicas')
1826
+ bp.add_url_rule('/bad', view_func=bad_replicas_view, methods=['post', ])
1827
+
1828
+ quarantine_replicas_view = QuarantineReplicas.as_view('quarantine_replicas')
1829
+ bp.add_url_rule('/quarantine', view_func=quarantine_replicas_view, methods=['post', ])
1830
+
1831
+ replicas_dids_view = ReplicasDIDs.as_view('replicas_dids')
1832
+ bp.add_url_rule('/dids', view_func=replicas_dids_view, methods=['post', ])
1833
+ dataset_replicas_view = DatasetReplicas.as_view('dataset_replicas')
1834
+ bp.add_url_rule('/<path:scope_name>/datasets', view_func=dataset_replicas_view, methods=['get', ])
1835
+ dataset_replicas_bulk_view = DatasetReplicasBulk.as_view('dataset_replicas_bulk')
1836
+ bp.add_url_rule('/datasets_bulk', view_func=dataset_replicas_bulk_view, methods=['post', ])
1837
+ dataset_replicas_vp_view = DatasetReplicasVP.as_view('dataset_replicas_vp')
1838
+ bp.add_url_rule('/<path:scope_name>', view_func=replicas_view, methods=['get', ])
1839
+ set_tombstone_view = Tombstone.as_view('set_tombstone')
1840
+ bp.add_url_rule('/tombstone', view_func=set_tombstone_view, methods=['post', ])
1841
+
1842
+ if not with_doc:
1843
+ bp.add_url_rule('/list/', view_func=list_replicas_view, methods=['post', ])
1844
+ bp.add_url_rule('/suspicious/', view_func=suspicious_replicas_view, methods=['get', 'post'])
1845
+ bp.add_url_rule('/bad/states/', view_func=bad_replicas_states_view, methods=['get', ])
1846
+ bp.add_url_rule('/bad/summary/', view_func=bad_replicas_summary_view, methods=['get', ])
1847
+ bp.add_url_rule('/bad/pfns/', view_func=bad_replicas_pfn_view, methods=['post', ])
1848
+ bp.add_url_rule('/bad/dids/', view_func=bad_replicas_dids_view, methods=['post', ])
1849
+ bp.add_url_rule('/rse/<rse>/', view_func=replicas_rse_view, methods=['get', ])
1850
+ bp.add_url_rule('/bad/', view_func=bad_replicas_view, methods=['post', ])
1851
+ bp.add_url_rule('/dids/', view_func=replicas_dids_view, methods=['post', ])
1852
+ bp.add_url_rule('/datasets_bulk/', view_func=dataset_replicas_bulk_view, methods=['post', ])
1853
+ bp.add_url_rule('/<path:scope_name>/datasets_vp', view_func=dataset_replicas_vp_view, methods=['get', ])
1854
+ bp.add_url_rule('/<path:scope_name>/', view_func=replicas_view, methods=['get', ])
1855
+ bp.add_url_rule('/tombstone/', view_func=set_tombstone_view, methods=['post', ])
1856
+ bp.add_url_rule('/quarantine/', view_func=quarantine_replicas_view, methods=['post', ])
1857
+
1858
+ bp.after_request(response_headers)
1859
+ return bp
1860
+
1861
+
1862
+ def make_doc():
1863
+ """ Only used for sphinx documentation """
1864
+ doc_app = Flask(__name__)
1865
+ doc_app.register_blueprint(blueprint(with_doc=True))
1866
+ return doc_app