rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,13 +14,11 @@
15
14
 
16
15
  from flask import Flask, request
17
16
 
18
- from rucio.api.dirac import add_files
19
- from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, DatabaseException, \
20
- Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, UnsupportedOperation
17
+ from rucio.common.exception import AccessDenied, DatabaseException, DataIdentifierAlreadyExists, Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, UnsupportedOperation
21
18
  from rucio.common.utils import parse_response
19
+ from rucio.gateway.dirac import add_files
22
20
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
23
- from rucio.web.rest.flaskapi.v1.common import response_headers, generate_http_error_flask, \
24
- ErrorHandlingMethodView, json_parameters, param_get
21
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, generate_http_error_flask, json_parameters, param_get, response_headers
25
22
 
26
23
 
27
24
  class AddFiles(ErrorHandlingMethodView):
@@ -54,6 +51,9 @@ class AddFiles(ErrorHandlingMethodView):
54
51
  ignore_availability:
55
52
  description: If the availability should be ignored.
56
53
  type: boolean
54
+ parents_metadata:
55
+ description: "Metadata for selected hierarchy DIDs."
56
+ type: object
57
57
  responses:
58
58
  201:
59
59
  description: OK
@@ -78,10 +78,10 @@ class AddFiles(ErrorHandlingMethodView):
78
78
  parameters = json_parameters(parse_response)
79
79
  lfns = param_get(parameters, 'lfns')
80
80
  ignore_availability = param_get(parameters, 'ignore_availability', default=False)
81
-
81
+ parents_metadata = param_get(parameters, 'parents_metadata', default=None)
82
82
  try:
83
83
  add_files(lfns=lfns, issuer=request.environ.get('issuer'), ignore_availability=ignore_availability,
84
- vo=request.environ.get('vo'))
84
+ parents_metadata=parents_metadata, vo=request.environ.get('vo'))
85
85
  except InvalidPath as error:
86
86
  return generate_http_error_flask(400, error)
87
87
  except AccessDenied as error:
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,12 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from flask import Flask, request, Response
15
+ from flask import Flask, Response, request
17
16
 
18
- from rucio.api.exporter import export_data
19
17
  from rucio.common.utils import render_json
18
+ from rucio.gateway.exporter import export_data
20
19
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
21
- from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
22
- ErrorHandlingMethodView
20
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers
23
21
 
24
22
 
25
23
  class Export(ErrorHandlingMethodView):
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,12 +16,11 @@ import json
17
16
 
18
17
  from flask import Flask, Response, request
19
18
 
20
- from rucio.api.heartbeat import list_heartbeats, create_heartbeat
21
- from rucio.common.exception import UnsupportedValueType, UnsupportedKeyType, KeyNotFound, AccessDenied
19
+ from rucio.common.exception import AccessDenied, KeyNotFound, UnsupportedKeyType, UnsupportedValueType
22
20
  from rucio.common.utils import APIEncoder
21
+ from rucio.gateway.heartbeat import create_heartbeat, list_heartbeats
23
22
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
24
- from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
25
- ErrorHandlingMethodView, json_parameters, param_get, generate_http_error_flask
23
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers
26
24
 
27
25
 
28
26
  class Heartbeat(ErrorHandlingMethodView):
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,12 +12,11 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from flask import Flask, request, jsonify
15
+ from flask import Flask, jsonify, request
17
16
 
18
- from rucio.api.identity import add_identity, add_account_identity, list_accounts_for_identity
17
+ from rucio.gateway.identity import add_account_identity, add_identity, list_accounts_for_identity
19
18
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
20
- from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
21
- ErrorHandlingMethodView
19
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers
22
20
 
23
21
 
24
22
  class UserPass(ErrorHandlingMethodView):
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +14,10 @@
15
14
 
16
15
  from flask import Flask, request
17
16
 
18
- from rucio.api.importer import import_data
19
17
  from rucio.common.utils import parse_response
18
+ from rucio.gateway.importer import import_data
20
19
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
21
- from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, json_parameters
20
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers
22
21
 
23
22
 
24
23
  class Import(ErrorHandlingMethodView):
@@ -81,7 +80,7 @@ class Import(ErrorHandlingMethodView):
81
80
  description: The email of an account.
82
81
  type: string
83
82
  identities:
84
- description: The identiies accociated with an account. Deletes old identites and adds the newly defined ones.
83
+ description: The identities associated with an account. Deletes old identities and adds the newly defined ones.
85
84
  type: array
86
85
  items:
87
86
  description: One identity associated with an account.
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,13 +16,11 @@ from json import dumps
17
16
 
18
17
  from flask import Flask, Response, request
19
18
 
20
- from rucio.api.lifetime_exception import list_exceptions, add_exception, update_exception
21
- from rucio.common.exception import LifetimeExceptionNotFound, UnsupportedOperation, InvalidObject, AccessDenied, \
22
- LifetimeExceptionDuplicate
19
+ from rucio.common.exception import AccessDenied, InvalidObject, LifetimeExceptionDuplicate, LifetimeExceptionNotFound, UnsupportedOperation
23
20
  from rucio.common.utils import APIEncoder
21
+ from rucio.gateway.lifetime_exception import add_exception, list_exceptions, update_exception
24
22
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
25
- from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
26
- try_stream, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
23
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers, try_stream
27
24
 
28
25
 
29
26
  class LifetimeException(ErrorHandlingMethodView):
@@ -63,10 +60,10 @@ class LifetimeException(ErrorHandlingMethodView):
63
60
  type: string
64
61
  enum: ['F', 'D', 'C', 'A', 'X', 'Y', 'Z']
65
62
  account:
66
- description: The account accociated with the lifetime exception.
63
+ description: The account associated with the lifetime exception.
67
64
  type: string
68
65
  pattern:
69
- description: The patter of the lifetime exception.
66
+ description: The pattern of the lifetime exception.
70
67
  type: string
71
68
  comments:
72
69
  description: The comments of the lifetime exception.
@@ -211,7 +208,7 @@ class LifetimeExceptionId(ErrorHandlingMethodView):
211
208
  description: The account associated with the lifetime exception.
212
209
  type: string
213
210
  pattern:
214
- description: The patter of the lifetime exception.
211
+ description: The pattern of the lifetime exception.
215
212
  type: string
216
213
  comments:
217
214
  description: The comments of the lifetime exception.
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +14,11 @@
15
14
 
16
15
  from flask import Flask, request
17
16
 
18
- from rucio.api.lock import get_dataset_locks_by_rse, get_dataset_locks, get_dataset_locks_bulk
19
17
  from rucio.common.exception import RSENotFound
20
18
  from rucio.common.utils import render_json
19
+ from rucio.gateway.lock import get_dataset_locks, get_dataset_locks_bulk, get_dataset_locks_by_rse
21
20
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
22
- from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \
23
- response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parse
21
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parse, parse_scope_name, response_headers, try_stream
24
22
 
25
23
 
26
24
  class LockByRSE(ErrorHandlingMethodView):
@@ -1,5 +1,4 @@
1
1
  #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +14,7 @@
15
14
  # limitations under the License.
16
15
 
17
16
  import importlib
17
+ import logging
18
18
 
19
19
  from flask import Flask
20
20
 
@@ -37,7 +37,7 @@ DEFAULT_ENDPOINTS = [
37
37
  'import',
38
38
  'lifetime_exceptions',
39
39
  'locks',
40
- 'meta',
40
+ 'meta_conventions',
41
41
  'ping',
42
42
  'redirect',
43
43
  'replicas',
@@ -51,6 +51,10 @@ DEFAULT_ENDPOINTS = [
51
51
 
52
52
  def apply_endpoints(app, modules):
53
53
  for blueprint_module in modules:
54
+ # Legacy patch - TODO Remove in 38.0.0
55
+ if blueprint_module == "meta":
56
+ logging.log(logging.WARNING, "Endpoint `meta` is depreciated and will be removed in future releases")
57
+ blueprint_module = "meta_conventions"
54
58
  try:
55
59
  # searches for module names locally
56
60
  blueprint_module = importlib.import_module('.' + blueprint_module,
@@ -60,6 +64,10 @@ def apply_endpoints(app, modules):
60
64
 
61
65
  if hasattr(blueprint_module, 'blueprint'):
62
66
  app.register_blueprint(blueprint_module.blueprint())
67
+
68
+ if hasattr(blueprint_module, "blueprint_legacy"):
69
+ app.register_blueprint(blueprint_module.blueprint_legacy())
70
+
63
71
  else:
64
72
  raise ConfigurationError(f'"{blueprint_module}" from the endpoints configuration value did not have a blueprint')
65
73
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,17 +12,16 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from flask import Flask, request, jsonify
15
+ from flask import Flask, jsonify, request
17
16
 
18
- from rucio.api.meta import add_key, add_value, list_keys, list_values
19
- from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedValueType, UnsupportedKeyType
17
+ from rucio.common.exception import Duplicate, InvalidValueForKey, KeyNotFound, UnsupportedKeyType, UnsupportedValueType
18
+ from rucio.gateway.meta_conventions import add_key, add_value, list_keys, list_values
20
19
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
21
- from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, response_headers, \
22
- generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
20
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, response_headers
23
21
 
24
22
 
25
- class Meta(ErrorHandlingMethodView):
26
- """ REST APIs for data identifier attribute keys. """
23
+ class MetaConventions(ErrorHandlingMethodView):
24
+ """ REST APIs for managing data identifier attribute metadata key formats. """
27
25
 
28
26
  @check_accept_header_wrapper_flask(['application/json'])
29
27
  def get(self):
@@ -39,7 +37,7 @@ class Meta(ErrorHandlingMethodView):
39
37
  application/json:
40
38
  schema:
41
39
  type: array
42
- descripton: List of all DID keys.
40
+ description: List of all DID keys.
43
41
  items:
44
42
  type: string
45
43
  description: Data Itentifier key
@@ -71,7 +69,7 @@ class Meta(ErrorHandlingMethodView):
71
69
  type: object
72
70
  properties:
73
71
  key_type:
74
- description: The key tpye.
72
+ description: The key type.
75
73
  type: string
76
74
  value_type:
77
75
  description: The value type.
@@ -207,9 +205,23 @@ class Values(ErrorHandlingMethodView):
207
205
 
208
206
 
209
207
  def blueprint():
208
+ bp = AuthenticatedBlueprint('meta_conventions', __name__, url_prefix='/meta_conventions')
209
+
210
+ meta_view = MetaConventions.as_view('meta_conventions')
211
+ bp.add_url_rule('/', view_func=meta_view, methods=['get', ])
212
+ bp.add_url_rule('/<key>', view_func=meta_view, methods=['post', ])
213
+ values_view = Values.as_view('values')
214
+ bp.add_url_rule('/<key>/', view_func=values_view, methods=['get', 'post'])
215
+
216
+ bp.after_request(response_headers)
217
+ return bp
218
+
219
+
220
+ def blueprint_legacy():
221
+ # TODO: Remove in 38.0
210
222
  bp = AuthenticatedBlueprint('meta', __name__, url_prefix='/meta')
211
223
 
212
- meta_view = Meta.as_view('meta')
224
+ meta_view = MetaConventions.as_view('meta')
213
225
  bp.add_url_rule('/', view_func=meta_view, methods=['get', ])
214
226
  bp.add_url_rule('/<key>', view_func=meta_view, methods=['post', ])
215
227
  values_view = Values.as_view('values')
@@ -222,5 +234,8 @@ def blueprint():
222
234
  def make_doc():
223
235
  """ Only used for sphinx documentation """
224
236
  doc_app = Flask(__name__)
237
+
225
238
  doc_app.register_blueprint(blueprint())
239
+ doc_app.register_blueprint(blueprint_legacy())
240
+
226
241
  return doc_app
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +12,7 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from flask import Flask, Blueprint
15
+ from flask import Blueprint, Flask
17
16
 
18
17
  from rucio.core.monitor import generate_prometheus_metrics
19
18
  from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,15 +15,16 @@
16
15
  import time
17
16
  from typing import TYPE_CHECKING
18
17
 
19
- from flask import Flask, Blueprint, request
18
+ from flask import Blueprint, Flask, request
20
19
  from werkzeug.datastructures import Headers
21
20
 
22
21
  from rucio.core.nongrid_trace import trace
23
- from rucio.web.rest.flaskapi.v1.common import response_headers, ErrorHandlingMethodView, json_parameters
22
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, json_parameters, response_headers
24
23
 
25
24
  if TYPE_CHECKING:
26
25
  from typing import Optional
27
- from rucio.web.rest.flaskapi.v1.common import HeadersType
26
+
27
+ from rucio.web.rest.flaskapi.v1.types import HeadersType
28
28
 
29
29
 
30
30
  class XAODTrace(ErrorHandlingMethodView):
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,16 +14,16 @@
15
14
 
16
15
  from typing import TYPE_CHECKING
17
16
 
18
- from flask import Flask, Blueprint, jsonify, request
17
+ from flask import Blueprint, Flask, jsonify, request
19
18
  from werkzeug.datastructures import Headers
20
19
 
21
20
  from rucio import version
22
- from rucio.web.rest.flaskapi.v1.common import response_headers, check_accept_header_wrapper_flask, \
23
- ErrorHandlingMethodView
21
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, response_headers
24
22
 
25
23
  if TYPE_CHECKING:
26
24
  from typing import Optional
27
- from rucio.web.rest.flaskapi.v1.common import HeadersType
25
+
26
+ from rucio.web.rest.flaskapi.v1.types import HeadersType
28
27
 
29
28
 
30
29
  class Ping(ErrorHandlingMethodView):
@@ -69,8 +68,8 @@ class Ping(ErrorHandlingMethodView):
69
68
  return response
70
69
 
71
70
 
72
- def blueprint(with_doc=False):
73
- bp = Blueprint('ping', __name__, url_prefix='/ping')
71
+ def blueprint(standalone=False, with_doc=False):
72
+ bp = Blueprint('ping', __name__, url_prefix='/' if standalone else '/ping')
74
73
 
75
74
  ping_view = Ping.as_view('ping')
76
75
  if not with_doc:
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,18 +15,18 @@
16
15
  import itertools
17
16
  from typing import TYPE_CHECKING
18
17
 
19
- from flask import Flask, Blueprint, request, redirect
18
+ from flask import Blueprint, Flask, redirect, request
20
19
  from werkzeug.datastructures import Headers
21
20
 
22
- from rucio.api.replica import list_replicas
23
- from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound
21
+ from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound, SortingAlgorithmNotSupported
24
22
  from rucio.core.replica_sorter import site_selector, sort_replicas
25
- from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, parse_scope_name, try_stream, \
26
- extract_vo, generate_http_error_flask, ErrorHandlingMethodView
23
+ from rucio.gateway.replica import list_replicas
24
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, extract_vo, generate_http_error_flask, parse_scope_name, try_stream
27
25
 
28
26
  if TYPE_CHECKING:
29
27
  from typing import Optional
30
- from rucio.web.rest.flaskapi.v1.common import HeadersType
28
+
29
+ from rucio.web.rest.flaskapi.v1.types import HeadersType
31
30
 
32
31
 
33
32
  class MetaLinkRedirector(ErrorHandlingMethodView):
@@ -179,7 +178,7 @@ class MetaLinkRedirector(ErrorHandlingMethodView):
179
178
  yield '</metalink>\n'
180
179
 
181
180
  return try_stream(generate(), content_type='application/metalink4+xml')
182
- except (DataIdentifierNotFound, ReplicaNotFound) as error:
181
+ except (DataIdentifierNotFound, ReplicaNotFound, SortingAlgorithmNotSupported) as error:
183
182
  return generate_http_error_flask(404, error, headers=headers)
184
183
 
185
184
 
@@ -342,7 +341,7 @@ class HeaderRedirector(ErrorHandlingMethodView):
342
341
  return response
343
342
 
344
343
  return 'no redirection possible - file does not exist', 404, headers
345
- except ReplicaNotFound as error:
344
+ except (ReplicaNotFound, SortingAlgorithmNotSupported) as error:
346
345
  return generate_http_error_flask(404, error, headers=headers)
347
346
 
348
347
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,21 +20,48 @@ from xml.sax.saxutils import escape
21
20
 
22
21
  from flask import Flask, Response, request
23
22
 
24
- from rucio.api.quarantined_replica import quarantine_file_replicas
25
- from rucio.api.replica import add_replicas, list_replicas, list_dataset_replicas, list_dataset_replicas_bulk, \
26
- delete_replicas, get_did_from_pfns, update_replicas_states, declare_bad_file_replicas, add_bad_dids, add_bad_pfns, \
27
- get_suspicious_files, declare_suspicious_file_replicas, list_bad_replicas_status, get_bad_replicas_summary, \
28
- list_datasets_per_rse, set_tombstone, list_dataset_replicas_vp
29
23
  from rucio.common.config import config_get, config_get_int
30
24
  from rucio.common.constants import SUPPORTED_PROTOCOLS
31
- from rucio.common.exception import AccessDenied, DataIdentifierAlreadyExists, InvalidType, DataIdentifierNotFound, \
32
- Duplicate, InvalidPath, ResourceTemporaryUnavailable, RSENotFound, ReplicaNotFound, InvalidObject, ScopeNotFound, ReplicaIsLocked
33
- from rucio.common.utils import parse_response, APIEncoder, render_json_list
25
+ from rucio.common.exception import (
26
+ AccessDenied,
27
+ DataIdentifierAlreadyExists,
28
+ DataIdentifierNotFound,
29
+ Duplicate,
30
+ InvalidObject,
31
+ InvalidPath,
32
+ InvalidType,
33
+ ReplicaIsLocked,
34
+ ReplicaNotFound,
35
+ ResourceTemporaryUnavailable,
36
+ RSENotFound,
37
+ ScopeNotFound,
38
+ SortingAlgorithmNotSupported,
39
+ )
40
+ from rucio.common.utils import APIEncoder, parse_response, render_json
34
41
  from rucio.core.replica_sorter import sort_replicas
35
42
  from rucio.db.sqla.constants import BadFilesStatus
43
+ from rucio.gateway.quarantined_replica import quarantine_file_replicas
44
+ from rucio.gateway.replica import (
45
+ add_bad_dids,
46
+ add_bad_pfns,
47
+ add_replicas,
48
+ declare_bad_file_replicas,
49
+ declare_suspicious_file_replicas,
50
+ delete_replicas,
51
+ get_bad_replicas_summary,
52
+ get_did_from_pfns,
53
+ get_suspicious_files,
54
+ list_bad_replicas_status,
55
+ list_dataset_replicas,
56
+ list_dataset_replicas_bulk,
57
+ list_dataset_replicas_vp,
58
+ list_datasets_per_rse,
59
+ list_replicas,
60
+ set_tombstone,
61
+ update_replicas_states,
62
+ )
36
63
  from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
37
- from rucio.web.rest.flaskapi.v1.common import check_accept_header_wrapper_flask, try_stream, parse_scope_name, \
38
- response_headers, generate_http_error_flask, ErrorHandlingMethodView, json_parameters, param_get
64
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, json_parameters, param_get, parse_scope_name, response_headers, try_stream
39
65
 
40
66
 
41
67
  def _sorted_with_priorities(replicas, sorted_pfns, limit=None):
@@ -240,7 +266,7 @@ class Replicas(ErrorHandlingMethodView):
240
266
  else:
241
267
  response_generator = _generate_json_response(rfiles)
242
268
  return try_stream(response_generator, content_type=content_type)
243
- except DataIdentifierNotFound as error:
269
+ except (DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
244
270
  return generate_http_error_flask(404, error)
245
271
 
246
272
  def post(self):
@@ -381,7 +407,7 @@ class Replicas(ErrorHandlingMethodView):
381
407
  description: The pfn of the replica.
382
408
  type: string
383
409
  error_message:
384
- description: The error message if an error occured.
410
+ description: The error message if an error occurred.
385
411
  type: string
386
412
  broken_rule_id:
387
413
  description: The id of the broken rule if one was found.
@@ -724,10 +750,8 @@ class ListReplicas(ErrorHandlingMethodView):
724
750
  else:
725
751
  response_generator = _generate_json_response(rfiles)
726
752
  return try_stream(response_generator, content_type=content_type)
727
- except InvalidObject as error:
753
+ except (InvalidObject, DataIdentifierNotFound, SortingAlgorithmNotSupported) as error:
728
754
  return generate_http_error_flask(400, error)
729
- except DataIdentifierNotFound as error:
730
- return generate_http_error_flask(404, error)
731
755
 
732
756
 
733
757
  class ReplicasDIDs(ErrorHandlingMethodView):
@@ -1054,7 +1078,7 @@ class SuspiciousReplicas(ErrorHandlingMethodView):
1054
1078
  nattempts = int(params['nattempts'][0])
1055
1079
 
1056
1080
  result = get_suspicious_files(rse_expression=rse_expression, younger_than=younger_than, nattempts=nattempts, vo=request.environ.get('vo'))
1057
- return Response(render_json_list(result), 200, content_type='application/json')
1081
+ return Response(render_json(result), 200, content_type='application/json')
1058
1082
 
1059
1083
 
1060
1084
  class BadReplicasStates(ErrorHandlingMethodView):
@@ -1114,7 +1138,7 @@ class BadReplicasStates(ErrorHandlingMethodView):
1114
1138
  - type: object
1115
1139
  properties:
1116
1140
  scope:
1117
- description: The scope fo the replica.
1141
+ description: The scope of the replica.
1118
1142
  type: string
1119
1143
  name:
1120
1144
  description: The name of the replica.
@@ -1125,7 +1149,7 @@ class BadReplicasStates(ErrorHandlingMethodView):
1125
1149
  - type: object
1126
1150
  properties:
1127
1151
  scope:
1128
- description: The scope fo the replica.
1152
+ description: The scope of the replica.
1129
1153
  type: string
1130
1154
  name:
1131
1155
  description: The name of the replica.