rucio 37.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (487) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/cli/__init__.py +14 -0
  4. rucio/cli/account.py +216 -0
  5. rucio/cli/bin_legacy/__init__.py +13 -0
  6. rucio/cli/bin_legacy/rucio.py +2825 -0
  7. rucio/cli/bin_legacy/rucio_admin.py +2500 -0
  8. rucio/cli/command.py +272 -0
  9. rucio/cli/config.py +72 -0
  10. rucio/cli/did.py +191 -0
  11. rucio/cli/download.py +128 -0
  12. rucio/cli/lifetime_exception.py +33 -0
  13. rucio/cli/replica.py +162 -0
  14. rucio/cli/rse.py +293 -0
  15. rucio/cli/rule.py +158 -0
  16. rucio/cli/scope.py +40 -0
  17. rucio/cli/subscription.py +73 -0
  18. rucio/cli/upload.py +60 -0
  19. rucio/cli/utils.py +226 -0
  20. rucio/client/__init__.py +15 -0
  21. rucio/client/accountclient.py +432 -0
  22. rucio/client/accountlimitclient.py +183 -0
  23. rucio/client/baseclient.py +983 -0
  24. rucio/client/client.py +120 -0
  25. rucio/client/configclient.py +126 -0
  26. rucio/client/credentialclient.py +59 -0
  27. rucio/client/didclient.py +868 -0
  28. rucio/client/diracclient.py +56 -0
  29. rucio/client/downloadclient.py +1783 -0
  30. rucio/client/exportclient.py +44 -0
  31. rucio/client/fileclient.py +50 -0
  32. rucio/client/importclient.py +42 -0
  33. rucio/client/lifetimeclient.py +90 -0
  34. rucio/client/lockclient.py +109 -0
  35. rucio/client/metaconventionsclient.py +140 -0
  36. rucio/client/pingclient.py +44 -0
  37. rucio/client/replicaclient.py +452 -0
  38. rucio/client/requestclient.py +125 -0
  39. rucio/client/richclient.py +317 -0
  40. rucio/client/rseclient.py +746 -0
  41. rucio/client/ruleclient.py +294 -0
  42. rucio/client/scopeclient.py +90 -0
  43. rucio/client/subscriptionclient.py +173 -0
  44. rucio/client/touchclient.py +82 -0
  45. rucio/client/uploadclient.py +969 -0
  46. rucio/common/__init__.py +13 -0
  47. rucio/common/bittorrent.py +234 -0
  48. rucio/common/cache.py +111 -0
  49. rucio/common/checksum.py +168 -0
  50. rucio/common/client.py +122 -0
  51. rucio/common/config.py +788 -0
  52. rucio/common/constants.py +217 -0
  53. rucio/common/constraints.py +17 -0
  54. rucio/common/didtype.py +237 -0
  55. rucio/common/dumper/__init__.py +342 -0
  56. rucio/common/dumper/consistency.py +497 -0
  57. rucio/common/dumper/data_models.py +362 -0
  58. rucio/common/dumper/path_parsing.py +75 -0
  59. rucio/common/exception.py +1208 -0
  60. rucio/common/extra.py +31 -0
  61. rucio/common/logging.py +420 -0
  62. rucio/common/pcache.py +1409 -0
  63. rucio/common/plugins.py +185 -0
  64. rucio/common/policy.py +93 -0
  65. rucio/common/schema/__init__.py +200 -0
  66. rucio/common/schema/generic.py +416 -0
  67. rucio/common/schema/generic_multi_vo.py +395 -0
  68. rucio/common/stomp_utils.py +423 -0
  69. rucio/common/stopwatch.py +55 -0
  70. rucio/common/test_rucio_server.py +154 -0
  71. rucio/common/types.py +483 -0
  72. rucio/common/utils.py +1688 -0
  73. rucio/core/__init__.py +13 -0
  74. rucio/core/account.py +496 -0
  75. rucio/core/account_counter.py +236 -0
  76. rucio/core/account_limit.py +425 -0
  77. rucio/core/authentication.py +620 -0
  78. rucio/core/config.py +437 -0
  79. rucio/core/credential.py +224 -0
  80. rucio/core/did.py +3004 -0
  81. rucio/core/did_meta_plugins/__init__.py +252 -0
  82. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  83. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  84. rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
  85. rucio/core/did_meta_plugins/filter_engine.py +672 -0
  86. rucio/core/did_meta_plugins/json_meta.py +240 -0
  87. rucio/core/did_meta_plugins/mongo_meta.py +229 -0
  88. rucio/core/did_meta_plugins/postgres_meta.py +352 -0
  89. rucio/core/dirac.py +237 -0
  90. rucio/core/distance.py +187 -0
  91. rucio/core/exporter.py +59 -0
  92. rucio/core/heartbeat.py +363 -0
  93. rucio/core/identity.py +301 -0
  94. rucio/core/importer.py +260 -0
  95. rucio/core/lifetime_exception.py +377 -0
  96. rucio/core/lock.py +577 -0
  97. rucio/core/message.py +288 -0
  98. rucio/core/meta_conventions.py +203 -0
  99. rucio/core/monitor.py +448 -0
  100. rucio/core/naming_convention.py +195 -0
  101. rucio/core/nongrid_trace.py +136 -0
  102. rucio/core/oidc.py +1463 -0
  103. rucio/core/permission/__init__.py +161 -0
  104. rucio/core/permission/generic.py +1124 -0
  105. rucio/core/permission/generic_multi_vo.py +1144 -0
  106. rucio/core/quarantined_replica.py +224 -0
  107. rucio/core/replica.py +4483 -0
  108. rucio/core/replica_sorter.py +362 -0
  109. rucio/core/request.py +3091 -0
  110. rucio/core/rse.py +2079 -0
  111. rucio/core/rse_counter.py +185 -0
  112. rucio/core/rse_expression_parser.py +459 -0
  113. rucio/core/rse_selector.py +304 -0
  114. rucio/core/rule.py +4484 -0
  115. rucio/core/rule_grouping.py +1620 -0
  116. rucio/core/scope.py +181 -0
  117. rucio/core/subscription.py +362 -0
  118. rucio/core/topology.py +490 -0
  119. rucio/core/trace.py +375 -0
  120. rucio/core/transfer.py +1531 -0
  121. rucio/core/vo.py +169 -0
  122. rucio/core/volatile_replica.py +151 -0
  123. rucio/daemons/__init__.py +13 -0
  124. rucio/daemons/abacus/__init__.py +13 -0
  125. rucio/daemons/abacus/account.py +116 -0
  126. rucio/daemons/abacus/collection_replica.py +124 -0
  127. rucio/daemons/abacus/rse.py +117 -0
  128. rucio/daemons/atropos/__init__.py +13 -0
  129. rucio/daemons/atropos/atropos.py +242 -0
  130. rucio/daemons/auditor/__init__.py +289 -0
  131. rucio/daemons/auditor/hdfs.py +97 -0
  132. rucio/daemons/auditor/srmdumps.py +355 -0
  133. rucio/daemons/automatix/__init__.py +13 -0
  134. rucio/daemons/automatix/automatix.py +304 -0
  135. rucio/daemons/badreplicas/__init__.py +13 -0
  136. rucio/daemons/badreplicas/minos.py +322 -0
  137. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  138. rucio/daemons/badreplicas/necromancer.py +196 -0
  139. rucio/daemons/bb8/__init__.py +13 -0
  140. rucio/daemons/bb8/bb8.py +353 -0
  141. rucio/daemons/bb8/common.py +759 -0
  142. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  143. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  144. rucio/daemons/cache/__init__.py +13 -0
  145. rucio/daemons/cache/consumer.py +133 -0
  146. rucio/daemons/common.py +405 -0
  147. rucio/daemons/conveyor/__init__.py +13 -0
  148. rucio/daemons/conveyor/common.py +562 -0
  149. rucio/daemons/conveyor/finisher.py +529 -0
  150. rucio/daemons/conveyor/poller.py +394 -0
  151. rucio/daemons/conveyor/preparer.py +205 -0
  152. rucio/daemons/conveyor/receiver.py +179 -0
  153. rucio/daemons/conveyor/stager.py +133 -0
  154. rucio/daemons/conveyor/submitter.py +403 -0
  155. rucio/daemons/conveyor/throttler.py +532 -0
  156. rucio/daemons/follower/__init__.py +13 -0
  157. rucio/daemons/follower/follower.py +101 -0
  158. rucio/daemons/hermes/__init__.py +13 -0
  159. rucio/daemons/hermes/hermes.py +534 -0
  160. rucio/daemons/judge/__init__.py +13 -0
  161. rucio/daemons/judge/cleaner.py +159 -0
  162. rucio/daemons/judge/evaluator.py +185 -0
  163. rucio/daemons/judge/injector.py +162 -0
  164. rucio/daemons/judge/repairer.py +154 -0
  165. rucio/daemons/oauthmanager/__init__.py +13 -0
  166. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  167. rucio/daemons/reaper/__init__.py +13 -0
  168. rucio/daemons/reaper/dark_reaper.py +282 -0
  169. rucio/daemons/reaper/reaper.py +739 -0
  170. rucio/daemons/replicarecoverer/__init__.py +13 -0
  171. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  172. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  173. rucio/daemons/rsedecommissioner/config.py +81 -0
  174. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  175. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  176. rucio/daemons/rsedecommissioner/profiles/generic.py +452 -0
  177. rucio/daemons/rsedecommissioner/profiles/types.py +93 -0
  178. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  179. rucio/daemons/storage/__init__.py +13 -0
  180. rucio/daemons/storage/consistency/__init__.py +13 -0
  181. rucio/daemons/storage/consistency/actions.py +848 -0
  182. rucio/daemons/tracer/__init__.py +13 -0
  183. rucio/daemons/tracer/kronos.py +511 -0
  184. rucio/daemons/transmogrifier/__init__.py +13 -0
  185. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  186. rucio/daemons/undertaker/__init__.py +13 -0
  187. rucio/daemons/undertaker/undertaker.py +137 -0
  188. rucio/db/__init__.py +13 -0
  189. rucio/db/sqla/__init__.py +52 -0
  190. rucio/db/sqla/constants.py +206 -0
  191. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  192. rucio/db/sqla/migrate_repo/env.py +110 -0
  193. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  194. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  195. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  196. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  197. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  198. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  199. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  200. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  201. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  202. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  203. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  204. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  205. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  206. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  207. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  208. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  209. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  210. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  211. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  212. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  213. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  214. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  215. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  216. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  217. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  218. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  219. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  220. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  221. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  222. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  223. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  224. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  225. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  226. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  227. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  228. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  229. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  230. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  231. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  232. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  233. rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
  234. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  235. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  236. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  237. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  238. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  239. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  240. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  241. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  242. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  243. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  244. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  245. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  246. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  247. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  248. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  249. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  251. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  252. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  253. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  254. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  255. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  256. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  257. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  258. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  259. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  260. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  261. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  262. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  263. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  264. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  265. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  266. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  267. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  268. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  269. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  270. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  271. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  272. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  273. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  274. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  275. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  276. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  278. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  279. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  280. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  281. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  282. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  283. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  284. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  285. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  286. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  287. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  288. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  289. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  290. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  291. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  292. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  293. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  294. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  295. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  296. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  297. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  298. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  299. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  300. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  301. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  302. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  303. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  304. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  305. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  306. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  307. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  308. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  309. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  310. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  311. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  312. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  313. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  314. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  316. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  317. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  318. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  319. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  320. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  321. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  322. rucio/db/sqla/models.py +1743 -0
  323. rucio/db/sqla/sautils.py +55 -0
  324. rucio/db/sqla/session.py +529 -0
  325. rucio/db/sqla/types.py +206 -0
  326. rucio/db/sqla/util.py +543 -0
  327. rucio/gateway/__init__.py +13 -0
  328. rucio/gateway/account.py +345 -0
  329. rucio/gateway/account_limit.py +363 -0
  330. rucio/gateway/authentication.py +381 -0
  331. rucio/gateway/config.py +227 -0
  332. rucio/gateway/credential.py +70 -0
  333. rucio/gateway/did.py +987 -0
  334. rucio/gateway/dirac.py +83 -0
  335. rucio/gateway/exporter.py +60 -0
  336. rucio/gateway/heartbeat.py +76 -0
  337. rucio/gateway/identity.py +189 -0
  338. rucio/gateway/importer.py +46 -0
  339. rucio/gateway/lifetime_exception.py +121 -0
  340. rucio/gateway/lock.py +153 -0
  341. rucio/gateway/meta_conventions.py +98 -0
  342. rucio/gateway/permission.py +74 -0
  343. rucio/gateway/quarantined_replica.py +79 -0
  344. rucio/gateway/replica.py +538 -0
  345. rucio/gateway/request.py +330 -0
  346. rucio/gateway/rse.py +632 -0
  347. rucio/gateway/rule.py +437 -0
  348. rucio/gateway/scope.py +100 -0
  349. rucio/gateway/subscription.py +280 -0
  350. rucio/gateway/vo.py +126 -0
  351. rucio/rse/__init__.py +96 -0
  352. rucio/rse/protocols/__init__.py +13 -0
  353. rucio/rse/protocols/bittorrent.py +194 -0
  354. rucio/rse/protocols/cache.py +111 -0
  355. rucio/rse/protocols/dummy.py +100 -0
  356. rucio/rse/protocols/gfal.py +708 -0
  357. rucio/rse/protocols/globus.py +243 -0
  358. rucio/rse/protocols/http_cache.py +82 -0
  359. rucio/rse/protocols/mock.py +123 -0
  360. rucio/rse/protocols/ngarc.py +209 -0
  361. rucio/rse/protocols/posix.py +250 -0
  362. rucio/rse/protocols/protocol.py +361 -0
  363. rucio/rse/protocols/rclone.py +365 -0
  364. rucio/rse/protocols/rfio.py +145 -0
  365. rucio/rse/protocols/srm.py +338 -0
  366. rucio/rse/protocols/ssh.py +414 -0
  367. rucio/rse/protocols/storm.py +195 -0
  368. rucio/rse/protocols/webdav.py +594 -0
  369. rucio/rse/protocols/xrootd.py +302 -0
  370. rucio/rse/rsemanager.py +881 -0
  371. rucio/rse/translation.py +260 -0
  372. rucio/tests/__init__.py +13 -0
  373. rucio/tests/common.py +280 -0
  374. rucio/tests/common_server.py +149 -0
  375. rucio/transfertool/__init__.py +13 -0
  376. rucio/transfertool/bittorrent.py +200 -0
  377. rucio/transfertool/bittorrent_driver.py +50 -0
  378. rucio/transfertool/bittorrent_driver_qbittorrent.py +134 -0
  379. rucio/transfertool/fts3.py +1600 -0
  380. rucio/transfertool/fts3_plugins.py +152 -0
  381. rucio/transfertool/globus.py +201 -0
  382. rucio/transfertool/globus_library.py +181 -0
  383. rucio/transfertool/mock.py +89 -0
  384. rucio/transfertool/transfertool.py +221 -0
  385. rucio/vcsversion.py +11 -0
  386. rucio/version.py +45 -0
  387. rucio/web/__init__.py +13 -0
  388. rucio/web/rest/__init__.py +13 -0
  389. rucio/web/rest/flaskapi/__init__.py +13 -0
  390. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  391. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  392. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  393. rucio/web/rest/flaskapi/v1/accounts.py +1103 -0
  394. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  395. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  396. rucio/web/rest/flaskapi/v1/common.py +426 -0
  397. rucio/web/rest/flaskapi/v1/config.py +304 -0
  398. rucio/web/rest/flaskapi/v1/credentials.py +213 -0
  399. rucio/web/rest/flaskapi/v1/dids.py +2340 -0
  400. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  401. rucio/web/rest/flaskapi/v1/export.py +75 -0
  402. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  403. rucio/web/rest/flaskapi/v1/identities.py +285 -0
  404. rucio/web/rest/flaskapi/v1/import.py +132 -0
  405. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  406. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  407. rucio/web/rest/flaskapi/v1/main.py +91 -0
  408. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  409. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  410. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  411. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  412. rucio/web/rest/flaskapi/v1/redirect.py +366 -0
  413. rucio/web/rest/flaskapi/v1/replicas.py +1894 -0
  414. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  415. rucio/web/rest/flaskapi/v1/rses.py +2250 -0
  416. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  417. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  418. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  419. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  420. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  421. rucio/web/rest/flaskapi/v1/traces.py +137 -0
  422. rucio/web/rest/flaskapi/v1/types.py +20 -0
  423. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  424. rucio/web/rest/main.py +18 -0
  425. rucio/web/rest/metrics.py +27 -0
  426. rucio/web/rest/ping.py +27 -0
  427. rucio-37.0.0rc1.data/data/rucio/etc/alembic.ini.template +71 -0
  428. rucio-37.0.0rc1.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  429. rucio-37.0.0rc1.data/data/rucio/etc/globus-config.yml.template +5 -0
  430. rucio-37.0.0rc1.data/data/rucio/etc/ldap.cfg.template +30 -0
  431. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  432. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  433. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  434. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  435. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  436. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  437. rucio-37.0.0rc1.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  438. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.atlas.client.template +43 -0
  439. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.template +241 -0
  440. rucio-37.0.0rc1.data/data/rucio/etc/rucio_multi_vo.cfg.template +217 -0
  441. rucio-37.0.0rc1.data/data/rucio/requirements.server.txt +297 -0
  442. rucio-37.0.0rc1.data/data/rucio/tools/bootstrap.py +34 -0
  443. rucio-37.0.0rc1.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  444. rucio-37.0.0rc1.data/data/rucio/tools/reset_database.py +40 -0
  445. rucio-37.0.0rc1.data/scripts/rucio +133 -0
  446. rucio-37.0.0rc1.data/scripts/rucio-abacus-account +74 -0
  447. rucio-37.0.0rc1.data/scripts/rucio-abacus-collection-replica +46 -0
  448. rucio-37.0.0rc1.data/scripts/rucio-abacus-rse +78 -0
  449. rucio-37.0.0rc1.data/scripts/rucio-admin +97 -0
  450. rucio-37.0.0rc1.data/scripts/rucio-atropos +60 -0
  451. rucio-37.0.0rc1.data/scripts/rucio-auditor +206 -0
  452. rucio-37.0.0rc1.data/scripts/rucio-automatix +50 -0
  453. rucio-37.0.0rc1.data/scripts/rucio-bb8 +57 -0
  454. rucio-37.0.0rc1.data/scripts/rucio-cache-client +141 -0
  455. rucio-37.0.0rc1.data/scripts/rucio-cache-consumer +42 -0
  456. rucio-37.0.0rc1.data/scripts/rucio-conveyor-finisher +58 -0
  457. rucio-37.0.0rc1.data/scripts/rucio-conveyor-poller +66 -0
  458. rucio-37.0.0rc1.data/scripts/rucio-conveyor-preparer +37 -0
  459. rucio-37.0.0rc1.data/scripts/rucio-conveyor-receiver +44 -0
  460. rucio-37.0.0rc1.data/scripts/rucio-conveyor-stager +76 -0
  461. rucio-37.0.0rc1.data/scripts/rucio-conveyor-submitter +139 -0
  462. rucio-37.0.0rc1.data/scripts/rucio-conveyor-throttler +104 -0
  463. rucio-37.0.0rc1.data/scripts/rucio-dark-reaper +53 -0
  464. rucio-37.0.0rc1.data/scripts/rucio-dumper +160 -0
  465. rucio-37.0.0rc1.data/scripts/rucio-follower +44 -0
  466. rucio-37.0.0rc1.data/scripts/rucio-hermes +54 -0
  467. rucio-37.0.0rc1.data/scripts/rucio-judge-cleaner +89 -0
  468. rucio-37.0.0rc1.data/scripts/rucio-judge-evaluator +137 -0
  469. rucio-37.0.0rc1.data/scripts/rucio-judge-injector +44 -0
  470. rucio-37.0.0rc1.data/scripts/rucio-judge-repairer +44 -0
  471. rucio-37.0.0rc1.data/scripts/rucio-kronos +44 -0
  472. rucio-37.0.0rc1.data/scripts/rucio-minos +53 -0
  473. rucio-37.0.0rc1.data/scripts/rucio-minos-temporary-expiration +50 -0
  474. rucio-37.0.0rc1.data/scripts/rucio-necromancer +120 -0
  475. rucio-37.0.0rc1.data/scripts/rucio-oauth-manager +63 -0
  476. rucio-37.0.0rc1.data/scripts/rucio-reaper +83 -0
  477. rucio-37.0.0rc1.data/scripts/rucio-replica-recoverer +248 -0
  478. rucio-37.0.0rc1.data/scripts/rucio-rse-decommissioner +66 -0
  479. rucio-37.0.0rc1.data/scripts/rucio-storage-consistency-actions +74 -0
  480. rucio-37.0.0rc1.data/scripts/rucio-transmogrifier +77 -0
  481. rucio-37.0.0rc1.data/scripts/rucio-undertaker +76 -0
  482. rucio-37.0.0rc1.dist-info/METADATA +92 -0
  483. rucio-37.0.0rc1.dist-info/RECORD +487 -0
  484. rucio-37.0.0rc1.dist-info/WHEEL +5 -0
  485. rucio-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
  486. rucio-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
  487. rucio-37.0.0rc1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,998 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import json
16
+ from typing import TYPE_CHECKING
17
+
18
+ import flask
19
+ from flask import Flask, Response
20
+
21
+ from rucio.common.exception import RequestNotFound
22
+ from rucio.common.utils import APIEncoder, render_json
23
+ from rucio.core.rse import get_rses_with_attribute_value
24
+ from rucio.db.sqla.constants import RequestState
25
+ from rucio.gateway import request
26
+ from rucio.web.rest.flaskapi.authenticated_bp import AuthenticatedBlueprint
27
+ from rucio.web.rest.flaskapi.v1.common import ErrorHandlingMethodView, check_accept_header_wrapper_flask, generate_http_error_flask, parse_scope_name, response_headers, try_stream
28
+
29
+ if TYPE_CHECKING:
30
+ from collections.abc import Iterator
31
+
32
+
33
+ class RequestGet(ErrorHandlingMethodView):
34
+ """ REST API to get requests. """
35
+
36
+ @check_accept_header_wrapper_flask(['application/json'])
37
+ def get(self, scope_name, rse):
38
+ """
39
+ ---
40
+ summary: Get Request
41
+ description: Get a request for a given DID to a destination RSE.
42
+ tags:
43
+ - Requests
44
+ parameters:
45
+ - name: scope_name
46
+ in: path
47
+ description: Data identifier (scope)/(name).
48
+ schema:
49
+ type: string
50
+ style: simple
51
+ - name: rse
52
+ in: path
53
+ description: Destination rse.
54
+ schema:
55
+ type: string
56
+ style: simple
57
+ responses:
58
+ 200:
59
+ description: OK
60
+ content:
61
+ application/json:
62
+ schema:
63
+ description: The request associated with the DID and destination RSE.
64
+ type: object
65
+ properties:
66
+ id:
67
+ description: The id of the request.
68
+ type: string
69
+ request_type:
70
+ description: The request type.
71
+ type: string
72
+ enum: ["T", "U", "D", "I", "O"]
73
+ scope:
74
+ description: The scope of the transfer.
75
+ type: string
76
+ name:
77
+ description: The name of the transfer.
78
+ type: string
79
+ did_type:
80
+ description: The did type.
81
+ type: string
82
+ dest_rse_id:
83
+ description: The destination RSE id.
84
+ type: string
85
+ source_rse_id:
86
+ description: The source RSE id.
87
+ type: string
88
+ attributes:
89
+ description: All attributes associated with the request.
90
+ type: string
91
+ state:
92
+ description: The state of the request.
93
+ type: string
94
+ enum: ["Q", "G", "S", "F", "D", "L", "N", "O", "A", "M", "U", "W", "P"]
95
+ external_id:
96
+ description: External id of the request.
97
+ type: string
98
+ external_host:
99
+ description: External host of the request.
100
+ type: string
101
+ retry_count:
102
+ description: The numbers of attempted retires.
103
+ type: integer
104
+ err_msg:
105
+ description: An error message if one occurred.
106
+ type: string
107
+ previous_attempt_id:
108
+ description: The id of the previous attempt.
109
+ type: string
110
+ rule_id:
111
+ description: The id of the associated replication rule.
112
+ type: string
113
+ activity:
114
+ description: The activity of the request.
115
+ type: string
116
+ bytes:
117
+ description: The size of the did in bytes.
118
+ type: integer
119
+ md5:
120
+ description: The md5 checksum of the did to transfer.
121
+ type: string
122
+ adler32:
123
+ description: The adler32 checksum of the did to transfer.
124
+ type: string
125
+ dest_url:
126
+ description: The destination url.
127
+ type: string
128
+ submitted_at:
129
+ description: The time the request got submitted.
130
+ type: string
131
+ started_at:
132
+ description: The time the request got started.
133
+ type: string
134
+ transferred_at:
135
+ description: The time the request got transferred.
136
+ type: string
137
+ estimated_at:
138
+ description: The time the request got estimated.
139
+ type: string
140
+ submitter_id:
141
+ description: The id of the submitter.
142
+ type: string
143
+ estimated_stated_at:
144
+ description: The estimation of the started at value.
145
+ type: string
146
+ estimated_transferred_at:
147
+ description: The estimation of the transferred at value.
148
+ type: string
149
+ staging_started_at:
150
+ description: The time the staging got started.
151
+ type: string
152
+ staging_finished_at:
153
+ description: The time the staging got finished.
154
+ type: string
155
+ account:
156
+ description: The account which issued the request.
157
+ type: string
158
+ requested_at:
159
+ description: The time the request got requested.
160
+ type: string
161
+ priority:
162
+ description: The priority of the request.
163
+ type: integer
164
+ transfertool:
165
+ description: The transfertool used.
166
+ type: string
167
+ source_rse:
168
+ description: The name of the source RSE.
169
+ type: string
170
+ dest_rse:
171
+ description: The name of the destination RSE.
172
+ type: string
173
+ 401:
174
+ description: Invalid Auth Token
175
+ 404:
176
+ description: Not found
177
+ 406:
178
+ description: Not acceptable
179
+ """
180
+ try:
181
+ scope, name = parse_scope_name(scope_name, flask.request.environ.get('vo'))
182
+ except ValueError as error:
183
+ return generate_http_error_flask(400, error)
184
+
185
+ try:
186
+ request_data = request.get_request_by_did(
187
+ scope=scope,
188
+ name=name,
189
+ rse=rse,
190
+ issuer=flask.request.environ.get('issuer'),
191
+ vo=flask.request.environ.get('vo'),
192
+ )
193
+ return Response(json.dumps(request_data, cls=APIEncoder), content_type='application/json')
194
+ except RequestNotFound as error:
195
+ return generate_http_error_flask(404, error.__class__.__name__, f'No request found for DID {scope}:{name} at RSE {rse}')
196
+
197
+
198
+ class RequestHistoryGet(ErrorHandlingMethodView):
199
+ """ REST API to get historical requests. """
200
+
201
+ @check_accept_header_wrapper_flask(['application/json'])
202
+ def get(self, scope_name, rse):
203
+ """
204
+ ---
205
+ summary: Get Historical Request
206
+ description: List a historical request for a given DID to a destination RSE.
207
+ tags:
208
+ - Requests
209
+ parameters:
210
+ - name: scope_name
211
+ in: path
212
+ description: Data identifier (scope)/(name).
213
+ schema:
214
+ type: string
215
+ style: simple
216
+ - name: rse
217
+ in: path
218
+ description: Destination rse.
219
+ schema:
220
+ type: string
221
+ style: simple
222
+ responses:
223
+ 200:
224
+ description: OK
225
+ content:
226
+ application/json:
227
+ schema:
228
+ description: The request associated with the DID and destination RSE.
229
+ type: object
230
+ properties:
231
+ id:
232
+ description: The id of the request.
233
+ type: string
234
+ request_type:
235
+ description: The request type.
236
+ type: string
237
+ enum: ["T", "U", "D", "I", "O"]
238
+ scope:
239
+ description: The scope of the transfer.
240
+ type: string
241
+ name:
242
+ description: The name of the transfer.
243
+ type: string
244
+ did_type:
245
+ description: The did type.
246
+ type: string
247
+ dest_rse_id:
248
+ description: The destination RSE id.
249
+ type: string
250
+ source_rse_id:
251
+ description: The source RSE id.
252
+ type: string
253
+ attributes:
254
+ description: All attributes associated with the request.
255
+ type: string
256
+ state:
257
+ description: The state of the request.
258
+ type: string
259
+ enum: ["Q", "G", "S", "F", "D", "L", "N", "O", "A", "M", "U", "W", "P"]
260
+ external_id:
261
+ description: External id of the request.
262
+ type: string
263
+ external_host:
264
+ description: External host of the request.
265
+ type: string
266
+ retry_count:
267
+ description: The numbers of attempted retires.
268
+ type: integer
269
+ err_msg:
270
+ description: An error message if one occurred.
271
+ type: string
272
+ previous_attempt_id:
273
+ description: The id of the previous attempt.
274
+ type: string
275
+ rule_id:
276
+ description: The id of the associated replication rule.
277
+ type: string
278
+ activity:
279
+ description: The activity of the request.
280
+ type: string
281
+ bytes:
282
+ description: The size of the did in bytes.
283
+ type: integer
284
+ md5:
285
+ description: The md5 checksum of the did to transfer.
286
+ type: string
287
+ adler32:
288
+ description: The adler32 checksum of the did to transfer.
289
+ type: string
290
+ dest_url:
291
+ description: The destination url.
292
+ type: string
293
+ submitted_at:
294
+ description: The time the request got submitted.
295
+ type: string
296
+ started_at:
297
+ description: The time the request got started.
298
+ type: string
299
+ transferred_at:
300
+ description: The time the request got transferred.
301
+ type: string
302
+ estimated_at:
303
+ description: The time the request got estimated.
304
+ type: string
305
+ submitter_id:
306
+ description: The id of the submitter.
307
+ type: string
308
+ estimated_stated_at:
309
+ description: The estimation of the started at value.
310
+ type: string
311
+ estimated_transferred_at:
312
+ description: The estimation of the transferred at value.
313
+ type: string
314
+ staging_started_at:
315
+ description: The time the staging got started.
316
+ type: string
317
+ staging_finished_at:
318
+ description: The time the staging got finished.
319
+ type: string
320
+ account:
321
+ description: The account which issued the request.
322
+ type: string
323
+ requested_at:
324
+ description: The time the request got requested.
325
+ type: string
326
+ priority:
327
+ description: The priority of the request.
328
+ type: integer
329
+ transfertool:
330
+ description: The transfertool used.
331
+ type: string
332
+ source_rse:
333
+ description: The name of the source RSE.
334
+ type: string
335
+ dest_rse:
336
+ description: The name of the destination RSE.
337
+ type: string
338
+ 401:
339
+ description: Invalid Auth Token
340
+ 404:
341
+ description: Not found
342
+ 406:
343
+ description: Not acceptable
344
+ """
345
+ try:
346
+ scope, name = parse_scope_name(scope_name, flask.request.environ.get('vo'))
347
+ except ValueError as error:
348
+ return generate_http_error_flask(400, error)
349
+
350
+ try:
351
+ request_data = request.get_request_history_by_did(
352
+ scope=scope,
353
+ name=name,
354
+ rse=rse,
355
+ issuer=flask.request.environ.get('issuer'),
356
+ vo=flask.request.environ.get('vo'),
357
+ )
358
+ return Response(json.dumps(request_data, cls=APIEncoder), content_type='application/json')
359
+ except RequestNotFound as error:
360
+ return generate_http_error_flask(404, error.__class__.__name__, f'No request found for DID {scope}:{name} at RSE {rse}')
361
+
362
+
363
+ class RequestList(ErrorHandlingMethodView):
364
+ """ REST API to get requests. """
365
+
366
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
367
+ def get(self):
368
+ """
369
+ ---
370
+ summary: List Requests
371
+ description: List requests for a given source and destination RSE or site.
372
+ tags:
373
+ - Requests
374
+ parameters:
375
+ - name: src_rse
376
+ in: query
377
+ description: The source rse.
378
+ schema:
379
+ type: array
380
+ items:
381
+ type: object
382
+ required:
383
+ - rse_id
384
+ properties:
385
+ rse_id:
386
+ description: The id of the rse.
387
+ type: string
388
+ - name: dest_rse
389
+ in: query
390
+ description: The destination rse.
391
+ schema:
392
+ type: array
393
+ items:
394
+ type: object
395
+ required:
396
+ - rse_id
397
+ properties:
398
+ rse_id:
399
+ description: The id of the rse.
400
+ type: string
401
+ - name: src_site
402
+ in: query
403
+ description: The source site.
404
+ schema:
405
+ type: string
406
+ - name: dest_site
407
+ in: query
408
+ description: The destination site.
409
+ schema:
410
+ type: string
411
+ - name: request_states
412
+ in: query
413
+ description: The accepted request states. Delimited by comma.
414
+ schema:
415
+ type: string
416
+ responses:
417
+ 200:
418
+ description: OK
419
+ content:
420
+ application/x-json-stream:
421
+ schema:
422
+ description: All requests matching the arguments. Separated by the new line character.
423
+ type: array
424
+ items:
425
+ description: A request.
426
+ type: object
427
+ properties:
428
+ id:
429
+ description: The id of the request.
430
+ type: string
431
+ request_type:
432
+ description: The request type.
433
+ type: string
434
+ enum: ["T", "U", "D", "I", "O"]
435
+ scope:
436
+ description: The scope of the transfer.
437
+ type: string
438
+ name:
439
+ description: The name of the transfer.
440
+ type: string
441
+ did_type:
442
+ description: The did type.
443
+ type: string
444
+ dest_rse_id:
445
+ description: The destination RSE id.
446
+ type: string
447
+ source_rse_id:
448
+ description: The source RSE id.
449
+ type: string
450
+ attributes:
451
+ description: All attributes associated with the request.
452
+ type: string
453
+ state:
454
+ description: The state of the request.
455
+ type: string
456
+ enum: ["Q", "G", "S", "F", "D", "L", "N", "O", "A", "M", "U", "W", "P"]
457
+ external_id:
458
+ description: External id of the request.
459
+ type: string
460
+ external_host:
461
+ description: External host of the request.
462
+ type: string
463
+ retry_count:
464
+ description: The numbers of attempted retires.
465
+ type: integer
466
+ err_msg:
467
+ description: An error message if one occurred.
468
+ type: string
469
+ previous_attempt_id:
470
+ description: The id of the previous attempt.
471
+ type: string
472
+ rule_id:
473
+ description: The id of the associated replication rule.
474
+ type: string
475
+ activity:
476
+ description: The activity of the request.
477
+ type: string
478
+ bytes:
479
+ description: The size of the did in bytes.
480
+ type: integer
481
+ md5:
482
+ description: The md5 checksum of the did to transfer.
483
+ type: string
484
+ adler32:
485
+ description: The adler32 checksum of the did to transfer.
486
+ type: string
487
+ dest_url:
488
+ description: The destination url.
489
+ type: string
490
+ submitted_at:
491
+ description: The time the request got submitted.
492
+ type: string
493
+ started_at:
494
+ description: The time the request got started.
495
+ type: string
496
+ transferred_at:
497
+ description: The time the request got transferred.
498
+ type: string
499
+ estimated_at:
500
+ description: The time the request got estimated.
501
+ type: string
502
+ submitter_id:
503
+ description: The id of the submitter.
504
+ type: string
505
+ estimated_stated_at:
506
+ description: The estimation of the started at value.
507
+ type: string
508
+ estimated_transferred_at:
509
+ description: The estimation of the transferred at value.
510
+ type: string
511
+ staging_started_at:
512
+ description: The time the staging got started.
513
+ type: string
514
+ staging_finished_at:
515
+ description: The time the staging got finished.
516
+ type: string
517
+ account:
518
+ description: The account which issued the request.
519
+ type: string
520
+ requested_at:
521
+ description: The time the request got requested.
522
+ type: string
523
+ priority:
524
+ description: The priority of the request.
525
+ type: integer
526
+ transfertool:
527
+ description: The transfertool used.
528
+ type: string
529
+ source_rse:
530
+ description: The name of the source RSE.
531
+ type: string
532
+ dest_rse:
533
+ description: The name of the destination RSE.
534
+ type: string
535
+ 401:
536
+ description: Invalid Auth Token
537
+ 404:
538
+ description: Not found
539
+ 406:
540
+ description: Not acceptable
541
+ """
542
+ src_rse = flask.request.args.get('src_rse', default=None)
543
+ dst_rse = flask.request.args.get('dst_rse', default=None)
544
+ src_site = flask.request.args.get('src_site', default=None)
545
+ dst_site = flask.request.args.get('dst_site', default=None)
546
+ request_states = flask.request.args.get('request_states', default=None)
547
+
548
+ if not request_states:
549
+ return generate_http_error_flask(400, 'MissingParameter', 'Request state is missing')
550
+ if src_rse and not dst_rse:
551
+ return generate_http_error_flask(400, 'MissingParameter', 'Destination RSE is missing')
552
+ elif dst_rse and not src_rse:
553
+ return generate_http_error_flask(400, 'MissingParameter', 'Source RSE is missing')
554
+ elif src_site and not dst_site:
555
+ return generate_http_error_flask(400, 'MissingParameter', 'Destination site is missing')
556
+ elif dst_site and not src_site:
557
+ return generate_http_error_flask(400, 'MissingParameter', 'Source site is missing')
558
+
559
+ try:
560
+ states = [RequestState(state) for state in request_states.split(',')]
561
+ except ValueError:
562
+ return generate_http_error_flask(400, 'Invalid', 'Request state value is invalid')
563
+
564
+ src_rses = []
565
+ dst_rses = []
566
+ if src_site:
567
+ src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ.get('vo'))
568
+ if not src_rses:
569
+ return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {src_site} to RSE')
570
+ src_rses = [rse['rse_name'] for rse in src_rses]
571
+ dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ.get('vo'))
572
+ if not dst_rses:
573
+ return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {dst_site} to RSE')
574
+ dst_rses = [rse['rse_name'] for rse in dst_rses]
575
+ else:
576
+ dst_rses = [dst_rse]
577
+ src_rses = [src_rse]
578
+
579
+ def generate(issuer, vo):
580
+ for result in request.list_requests(src_rses, dst_rses, states, issuer=issuer, vo=vo):
581
+ yield render_json(**result) + '\n'
582
+
583
+ return try_stream(generate(issuer=flask.request.environ.get('issuer'), vo=flask.request.environ.get('vo')))
584
+
585
+
586
+ class RequestHistoryList(ErrorHandlingMethodView):
587
+ """ REST API to get requests. """
588
+
589
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
590
+ def get(self):
591
+ """
592
+ ---
593
+ summary: List Historic Requests
594
+ description: List historical requests for a given source and destination RSE or site.
595
+ tags:
596
+ - Requests
597
+ parameters:
598
+ - name: src_rse
599
+ in: query
600
+ description: The source rse.
601
+ schema:
602
+ type: array
603
+ items:
604
+ type: object
605
+ required:
606
+ - rse_id
607
+ properties:
608
+ rse_id:
609
+ description: The id of the rse.
610
+ type: string
611
+ - name: dest_rse
612
+ in: query
613
+ description: The destination rse.
614
+ schema:
615
+ type: array
616
+ items:
617
+ type: object
618
+ required:
619
+ - rse_id
620
+ properties:
621
+ rse_id:
622
+ description: The id of the rse.
623
+ type: string
624
+ - name: src_site
625
+ in: query
626
+ description: The source site.
627
+ schema:
628
+ type: string
629
+ - name: dest_site
630
+ in: query
631
+ description: The destination site.
632
+ schema:
633
+ type: string
634
+ - name: request_states
635
+ in: query
636
+ description: The accepted request states. Delimited by comma.
637
+ schema:
638
+ type: string
639
+ - name: offset
640
+ in: query
641
+ description: The offset of the list.
642
+ schema:
643
+ type: integer
644
+ default: 0
645
+ - name: limit
646
+ in: query
647
+ description: The maximum number of items to return.
648
+ schema:
649
+ type: integer
650
+ default: 100
651
+ responses:
652
+ 200:
653
+ description: OK
654
+ content:
655
+ application/x-json-stream:
656
+ schema:
657
+ description: All requests matching the arguments. Separated by a new line character.
658
+ type: array
659
+ items:
660
+ description: A request.
661
+ type: object
662
+ properties:
663
+ id:
664
+ description: The id of the request.
665
+ type: string
666
+ request_type:
667
+ description: The request type.
668
+ type: string
669
+ enum: ["T", "U", "D", "I", "O"]
670
+ scope:
671
+ description: The scope of the transfer.
672
+ type: string
673
+ name:
674
+ description: The name of the transfer.
675
+ type: string
676
+ did_type:
677
+ description: The did type.
678
+ type: string
679
+ dest_rse_id:
680
+ description: The destination RSE id.
681
+ type: string
682
+ source_rse_id:
683
+ description: The source RSE id.
684
+ type: string
685
+ attributes:
686
+ description: All attributes associated with the request.
687
+ type: string
688
+ state:
689
+ description: The state of the request.
690
+ type: string
691
+ enum: ["Q", "G", "S", "F", "D", "L", "N", "O", "A", "M", "U", "W", "P"]
692
+ external_id:
693
+ description: External id of the request.
694
+ type: string
695
+ external_host:
696
+ description: External host of the request.
697
+ type: string
698
+ retry_count:
699
+ description: The numbers of attempted retires.
700
+ type: integer
701
+ err_msg:
702
+ description: An error message if one occurred.
703
+ type: string
704
+ previous_attempt_id:
705
+ description: The id of the previous attempt.
706
+ type: string
707
+ rule_id:
708
+ description: The id of the associated replication rule.
709
+ type: string
710
+ activity:
711
+ description: The activity of the request.
712
+ type: string
713
+ bytes:
714
+ description: The size of the did in bytes.
715
+ type: integer
716
+ md5:
717
+ description: The md5 checksum of the did to transfer.
718
+ type: string
719
+ adler32:
720
+ description: The adler32 checksum of the did to transfer.
721
+ type: string
722
+ dest_url:
723
+ description: The destination url.
724
+ type: string
725
+ submitted_at:
726
+ description: The time the request got submitted.
727
+ type: string
728
+ started_at:
729
+ description: The time the request got started.
730
+ type: string
731
+ transferred_at:
732
+ description: The time the request got transferred.
733
+ type: string
734
+ estimated_at:
735
+ description: The time the request got estimated.
736
+ type: string
737
+ submitter_id:
738
+ description: The id of the submitter.
739
+ type: string
740
+ estimated_stated_at:
741
+ description: The estimation of the started at value.
742
+ type: string
743
+ estimated_transferred_at:
744
+ description: The estimation of the transferred at value.
745
+ type: string
746
+ staging_started_at:
747
+ description: The time the staging got started.
748
+ type: string
749
+ staging_finished_at:
750
+ description: The time the staging got finished.
751
+ type: string
752
+ account:
753
+ description: The account which issued the request.
754
+ type: string
755
+ requested_at:
756
+ description: The time the request got requested.
757
+ type: string
758
+ priority:
759
+ description: The priority of the request.
760
+ type: integer
761
+ transfertool:
762
+ description: The transfertool used.
763
+ type: string
764
+ source_rse:
765
+ description: The name of the source RSE.
766
+ type: string
767
+ dest_rse:
768
+ description: The name of the destination RSE.
769
+ type: string
770
+ 401:
771
+ description: Invalid Auth Token
772
+ 404:
773
+ description: Not found
774
+ 406:
775
+ description: Not acceptable
776
+ """
777
+ src_rse = flask.request.args.get('src_rse', default=None)
778
+ dst_rse = flask.request.args.get('dst_rse', default=None)
779
+ src_site = flask.request.args.get('src_site', default=None)
780
+ dst_site = flask.request.args.get('dst_site', default=None)
781
+ request_states = flask.request.args.get('request_states', default=None)
782
+ offset = flask.request.args.get('offset', default=0)
783
+ limit = flask.request.args.get('limit', default=100)
784
+
785
+ if not request_states:
786
+ return generate_http_error_flask(400, 'MissingParameter', 'Request state is missing')
787
+ if src_rse and not dst_rse:
788
+ return generate_http_error_flask(400, 'MissingParameter', 'Destination RSE is missing')
789
+ elif dst_rse and not src_rse:
790
+ return generate_http_error_flask(400, 'MissingParameter', 'Source RSE is missing')
791
+ elif src_site and not dst_site:
792
+ return generate_http_error_flask(400, 'MissingParameter', 'Destination site is missing')
793
+ elif dst_site and not src_site:
794
+ return generate_http_error_flask(400, 'MissingParameter', 'Source site is missing')
795
+
796
+ try:
797
+ states = [RequestState(state) for state in request_states.split(',')]
798
+ except ValueError:
799
+ return generate_http_error_flask(400, 'Invalid', 'Request state value is invalid')
800
+
801
+ src_rses = []
802
+ dst_rses = []
803
+ if src_site:
804
+ src_rses = get_rses_with_attribute_value(key='site', value=src_site, vo=flask.request.environ.get('vo'))
805
+ if not src_rses:
806
+ return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {src_site} to RSE')
807
+ src_rses = [rse['rse_name'] for rse in src_rses]
808
+ dst_rses = get_rses_with_attribute_value(key='site', value=dst_site, vo=flask.request.environ.get('vo'))
809
+ if not dst_rses:
810
+ return generate_http_error_flask(404, 'NotFound', f'Could not resolve site name {dst_site} to RSE')
811
+ dst_rses = [rse['rse_name'] for rse in dst_rses]
812
+ else:
813
+ dst_rses = [dst_rse]
814
+ src_rses = [src_rse]
815
+
816
+ def generate(issuer, vo):
817
+ for result in request.list_requests_history(src_rses, dst_rses, states, issuer=issuer, vo=vo, offset=offset, limit=limit):
818
+ yield render_json(**result) + '\n'
819
+
820
+ return try_stream(generate(issuer=flask.request.environ.get('issuer'), vo=flask.request.environ.get('vo')))
821
+
822
+
823
+ class RequestMetricsGet(ErrorHandlingMethodView):
824
+ """ REST API to get request stats. """
825
+
826
+ @check_accept_header_wrapper_flask(['application/x-json-stream'])
827
+ def get(self):
828
+ """
829
+ ---
830
+ summary: Get Request Statistics
831
+ description: Get statistics of requests grouped by source, destination, and activity.
832
+ tags:
833
+ - Requests
834
+ parameters:
835
+ - name: dest_rse
836
+ in: query
837
+ description: The destination RSE name
838
+ schema:
839
+ type: string
840
+ - name: source_rse
841
+ in: query
842
+ description: The source RSE name
843
+ schema:
844
+ type: string
845
+ - name: activity
846
+ in: query
847
+ description: The activity
848
+ schema:
849
+ type: string
850
+ - name: group_by_rse_attribute
851
+ in: query
852
+ description: The parameter to group the RSEs by.
853
+ schema:
854
+ type: string
855
+ responses:
856
+ 200:
857
+ description: OK
858
+ content:
859
+ application/x-json-stream:
860
+ schema:
861
+ description: Statistics of requests by source, destination, and activity.
862
+ type: array
863
+ items:
864
+ description: Statistics of the request group for a given (source, destination, activity) tuple.
865
+ type: object
866
+ properties:
867
+ src_rse:
868
+ type: string
869
+ description: The name of this links source RSE
870
+ dst_rse:
871
+ type: string
872
+ description: The name of this links destination RSE
873
+ distance:
874
+ type: integer
875
+ description: The distance between the source and destination RSE
876
+ files:
877
+ type: object
878
+ properties:
879
+ done-total-1h:
880
+ type: integer
881
+ description: The total number of files successfully transferred in the last 1 hour
882
+ done-total-6h:
883
+ type: integer
884
+ description: The total number of files successfully transferred in the last 6 hours
885
+ failed-total-1h:
886
+ type: integer
887
+ description: The total number of transfer failures in the last 1 hour
888
+ failed-total-6h:
889
+ type: integer
890
+ description: The total number of transfer failures in the last 6 hours
891
+ queued-total:
892
+ type: integer
893
+ description: The total number of files queued in rucio
894
+ queued:
895
+ type: object
896
+ description: Per-activity number of queued files
897
+ additionalProperties:
898
+ type: integer
899
+ done:
900
+ type: object
901
+ additionalProperties:
902
+ type: object
903
+ properties:
904
+ 1h:
905
+ type: integer
906
+ 6h:
907
+ type: integer
908
+ failed:
909
+ type: object
910
+ description: Per-activity number of transfer failures in the last 1 and 6 hours
911
+ additionalProperties:
912
+ type: object
913
+ properties:
914
+ 1h:
915
+ type: integer
916
+ 6h:
917
+ type: integer
918
+ bytes:
919
+ type: object
920
+ properties:
921
+ done-total-1h:
922
+ type: integer
923
+ description: The total number of bytes successfully transferred in the last 1 hour
924
+ done-total-6h:
925
+ type: integer
926
+ description: The total number of bytes successfully transferred in the last 6 hours
927
+ queued-total:
928
+ type: integer
929
+ description: The total number of bytes queued to be transferred by rucio
930
+ queued:
931
+ type: object
932
+ description: Per-activity amount of queued bytes
933
+ additionalProperties:
934
+ type: integer
935
+ done:
936
+ type: object
937
+ description: Per-activity number of transferred bytes in the last 1 and 6 hours
938
+ additionalProperties:
939
+ type: object
940
+ properties:
941
+ 1h:
942
+ type: integer
943
+ 6h:
944
+ type: integer
945
+ required:
946
+ - distance
947
+ - src_rse
948
+ - dst_rse
949
+ 401:
950
+ description: Invalid Auth Token
951
+ """
952
+ dst_rse = flask.request.args.get('dst_rse', default=None)
953
+ src_rse = flask.request.args.get('src_rse', default=None)
954
+ activity = flask.request.args.get('activity', default=None)
955
+ group_by_rse_attribute = flask.request.args.get('group_by_rse_attribute', default=None)
956
+ format = flask.request.args.get('format', default=None)
957
+
958
+ metrics = request.get_request_metrics(
959
+ dst_rse=dst_rse,
960
+ src_rse=src_rse,
961
+ activity=activity,
962
+ group_by_rse_attribute=group_by_rse_attribute,
963
+ issuer=flask.request.environ.get('issuer'),
964
+ vo=flask.request.environ.get('vo')
965
+ )
966
+
967
+ if format == 'panda':
968
+ return Response(json.dumps(metrics, cls=APIEncoder), content_type='application/json')
969
+
970
+ def generate() -> "Iterator[str]":
971
+ for result in metrics.values():
972
+ yield render_json(**result) + '\n'
973
+ return try_stream(generate())
974
+
975
+
976
+ def blueprint():
977
+ bp = AuthenticatedBlueprint('requests', __name__, url_prefix='/requests')
978
+
979
+ request_get_view = RequestGet.as_view('request_get')
980
+ bp.add_url_rule('/<path:scope_name>/<rse>', view_func=request_get_view, methods=['get', ])
981
+ request_history_get_view = RequestHistoryGet.as_view('request_history_get')
982
+ bp.add_url_rule('/history/<path:scope_name>/<rse>', view_func=request_history_get_view, methods=['get', ])
983
+ request_list_view = RequestList.as_view('request_list')
984
+ bp.add_url_rule('/list', view_func=request_list_view, methods=['get', ])
985
+ request_history_list_view = RequestHistoryList.as_view('request_history_list')
986
+ bp.add_url_rule('/history/list', view_func=request_history_list_view, methods=['get', ])
987
+ request_metrics_view = RequestMetricsGet.as_view('request_metrics_get')
988
+ bp.add_url_rule('/metrics', view_func=request_metrics_view, methods=['get', ])
989
+
990
+ bp.after_request(response_headers)
991
+ return bp
992
+
993
+
994
+ def make_doc():
995
+ """ Only used for sphinx documentation """
996
+ doc_app = Flask(__name__)
997
+ doc_app.register_blueprint(blueprint())
998
+ return doc_app