rucio 35.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (493) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/client/__init__.py +15 -0
  4. rucio/client/accountclient.py +433 -0
  5. rucio/client/accountlimitclient.py +183 -0
  6. rucio/client/baseclient.py +974 -0
  7. rucio/client/client.py +76 -0
  8. rucio/client/configclient.py +126 -0
  9. rucio/client/credentialclient.py +59 -0
  10. rucio/client/didclient.py +866 -0
  11. rucio/client/diracclient.py +56 -0
  12. rucio/client/downloadclient.py +1785 -0
  13. rucio/client/exportclient.py +44 -0
  14. rucio/client/fileclient.py +50 -0
  15. rucio/client/importclient.py +42 -0
  16. rucio/client/lifetimeclient.py +90 -0
  17. rucio/client/lockclient.py +109 -0
  18. rucio/client/metaconventionsclient.py +140 -0
  19. rucio/client/pingclient.py +44 -0
  20. rucio/client/replicaclient.py +454 -0
  21. rucio/client/requestclient.py +125 -0
  22. rucio/client/rseclient.py +746 -0
  23. rucio/client/ruleclient.py +294 -0
  24. rucio/client/scopeclient.py +90 -0
  25. rucio/client/subscriptionclient.py +173 -0
  26. rucio/client/touchclient.py +82 -0
  27. rucio/client/uploadclient.py +955 -0
  28. rucio/common/__init__.py +13 -0
  29. rucio/common/cache.py +74 -0
  30. rucio/common/config.py +801 -0
  31. rucio/common/constants.py +159 -0
  32. rucio/common/constraints.py +17 -0
  33. rucio/common/didtype.py +189 -0
  34. rucio/common/dumper/__init__.py +335 -0
  35. rucio/common/dumper/consistency.py +452 -0
  36. rucio/common/dumper/data_models.py +318 -0
  37. rucio/common/dumper/path_parsing.py +64 -0
  38. rucio/common/exception.py +1151 -0
  39. rucio/common/extra.py +36 -0
  40. rucio/common/logging.py +420 -0
  41. rucio/common/pcache.py +1408 -0
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +84 -0
  44. rucio/common/schema/__init__.py +150 -0
  45. rucio/common/schema/atlas.py +413 -0
  46. rucio/common/schema/belleii.py +408 -0
  47. rucio/common/schema/domatpc.py +401 -0
  48. rucio/common/schema/escape.py +426 -0
  49. rucio/common/schema/generic.py +433 -0
  50. rucio/common/schema/generic_multi_vo.py +412 -0
  51. rucio/common/schema/icecube.py +406 -0
  52. rucio/common/stomp_utils.py +159 -0
  53. rucio/common/stopwatch.py +55 -0
  54. rucio/common/test_rucio_server.py +148 -0
  55. rucio/common/types.py +403 -0
  56. rucio/common/utils.py +2238 -0
  57. rucio/core/__init__.py +13 -0
  58. rucio/core/account.py +496 -0
  59. rucio/core/account_counter.py +236 -0
  60. rucio/core/account_limit.py +423 -0
  61. rucio/core/authentication.py +620 -0
  62. rucio/core/config.py +456 -0
  63. rucio/core/credential.py +225 -0
  64. rucio/core/did.py +3000 -0
  65. rucio/core/did_meta_plugins/__init__.py +252 -0
  66. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  68. rucio/core/did_meta_plugins/filter_engine.py +613 -0
  69. rucio/core/did_meta_plugins/json_meta.py +240 -0
  70. rucio/core/did_meta_plugins/mongo_meta.py +216 -0
  71. rucio/core/did_meta_plugins/postgres_meta.py +316 -0
  72. rucio/core/dirac.py +237 -0
  73. rucio/core/distance.py +187 -0
  74. rucio/core/exporter.py +59 -0
  75. rucio/core/heartbeat.py +363 -0
  76. rucio/core/identity.py +300 -0
  77. rucio/core/importer.py +259 -0
  78. rucio/core/lifetime_exception.py +377 -0
  79. rucio/core/lock.py +576 -0
  80. rucio/core/message.py +282 -0
  81. rucio/core/meta_conventions.py +203 -0
  82. rucio/core/monitor.py +447 -0
  83. rucio/core/naming_convention.py +195 -0
  84. rucio/core/nongrid_trace.py +136 -0
  85. rucio/core/oidc.py +1461 -0
  86. rucio/core/permission/__init__.py +119 -0
  87. rucio/core/permission/atlas.py +1348 -0
  88. rucio/core/permission/belleii.py +1077 -0
  89. rucio/core/permission/escape.py +1078 -0
  90. rucio/core/permission/generic.py +1130 -0
  91. rucio/core/permission/generic_multi_vo.py +1150 -0
  92. rucio/core/quarantined_replica.py +223 -0
  93. rucio/core/replica.py +4158 -0
  94. rucio/core/replica_sorter.py +366 -0
  95. rucio/core/request.py +3089 -0
  96. rucio/core/rse.py +1875 -0
  97. rucio/core/rse_counter.py +186 -0
  98. rucio/core/rse_expression_parser.py +459 -0
  99. rucio/core/rse_selector.py +302 -0
  100. rucio/core/rule.py +4483 -0
  101. rucio/core/rule_grouping.py +1618 -0
  102. rucio/core/scope.py +180 -0
  103. rucio/core/subscription.py +364 -0
  104. rucio/core/topology.py +490 -0
  105. rucio/core/trace.py +375 -0
  106. rucio/core/transfer.py +1517 -0
  107. rucio/core/vo.py +169 -0
  108. rucio/core/volatile_replica.py +150 -0
  109. rucio/daemons/__init__.py +13 -0
  110. rucio/daemons/abacus/__init__.py +13 -0
  111. rucio/daemons/abacus/account.py +116 -0
  112. rucio/daemons/abacus/collection_replica.py +124 -0
  113. rucio/daemons/abacus/rse.py +117 -0
  114. rucio/daemons/atropos/__init__.py +13 -0
  115. rucio/daemons/atropos/atropos.py +242 -0
  116. rucio/daemons/auditor/__init__.py +289 -0
  117. rucio/daemons/auditor/hdfs.py +97 -0
  118. rucio/daemons/auditor/srmdumps.py +355 -0
  119. rucio/daemons/automatix/__init__.py +13 -0
  120. rucio/daemons/automatix/automatix.py +293 -0
  121. rucio/daemons/badreplicas/__init__.py +13 -0
  122. rucio/daemons/badreplicas/minos.py +322 -0
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  124. rucio/daemons/badreplicas/necromancer.py +196 -0
  125. rucio/daemons/bb8/__init__.py +13 -0
  126. rucio/daemons/bb8/bb8.py +353 -0
  127. rucio/daemons/bb8/common.py +759 -0
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  129. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  130. rucio/daemons/c3po/__init__.py +13 -0
  131. rucio/daemons/c3po/algorithms/__init__.py +13 -0
  132. rucio/daemons/c3po/algorithms/simple.py +134 -0
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +128 -0
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +130 -0
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +294 -0
  136. rucio/daemons/c3po/c3po.py +371 -0
  137. rucio/daemons/c3po/collectors/__init__.py +13 -0
  138. rucio/daemons/c3po/collectors/agis.py +108 -0
  139. rucio/daemons/c3po/collectors/free_space.py +81 -0
  140. rucio/daemons/c3po/collectors/jedi_did.py +57 -0
  141. rucio/daemons/c3po/collectors/mock_did.py +51 -0
  142. rucio/daemons/c3po/collectors/network_metrics.py +71 -0
  143. rucio/daemons/c3po/collectors/workload.py +112 -0
  144. rucio/daemons/c3po/utils/__init__.py +13 -0
  145. rucio/daemons/c3po/utils/dataset_cache.py +50 -0
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +56 -0
  147. rucio/daemons/c3po/utils/expiring_list.py +62 -0
  148. rucio/daemons/c3po/utils/popularity.py +85 -0
  149. rucio/daemons/c3po/utils/timeseries.py +89 -0
  150. rucio/daemons/cache/__init__.py +13 -0
  151. rucio/daemons/cache/consumer.py +197 -0
  152. rucio/daemons/common.py +415 -0
  153. rucio/daemons/conveyor/__init__.py +13 -0
  154. rucio/daemons/conveyor/common.py +562 -0
  155. rucio/daemons/conveyor/finisher.py +529 -0
  156. rucio/daemons/conveyor/poller.py +404 -0
  157. rucio/daemons/conveyor/preparer.py +205 -0
  158. rucio/daemons/conveyor/receiver.py +249 -0
  159. rucio/daemons/conveyor/stager.py +132 -0
  160. rucio/daemons/conveyor/submitter.py +403 -0
  161. rucio/daemons/conveyor/throttler.py +532 -0
  162. rucio/daemons/follower/__init__.py +13 -0
  163. rucio/daemons/follower/follower.py +101 -0
  164. rucio/daemons/hermes/__init__.py +13 -0
  165. rucio/daemons/hermes/hermes.py +774 -0
  166. rucio/daemons/judge/__init__.py +13 -0
  167. rucio/daemons/judge/cleaner.py +159 -0
  168. rucio/daemons/judge/evaluator.py +185 -0
  169. rucio/daemons/judge/injector.py +162 -0
  170. rucio/daemons/judge/repairer.py +154 -0
  171. rucio/daemons/oauthmanager/__init__.py +13 -0
  172. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  173. rucio/daemons/reaper/__init__.py +13 -0
  174. rucio/daemons/reaper/dark_reaper.py +278 -0
  175. rucio/daemons/reaper/reaper.py +743 -0
  176. rucio/daemons/replicarecoverer/__init__.py +13 -0
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  178. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +13 -0
  186. rucio/daemons/storage/consistency/__init__.py +13 -0
  187. rucio/daemons/storage/consistency/actions.py +846 -0
  188. rucio/daemons/tracer/__init__.py +13 -0
  189. rucio/daemons/tracer/kronos.py +536 -0
  190. rucio/daemons/transmogrifier/__init__.py +13 -0
  191. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  192. rucio/daemons/undertaker/__init__.py +13 -0
  193. rucio/daemons/undertaker/undertaker.py +137 -0
  194. rucio/db/__init__.py +13 -0
  195. rucio/db/sqla/__init__.py +52 -0
  196. rucio/db/sqla/constants.py +201 -0
  197. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  198. rucio/db/sqla/migrate_repo/env.py +110 -0
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  327. rucio/db/sqla/models.py +1740 -0
  328. rucio/db/sqla/sautils.py +55 -0
  329. rucio/db/sqla/session.py +498 -0
  330. rucio/db/sqla/types.py +206 -0
  331. rucio/db/sqla/util.py +543 -0
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/gateway/account.py +339 -0
  334. rucio/gateway/account_limit.py +286 -0
  335. rucio/gateway/authentication.py +375 -0
  336. rucio/gateway/config.py +217 -0
  337. rucio/gateway/credential.py +71 -0
  338. rucio/gateway/did.py +970 -0
  339. rucio/gateway/dirac.py +81 -0
  340. rucio/gateway/exporter.py +59 -0
  341. rucio/gateway/heartbeat.py +74 -0
  342. rucio/gateway/identity.py +204 -0
  343. rucio/gateway/importer.py +45 -0
  344. rucio/gateway/lifetime_exception.py +120 -0
  345. rucio/gateway/lock.py +153 -0
  346. rucio/gateway/meta_conventions.py +87 -0
  347. rucio/gateway/permission.py +71 -0
  348. rucio/gateway/quarantined_replica.py +78 -0
  349. rucio/gateway/replica.py +529 -0
  350. rucio/gateway/request.py +321 -0
  351. rucio/gateway/rse.py +600 -0
  352. rucio/gateway/rule.py +417 -0
  353. rucio/gateway/scope.py +99 -0
  354. rucio/gateway/subscription.py +277 -0
  355. rucio/gateway/vo.py +122 -0
  356. rucio/rse/__init__.py +96 -0
  357. rucio/rse/protocols/__init__.py +13 -0
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +122 -0
  360. rucio/rse/protocols/dummy.py +111 -0
  361. rucio/rse/protocols/gfal.py +703 -0
  362. rucio/rse/protocols/globus.py +243 -0
  363. rucio/rse/protocols/gsiftp.py +92 -0
  364. rucio/rse/protocols/http_cache.py +82 -0
  365. rucio/rse/protocols/mock.py +123 -0
  366. rucio/rse/protocols/ngarc.py +209 -0
  367. rucio/rse/protocols/posix.py +250 -0
  368. rucio/rse/protocols/protocol.py +594 -0
  369. rucio/rse/protocols/rclone.py +364 -0
  370. rucio/rse/protocols/rfio.py +136 -0
  371. rucio/rse/protocols/srm.py +338 -0
  372. rucio/rse/protocols/ssh.py +413 -0
  373. rucio/rse/protocols/storm.py +206 -0
  374. rucio/rse/protocols/webdav.py +550 -0
  375. rucio/rse/protocols/xrootd.py +301 -0
  376. rucio/rse/rsemanager.py +764 -0
  377. rucio/tests/__init__.py +13 -0
  378. rucio/tests/common.py +270 -0
  379. rucio/tests/common_server.py +132 -0
  380. rucio/transfertool/__init__.py +13 -0
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +1596 -0
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +201 -0
  387. rucio/transfertool/globus_library.py +181 -0
  388. rucio/transfertool/mock.py +90 -0
  389. rucio/transfertool/transfertool.py +221 -0
  390. rucio/vcsversion.py +11 -0
  391. rucio/version.py +38 -0
  392. rucio/web/__init__.py +13 -0
  393. rucio/web/rest/__init__.py +13 -0
  394. rucio/web/rest/flaskapi/__init__.py +13 -0
  395. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  396. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  398. rucio/web/rest/flaskapi/v1/accounts.py +1089 -0
  399. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  400. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  401. rucio/web/rest/flaskapi/v1/common.py +426 -0
  402. rucio/web/rest/flaskapi/v1/config.py +304 -0
  403. rucio/web/rest/flaskapi/v1/credentials.py +212 -0
  404. rucio/web/rest/flaskapi/v1/dids.py +2334 -0
  405. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  406. rucio/web/rest/flaskapi/v1/export.py +75 -0
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  408. rucio/web/rest/flaskapi/v1/identities.py +261 -0
  409. rucio/web/rest/flaskapi/v1/import.py +132 -0
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  411. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  412. rucio/web/rest/flaskapi/v1/main.py +91 -0
  413. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  414. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  416. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  417. rucio/web/rest/flaskapi/v1/redirect.py +365 -0
  418. rucio/web/rest/flaskapi/v1/replicas.py +1890 -0
  419. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  420. rucio/web/rest/flaskapi/v1/rses.py +2239 -0
  421. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  422. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  424. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  425. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  426. rucio/web/rest/flaskapi/v1/traces.py +100 -0
  427. rucio/web/rest/flaskapi/v1/types.py +20 -0
  428. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  429. rucio/web/rest/main.py +18 -0
  430. rucio/web/rest/metrics.py +27 -0
  431. rucio/web/rest/ping.py +27 -0
  432. rucio-35.7.0.data/data/rucio/etc/alembic.ini.template +71 -0
  433. rucio-35.7.0.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  434. rucio-35.7.0.data/data/rucio/etc/globus-config.yml.template +5 -0
  435. rucio-35.7.0.data/data/rucio/etc/ldap.cfg.template +30 -0
  436. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  437. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  438. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  439. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  440. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  441. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  442. rucio-35.7.0.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  443. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.atlas.client.template +42 -0
  444. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.template +257 -0
  445. rucio-35.7.0.data/data/rucio/etc/rucio_multi_vo.cfg.template +234 -0
  446. rucio-35.7.0.data/data/rucio/requirements.server.txt +268 -0
  447. rucio-35.7.0.data/data/rucio/tools/bootstrap.py +34 -0
  448. rucio-35.7.0.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  449. rucio-35.7.0.data/data/rucio/tools/reset_database.py +40 -0
  450. rucio-35.7.0.data/scripts/rucio +2542 -0
  451. rucio-35.7.0.data/scripts/rucio-abacus-account +74 -0
  452. rucio-35.7.0.data/scripts/rucio-abacus-collection-replica +46 -0
  453. rucio-35.7.0.data/scripts/rucio-abacus-rse +78 -0
  454. rucio-35.7.0.data/scripts/rucio-admin +2447 -0
  455. rucio-35.7.0.data/scripts/rucio-atropos +60 -0
  456. rucio-35.7.0.data/scripts/rucio-auditor +205 -0
  457. rucio-35.7.0.data/scripts/rucio-automatix +50 -0
  458. rucio-35.7.0.data/scripts/rucio-bb8 +57 -0
  459. rucio-35.7.0.data/scripts/rucio-c3po +85 -0
  460. rucio-35.7.0.data/scripts/rucio-cache-client +134 -0
  461. rucio-35.7.0.data/scripts/rucio-cache-consumer +42 -0
  462. rucio-35.7.0.data/scripts/rucio-conveyor-finisher +58 -0
  463. rucio-35.7.0.data/scripts/rucio-conveyor-poller +66 -0
  464. rucio-35.7.0.data/scripts/rucio-conveyor-preparer +37 -0
  465. rucio-35.7.0.data/scripts/rucio-conveyor-receiver +43 -0
  466. rucio-35.7.0.data/scripts/rucio-conveyor-stager +76 -0
  467. rucio-35.7.0.data/scripts/rucio-conveyor-submitter +139 -0
  468. rucio-35.7.0.data/scripts/rucio-conveyor-throttler +104 -0
  469. rucio-35.7.0.data/scripts/rucio-dark-reaper +53 -0
  470. rucio-35.7.0.data/scripts/rucio-dumper +160 -0
  471. rucio-35.7.0.data/scripts/rucio-follower +44 -0
  472. rucio-35.7.0.data/scripts/rucio-hermes +54 -0
  473. rucio-35.7.0.data/scripts/rucio-judge-cleaner +89 -0
  474. rucio-35.7.0.data/scripts/rucio-judge-evaluator +137 -0
  475. rucio-35.7.0.data/scripts/rucio-judge-injector +44 -0
  476. rucio-35.7.0.data/scripts/rucio-judge-repairer +44 -0
  477. rucio-35.7.0.data/scripts/rucio-kronos +43 -0
  478. rucio-35.7.0.data/scripts/rucio-minos +53 -0
  479. rucio-35.7.0.data/scripts/rucio-minos-temporary-expiration +50 -0
  480. rucio-35.7.0.data/scripts/rucio-necromancer +120 -0
  481. rucio-35.7.0.data/scripts/rucio-oauth-manager +63 -0
  482. rucio-35.7.0.data/scripts/rucio-reaper +83 -0
  483. rucio-35.7.0.data/scripts/rucio-replica-recoverer +248 -0
  484. rucio-35.7.0.data/scripts/rucio-rse-decommissioner +66 -0
  485. rucio-35.7.0.data/scripts/rucio-storage-consistency-actions +74 -0
  486. rucio-35.7.0.data/scripts/rucio-transmogrifier +77 -0
  487. rucio-35.7.0.data/scripts/rucio-undertaker +76 -0
  488. rucio-35.7.0.dist-info/METADATA +72 -0
  489. rucio-35.7.0.dist-info/RECORD +493 -0
  490. rucio-35.7.0.dist-info/WHEEL +5 -0
  491. rucio-35.7.0.dist-info/licenses/AUTHORS.rst +97 -0
  492. rucio-35.7.0.dist-info/licenses/LICENSE +201 -0
  493. rucio-35.7.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,13 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
rucio/tests/common.py ADDED
@@ -0,0 +1,270 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import contextlib
16
+ import itertools
17
+ import json
18
+ import os
19
+ import tempfile
20
+ from collections import namedtuple
21
+ from collections.abc import Callable, Iterable
22
+ from functools import wraps
23
+ from os import rename
24
+ from random import choice, choices
25
+ from string import ascii_letters, ascii_uppercase, digits
26
+ from typing import Any, Optional
27
+
28
+ import pytest
29
+ import requests
30
+
31
+ from rucio.common.config import config_get, config_get_bool, get_config_dirs
32
+ from rucio.common.utils import execute
33
+ from rucio.common.utils import generate_uuid as uuid
34
+
35
+ skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
36
+ reason='fails if no rse-accounts.cfg found')
37
+ skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and os.environ['RDBMS'] == 'sqlite',
38
+ reason="does not work in SQLite because of missing features")
39
+ skip_multivo = pytest.mark.skipif('SUITE' in os.environ and os.environ['SUITE'] == 'multi_vo',
40
+ reason="does not work for multiVO")
41
+ skip_non_belleii = pytest.mark.skipif(not ('POLICY' in os.environ and os.environ['POLICY'] == 'belleii'),
42
+ reason="specific belleii tests")
43
+
44
+
45
+ def is_influxdb_available() -> bool:
46
+ """Return True if influxdb is available, else return False."""
47
+ try:
48
+ response = requests.get('http://localhost:8086/ping')
49
+ return response.status_code == 204
50
+ except requests.exceptions.ConnectionError:
51
+ print('InfluxDB is not running at localhost:8086')
52
+ return False
53
+
54
+
55
+ def is_elasticsearch_available() -> bool:
56
+ """Return True if elasticsearch is available, else return False."""
57
+ try:
58
+ response = requests.get('http://localhost:9200/')
59
+ return response.status_code == 200
60
+ except requests.exceptions.ConnectionError:
61
+ print('Elasticsearch is not running at localhost:9200')
62
+ return False
63
+
64
+
65
+ skip_missing_elasticsearch_influxdb_in_env = pytest.mark.skipif(not (is_influxdb_available() and is_elasticsearch_available()), reason='influxdb is not available')
66
+
67
+
68
+ def get_long_vo() -> str:
69
+ """ Get the VO name from the config file for testing.
70
+ Don't map the name to a short version.
71
+ :returns: VO name string.
72
+ """
73
+ vo_name = 'def'
74
+ if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
75
+ vo = config_get('client', 'vo', raise_exception=False, default=None)
76
+ if vo is not None:
77
+ vo_name = vo
78
+ return vo_name
79
+
80
+
81
+ def account_name_generator() -> str:
82
+ """ Generate random account name.
83
+
84
+ :returns: A random account name
85
+ """
86
+ return 'jdoe-' + str(uuid()).lower()[:16]
87
+
88
+
89
+ def scope_name_generator() -> str:
90
+ """ Generate random scope name.
91
+
92
+ :returns: A random scope name
93
+ """
94
+ return 'mock_' + str(uuid()).lower()[:16]
95
+
96
+
97
+ def did_name_generator(did_type: str = 'file', name_prefix: str = '', name_suffix: str = '', path: Optional[str] = None) -> str:
98
+ """ Generate random did name.
99
+ :param did_type: A string to create a meaningful did_name depending on the did_type (file, dataset, container)
100
+ :param name_prefix: String to prefix to the did name
101
+ :param name_suffix: String to append to the did name
102
+ :param path: If specified, use the path to generate the did_name
103
+
104
+ :returns: A random did name
105
+ """
106
+ if os.getenv('POLICY') == 'belleii':
107
+ if path is not None:
108
+ return path
109
+
110
+ container_path = os.path.join("/belle", name_prefix if name_prefix else "mock", 'cont_%s' % str(uuid()))
111
+ if did_type == 'container':
112
+ return container_path
113
+
114
+ dataset_path = os.path.join(container_path, 'dataset_%s' % str(uuid()))
115
+ if did_type == 'dataset':
116
+ return dataset_path
117
+
118
+ file_path = os.path.join(dataset_path, 'file_%s%s' % (str(uuid()), name_suffix))
119
+ return file_path
120
+
121
+ if path is not None:
122
+ return os.path.basename(path)
123
+
124
+ return '%s%s_%s%s' % (name_prefix, did_type, str(uuid()), name_suffix)
125
+
126
+
127
+ def rse_name_generator(size: int = 10) -> str:
128
+ """ Generate random RSE name.
129
+
130
+ :returns: A random RSE name
131
+ """
132
+ return 'MOCK-' + ''.join(choice(ascii_uppercase) for x in range(size)) # noqa: S311
133
+
134
+
135
+ def rfc2253_dn_generator() -> str:
136
+ """ Generate a random DN in RFC 2253 format.
137
+
138
+ :returns: A random DN
139
+ """
140
+ random_cn = ''.join(choices(ascii_letters + digits, k=8)) # noqa: S311
141
+ random_o = ''.join(choices(ascii_letters + digits, k=8)) # noqa: S311
142
+ random_c = ''.join(choices(ascii_letters, k=2)) # noqa: S311
143
+ random_dn = "CN={}, O={}, C={}".format(random_cn, random_o, random_c)
144
+ return random_dn
145
+
146
+
147
+ def file_generator(size: int = 2, namelen: int = 10) -> str:
148
+ """ Create a bogus file and returns it's name.
149
+ :param size: size in bytes
150
+ :returns: The name of the generated file.
151
+ """
152
+ fn = '/tmp/file_' + ''.join(choice(ascii_uppercase) for x in range(namelen)) # noqa: S311
153
+ execute('dd if=/dev/urandom of={0} count={1} bs=1'.format(fn, size))
154
+ return fn
155
+
156
+
157
+ def make_temp_file(dir_: str, data: str) -> str:
158
+ """
159
+ Creates a temporal file and write `data` on it.
160
+ :param data: String to be written on the created file.
161
+ :returns: Name of the temporal file.
162
+ """
163
+ fd, path = tempfile.mkstemp(dir=dir_)
164
+ with os.fdopen(fd, 'w', encoding='utf-8') as f:
165
+ f.write(data)
166
+ return path
167
+
168
+
169
+ @contextlib.contextmanager
170
+ def mock_open(module, file_like_object):
171
+ call_info = {}
172
+
173
+ def mocked_open(filename, mode='r'):
174
+ call_info['filename'] = filename
175
+ call_info['mode'] = mode
176
+ file_like_object.close = lambda: None
177
+ return contextlib.closing(file_like_object)
178
+
179
+ setattr(module, 'open', mocked_open)
180
+ try:
181
+ yield call_info
182
+ finally:
183
+ file_like_object.seek(0)
184
+ delattr(module, 'open')
185
+
186
+
187
+ def print_response(rest_response):
188
+ print('Status:', rest_response.status)
189
+ print()
190
+ nohdrs = True
191
+ for hdr, val in rest_response.headers.items():
192
+ if nohdrs:
193
+ print('Headers:')
194
+ print('-------')
195
+ nohdrs = False
196
+ print('%s: %s' % (hdr, val))
197
+
198
+ if not nohdrs:
199
+ print()
200
+
201
+ text = rest_response.get_data(as_text=True)
202
+ print(text if text else '<no content>')
203
+
204
+
205
+ def headers(*iterables: Iterable):
206
+ return list(itertools.chain(*iterables))
207
+
208
+
209
+ def loginhdr(account: str, username: str, password: str):
210
+ yield 'X-Rucio-Account', str(account)
211
+ yield 'X-Rucio-Username', str(username)
212
+ yield 'X-Rucio-Password', str(password)
213
+
214
+
215
+ def auth(token):
216
+ yield 'X-Rucio-Auth-Token', str(token)
217
+
218
+
219
+ def vohdr(vo: str):
220
+ if vo:
221
+ yield 'X-Rucio-VO', str(vo)
222
+
223
+
224
+ def hdrdict(dictionary: dict):
225
+ for key in dictionary:
226
+ yield str(key), str(dictionary[key])
227
+
228
+
229
+ def accept(mimetype):
230
+ yield 'Accept', mimetype
231
+
232
+
233
+ class Mime:
234
+ """ Enum-type class for mimetypes. """
235
+ METALINK = 'application/metalink4+xml'
236
+ JSON = 'application/json'
237
+ JSON_STREAM = 'application/x-json-stream'
238
+ BINARY = 'application/octet-stream'
239
+
240
+
241
+ def load_test_conf_file(file_name: str) -> dict[str, Any]:
242
+ config_dir = next(filter(lambda d: os.path.exists(os.path.join(d, file_name)), get_config_dirs()))
243
+ with open(os.path.join(config_dir, file_name)) as f:
244
+ return json.load(f)
245
+
246
+
247
+ def remove_config(func: Callable) -> Callable:
248
+ @wraps(func)
249
+ def wrapper(*args, **kwargs):
250
+ for configfile in get_config_dirs():
251
+ # Rename the config to <config>.tmp
252
+ try:
253
+ rename(f"{configfile}rucio.cfg", f"{configfile}rucio.cfg.tmp")
254
+ except FileNotFoundError:
255
+ pass # When a test uses a os.env assigned conf, there's nothing stating the default location has something
256
+ try:
257
+ # Execute the test
258
+ func(*args, **kwargs)
259
+ finally:
260
+ # And put the config back
261
+ for configfile in get_config_dirs():
262
+ try:
263
+ rename(f"{configfile}rucio.cfg.tmp", f"{configfile}rucio.cfg")
264
+ except FileNotFoundError:
265
+ pass
266
+
267
+ return wrapper
268
+
269
+
270
+ RSE_namedtuple = namedtuple('RSE_namedtuple', ['name', 'id'])
@@ -0,0 +1,132 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import copy
15
+
16
+ from sqlalchemy import and_, delete, exists, select
17
+ from sqlalchemy.orm import aliased
18
+
19
+ from rucio.core import config as core_config
20
+ from rucio.core.vo import map_vo
21
+ from rucio.db.sqla import models
22
+ from rucio.db.sqla.session import get_session, transactional_session
23
+
24
+ from .common import get_long_vo
25
+
26
+ # Functions containing server-only includes that can't be included in client tests
27
+ # For each table, get the foreign key constraints from all other tables towards this table.
28
+ INBOUND_FOREIGN_KEYS = {}
29
+ for __table in models.BASE.metadata.tables.values():
30
+ for __fk in __table.foreign_key_constraints:
31
+ INBOUND_FOREIGN_KEYS.setdefault(__fk.referred_table, set()).add(__fk)
32
+
33
+
34
+ def _dependency_paths(stack, nb_times_in_stack, cur_table):
35
+ """
36
+ Generates lists of foreign keys: paths starting at cur_table and
37
+ navigating the table graph via foreign key constraints.
38
+
39
+ For example: As of time of writing, for cur_table = models.ReplicationRule.__table__,
40
+ it will generate
41
+ [DATASET_LOCKS_RULE_ID_FK] # rule.id <-> dataset_locks.rule_id
42
+ [LOCKS_RULE_ID_FK] # rule.id <-> locks.rule_id
43
+ [RULES_CHILD_RULE_ID_FK, DATASET_LOCKS_RULE_ID_FK] # rule.id <-> rule(alias).child_rule_id, rule(alias).id <-> dataset_locks.rule_id
44
+ [RULES_CHILD_RULE_ID_FK, LOCKS_RULE_ID_FK] # rule.id <-> rule(alias).child_rule_id, rule(alias).id <-> locks.rule_id
45
+ [RULES_CHILD_RULE_ID_FK] # rule.id <-> rule(alias).child_rule_id
46
+ """
47
+ nb_times_in_stack[cur_table] = nb_times_in_stack.get(cur_table, 0) + 1
48
+
49
+ for fk in INBOUND_FOREIGN_KEYS.get(cur_table, []):
50
+ if nb_times_in_stack.get(fk.table, 0) > 1:
51
+ # Only allow a table to appear twice in the stack.
52
+ # This handles recursive constraints (like the one between rules and itself)
53
+ continue
54
+ stack.append(fk)
55
+ yield from _dependency_paths(stack, nb_times_in_stack, fk.table)
56
+
57
+ if stack:
58
+ yield copy.copy(stack)
59
+ fk = stack.pop()
60
+ nb_times_in_stack[fk.table] -= 1
61
+
62
+
63
+ @transactional_session
64
+ def cleanup_db_deps(model, select_rows_stmt, *, session=None):
65
+ """
66
+ Removes rows which have foreign key constraints pointing to rows
67
+ selected by `select_rows_stmt` in `model`. The deletion is transitive.
68
+ This implements a behavior similar to "ON DELETE CASCADE", but without
69
+ removing the initial rows from `model`, only their dependencies.
70
+ """
71
+
72
+ for fk_path in _dependency_paths(stack=[], nb_times_in_stack={}, cur_table=model.__table__):
73
+ seen_tables = set()
74
+ referred_table = model.__table__
75
+ current_table = fk_path[-1].table
76
+ filters = []
77
+ for i, fk in enumerate(fk_path):
78
+ current_table = fk.table
79
+ if current_table in seen_tables:
80
+ current_table = aliased(current_table)
81
+ else:
82
+ seen_tables.add(current_table)
83
+
84
+ filters.append(and_(current_table.columns.get(e.parent.name) == referred_table.columns.get(e.column.name) for e in fk.elements))
85
+ referred_table = current_table
86
+
87
+ if session.bind.dialect.name == 'mysql':
88
+ stmt = delete(
89
+ current_table
90
+ ).where(
91
+ and_(*filters)
92
+ ).where(
93
+ select_rows_stmt
94
+ )
95
+ else:
96
+ stmt = delete(
97
+ current_table,
98
+ ).where(
99
+ exists(
100
+ select(
101
+ 1
102
+ ).where(
103
+ and_(*filters)
104
+ ).where(
105
+ select_rows_stmt
106
+ )
107
+ )
108
+ )
109
+
110
+ stmt = stmt.execution_options(
111
+ synchronize_session=False
112
+ )
113
+
114
+ session.execute(stmt)
115
+
116
+
117
+ def reset_config_table():
118
+ """ Clear the config table and install any default entries needed for the tests.
119
+ """
120
+ db_session = get_session()
121
+ db_session.query(models.Config).delete()
122
+ db_session.commit()
123
+ core_config.set("vo-map", "testvo1", "tst")
124
+ core_config.set("vo-map", "testvo2", "ts2")
125
+
126
+
127
+ def get_vo():
128
+ """ Gets the current short/mapped VO name for testing.
129
+ Maps the vo name to the short name, if configured.
130
+ :returns: VO name string.
131
+ """
132
+ return map_vo(get_long_vo())
@@ -0,0 +1,13 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -0,0 +1,199 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import base64
16
+ import logging
17
+ from collections.abc import Mapping, Sequence
18
+ from os import path
19
+ from typing import TYPE_CHECKING, Any, Optional
20
+
21
+ from rucio.common import types
22
+ from rucio.common.config import config_get
23
+ from rucio.common.constants import RseAttr
24
+ from rucio.common.extra import import_extras
25
+ from rucio.common.utils import construct_torrent
26
+ from rucio.core.did_meta_plugins import get_metadata
27
+ from rucio.transfertool.transfertool import TransferStatusReport, Transfertool, TransferToolBuilder
28
+
29
+ from .bittorrent_driver import BittorrentDriver
30
+
31
+ if TYPE_CHECKING:
32
+ from rucio.core.request import DirectTransfer
33
+ from rucio.core.rse import RseData
34
+
35
+ DRIVER_NAME_RSE_ATTRIBUTE = 'bittorrent_driver'
36
+ DRIVER_CLASSES_BY_NAME: dict[str, type[BittorrentDriver]] = {}
37
+
38
+ EXTRA_MODULES = import_extras(['qbittorrentapi'])
39
+
40
+ if EXTRA_MODULES['qbittorrentapi']:
41
+ from .bittorrent_driver_qbittorrent import QBittorrentDriver
42
+ DRIVER_CLASSES_BY_NAME[QBittorrentDriver.external_name] = QBittorrentDriver
43
+
44
+
45
+ class BittorrentTransfertool(Transfertool):
46
+ """
47
+ Use bittorrent to perform the peer-to-peer transfer.
48
+ """
49
+ external_name = 'bittorrent'
50
+ supported_schemes = {'magnet'}
51
+
52
+ required_rse_attrs = (DRIVER_NAME_RSE_ATTRIBUTE, )
53
+
54
+ def __init__(self, external_host: str, logger: types.LoggerFunction = logging.log) -> None:
55
+ super().__init__(external_host=external_host, logger=logger)
56
+
57
+ self._drivers_by_rse_id = {}
58
+ self.ca_cert, self.ca_key = None, None
59
+
60
+ self.tracker = config_get('transfers', 'bittorrent_tracker_addr', raise_exception=False, default=None)
61
+
62
+ @classmethod
63
+ def _pick_management_api_driver_cls(cls: "type[BittorrentTransfertool]", rse: "RseData") -> Optional[type[BittorrentDriver]]:
64
+ driver_cls = DRIVER_CLASSES_BY_NAME.get(rse.attributes.get(DRIVER_NAME_RSE_ATTRIBUTE, ''))
65
+ if driver_cls is None:
66
+ return None
67
+ if not all(rse.attributes.get(attribute) is not None for attribute in driver_cls.required_rse_attrs):
68
+ return None
69
+ return driver_cls
70
+
71
+ def _driver_for_rse(self, rse: "RseData") -> Optional[BittorrentDriver]:
72
+ driver = self._drivers_by_rse_id.get(rse.id)
73
+ if driver:
74
+ return driver
75
+
76
+ driver_cls = self._pick_management_api_driver_cls(rse)
77
+ if not driver_cls:
78
+ return None
79
+
80
+ driver = driver_cls.make_driver(rse)
81
+ self._drivers_by_rse_id[rse.id] = driver
82
+ return driver
83
+
84
+ @staticmethod
85
+ def _get_torrent_meta(scope: "types.InternalScope", name: str) -> tuple[bytes, bytes, int]:
86
+ meta = get_metadata(scope=scope, name=name, plugin='all')
87
+ pieces_root = base64.b64decode(meta.get('bittorrent_pieces_root', ''))
88
+ pieces_layers = base64.b64decode(meta.get('bittorrent_pieces_layers', ''))
89
+ piece_length = meta.get('bittorrent_piece_length', 0)
90
+ return pieces_root, pieces_layers, piece_length
91
+
92
+ @classmethod
93
+ def submission_builder_for_path(
94
+ cls: "type[BittorrentTransfertool]",
95
+ transfer_path: "list[DirectTransfer]",
96
+ logger: types.LoggerFunction = logging.log
97
+ ) -> "tuple[list[DirectTransfer], Optional[TransferToolBuilder]]":
98
+ hop = transfer_path[0]
99
+ if hop.rws.byte_count == 0:
100
+ logger(logging.INFO, f"Bittorrent cannot transfer fully empty torrents. Skipping {hop}")
101
+ return [], None
102
+
103
+ if not cls.can_perform_transfer(hop.src.rse, hop.dst.rse):
104
+ logger(logging.INFO, f"The required RSE attributes are not set. Skipping {hop}")
105
+ return [], None
106
+
107
+ for rse in [hop.src.rse, hop.dst.rse]:
108
+ driver_cls = cls._pick_management_api_driver_cls(rse)
109
+ if not driver_cls:
110
+ logger(logging.INFO, f"The rse '{rse}' is not configured correctly for bittorrent")
111
+ return [], None
112
+
113
+ pieces_root, _pieces_layers, piece_length = cls._get_torrent_meta(hop.rws.scope, hop.rws.name)
114
+ if not pieces_root or not piece_length:
115
+ logger(logging.INFO, "The required bittorrent metadata not set on the DID")
116
+ return [], None
117
+
118
+ return [hop], TransferToolBuilder(cls, external_host='Bittorrent Transfertool')
119
+
120
+ def group_into_submit_jobs(self, transfer_paths: "Sequence[list[DirectTransfer]]") -> list[dict[str, Any]]:
121
+ return [{'transfers': transfer_path, 'job_params': {}} for transfer_path in transfer_paths]
122
+
123
+ @staticmethod
124
+ def _connect_directly(torrent_id: str, peers_drivers: Sequence[BittorrentDriver]) -> None:
125
+ peer_addr = []
126
+ for i, driver in enumerate(peers_drivers):
127
+ peer_addr.append(driver.listen_addr())
128
+
129
+ for driver in peers_drivers:
130
+ driver.add_peers(torrent_id=torrent_id, peers=peer_addr)
131
+
132
+ def submit(self, transfers: "Sequence[DirectTransfer]", job_params: dict[str, str], timeout: Optional[int] = None) -> str:
133
+ [transfer] = transfers
134
+ rws = transfer.rws
135
+
136
+ tracker = transfer.dst.rse.attributes.get(RseAttr.BITTORRENT_TRACKER_ADDR, self.tracker)
137
+
138
+ src_drivers = {}
139
+ for source in transfer.sources:
140
+ driver = self._driver_for_rse(source.rse)
141
+ if driver:
142
+ src_drivers[source] = driver
143
+
144
+ dst_driver = self._driver_for_rse(transfer.dst.rse)
145
+
146
+ if not dst_driver or not src_drivers:
147
+ raise Exception('Cannot initialize bittorrent drivers to submit transfers')
148
+
149
+ pieces_root, pieces_layers, piece_length = self._get_torrent_meta(rws.scope, rws.name)
150
+ torrent_id, torrent = construct_torrent(
151
+ scope=str(rws.scope),
152
+ name=rws.name,
153
+ length=rws.byte_count,
154
+ piece_length=piece_length,
155
+ pieces_root=pieces_root,
156
+ pieces_layers=pieces_layers,
157
+ trackers=[tracker] if tracker else None,
158
+ )
159
+
160
+ for source, driver in src_drivers.items():
161
+ source_protocol = transfer.source_protocol(source)
162
+ [lfn] = source_protocol.parse_pfns([transfer.source_url(source)]).values()
163
+ driver.add_torrent(
164
+ file_name=rws.name,
165
+ file_content=torrent,
166
+ download_location=lfn['prefix'] + path.dirname(lfn['path']),
167
+ seed_mode=True,
168
+ )
169
+
170
+ dest_protocol = transfer.dest_protocol()
171
+ [lfn] = dest_protocol.parse_pfns([transfer.dest_url]).values()
172
+ dst_driver.add_torrent(
173
+ file_name=rws.name,
174
+ file_content=torrent,
175
+ download_location=lfn['prefix'] + lfn['path'],
176
+ )
177
+
178
+ self._connect_directly(torrent_id, [dst_driver] + list(src_drivers.values()))
179
+ return torrent_id
180
+
181
+ def bulk_query(self, requests_by_eid, timeout: Optional[int] = None) -> Mapping[str, Mapping[str, TransferStatusReport]]:
182
+ response = {}
183
+ for transfer_id, requests in requests_by_eid.items():
184
+ for request_id, request in requests.items():
185
+ driver = self._driver_for_rse(request['dst_rse'])
186
+ if not driver:
187
+ self.logger(f'Cannot instantiate BitTorrent driver for {request["dest_rse"]}')
188
+ continue
189
+ response.setdefault(transfer_id, {})[request_id] = driver.get_status(request_id=request_id, torrent_id=transfer_id)
190
+ return response
191
+
192
+ def query(self, transfer_ids: Sequence[str], details: bool = False, timeout: Optional[int] = None) -> None:
193
+ pass
194
+
195
+ def cancel(self, transfer_ids: Sequence[str], timeout: Optional[int] = None) -> None:
196
+ pass
197
+
198
+ def update_priority(self, transfer_id: str, priority: int, timeout: Optional[int] = None) -> None:
199
+ pass
@@ -0,0 +1,52 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import logging
16
+ from abc import ABCMeta, abstractmethod
17
+ from collections.abc import Sequence
18
+ from typing import TYPE_CHECKING
19
+
20
+ from rucio.common import types
21
+
22
+ if TYPE_CHECKING:
23
+ from typing import Optional
24
+
25
+ from rucio.core.rse import RseData
26
+ from rucio.transfertool.transfertool import TransferStatusReport
27
+
28
+
29
+ class BittorrentDriver(metaclass=ABCMeta):
30
+ external_name = ''
31
+ required_rse_attrs = tuple()
32
+
33
+ @classmethod
34
+ @abstractmethod
35
+ def make_driver(cls: "type[BittorrentDriver]", rse: "RseData", logger: types.LoggerFunction = logging.log) -> "Optional[BittorrentDriver]":
36
+ pass
37
+
38
+ @abstractmethod
39
+ def listen_addr(self) -> tuple[str, int]:
40
+ pass
41
+
42
+ @abstractmethod
43
+ def add_torrent(self, file_name: str, file_content: bytes, download_location: str, seed_mode: bool = False) -> None:
44
+ pass
45
+
46
+ @abstractmethod
47
+ def add_peers(self, torrent_id: str, peers: Sequence[tuple[str, int]]) -> None:
48
+ pass
49
+
50
+ @abstractmethod
51
+ def get_status(self, request_id: str, torrent_id: str) -> "TransferStatusReport":
52
+ pass