rucio 37.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (487) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/cli/__init__.py +14 -0
  4. rucio/cli/account.py +216 -0
  5. rucio/cli/bin_legacy/__init__.py +13 -0
  6. rucio/cli/bin_legacy/rucio.py +2825 -0
  7. rucio/cli/bin_legacy/rucio_admin.py +2500 -0
  8. rucio/cli/command.py +272 -0
  9. rucio/cli/config.py +72 -0
  10. rucio/cli/did.py +191 -0
  11. rucio/cli/download.py +128 -0
  12. rucio/cli/lifetime_exception.py +33 -0
  13. rucio/cli/replica.py +162 -0
  14. rucio/cli/rse.py +293 -0
  15. rucio/cli/rule.py +158 -0
  16. rucio/cli/scope.py +40 -0
  17. rucio/cli/subscription.py +73 -0
  18. rucio/cli/upload.py +60 -0
  19. rucio/cli/utils.py +226 -0
  20. rucio/client/__init__.py +15 -0
  21. rucio/client/accountclient.py +432 -0
  22. rucio/client/accountlimitclient.py +183 -0
  23. rucio/client/baseclient.py +983 -0
  24. rucio/client/client.py +120 -0
  25. rucio/client/configclient.py +126 -0
  26. rucio/client/credentialclient.py +59 -0
  27. rucio/client/didclient.py +868 -0
  28. rucio/client/diracclient.py +56 -0
  29. rucio/client/downloadclient.py +1783 -0
  30. rucio/client/exportclient.py +44 -0
  31. rucio/client/fileclient.py +50 -0
  32. rucio/client/importclient.py +42 -0
  33. rucio/client/lifetimeclient.py +90 -0
  34. rucio/client/lockclient.py +109 -0
  35. rucio/client/metaconventionsclient.py +140 -0
  36. rucio/client/pingclient.py +44 -0
  37. rucio/client/replicaclient.py +452 -0
  38. rucio/client/requestclient.py +125 -0
  39. rucio/client/richclient.py +317 -0
  40. rucio/client/rseclient.py +746 -0
  41. rucio/client/ruleclient.py +294 -0
  42. rucio/client/scopeclient.py +90 -0
  43. rucio/client/subscriptionclient.py +173 -0
  44. rucio/client/touchclient.py +82 -0
  45. rucio/client/uploadclient.py +969 -0
  46. rucio/common/__init__.py +13 -0
  47. rucio/common/bittorrent.py +234 -0
  48. rucio/common/cache.py +111 -0
  49. rucio/common/checksum.py +168 -0
  50. rucio/common/client.py +122 -0
  51. rucio/common/config.py +788 -0
  52. rucio/common/constants.py +217 -0
  53. rucio/common/constraints.py +17 -0
  54. rucio/common/didtype.py +237 -0
  55. rucio/common/dumper/__init__.py +342 -0
  56. rucio/common/dumper/consistency.py +497 -0
  57. rucio/common/dumper/data_models.py +362 -0
  58. rucio/common/dumper/path_parsing.py +75 -0
  59. rucio/common/exception.py +1208 -0
  60. rucio/common/extra.py +31 -0
  61. rucio/common/logging.py +420 -0
  62. rucio/common/pcache.py +1409 -0
  63. rucio/common/plugins.py +185 -0
  64. rucio/common/policy.py +93 -0
  65. rucio/common/schema/__init__.py +200 -0
  66. rucio/common/schema/generic.py +416 -0
  67. rucio/common/schema/generic_multi_vo.py +395 -0
  68. rucio/common/stomp_utils.py +423 -0
  69. rucio/common/stopwatch.py +55 -0
  70. rucio/common/test_rucio_server.py +154 -0
  71. rucio/common/types.py +483 -0
  72. rucio/common/utils.py +1688 -0
  73. rucio/core/__init__.py +13 -0
  74. rucio/core/account.py +496 -0
  75. rucio/core/account_counter.py +236 -0
  76. rucio/core/account_limit.py +425 -0
  77. rucio/core/authentication.py +620 -0
  78. rucio/core/config.py +437 -0
  79. rucio/core/credential.py +224 -0
  80. rucio/core/did.py +3004 -0
  81. rucio/core/did_meta_plugins/__init__.py +252 -0
  82. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  83. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  84. rucio/core/did_meta_plugins/elasticsearch_meta.py +407 -0
  85. rucio/core/did_meta_plugins/filter_engine.py +672 -0
  86. rucio/core/did_meta_plugins/json_meta.py +240 -0
  87. rucio/core/did_meta_plugins/mongo_meta.py +229 -0
  88. rucio/core/did_meta_plugins/postgres_meta.py +352 -0
  89. rucio/core/dirac.py +237 -0
  90. rucio/core/distance.py +187 -0
  91. rucio/core/exporter.py +59 -0
  92. rucio/core/heartbeat.py +363 -0
  93. rucio/core/identity.py +301 -0
  94. rucio/core/importer.py +260 -0
  95. rucio/core/lifetime_exception.py +377 -0
  96. rucio/core/lock.py +577 -0
  97. rucio/core/message.py +288 -0
  98. rucio/core/meta_conventions.py +203 -0
  99. rucio/core/monitor.py +448 -0
  100. rucio/core/naming_convention.py +195 -0
  101. rucio/core/nongrid_trace.py +136 -0
  102. rucio/core/oidc.py +1463 -0
  103. rucio/core/permission/__init__.py +161 -0
  104. rucio/core/permission/generic.py +1124 -0
  105. rucio/core/permission/generic_multi_vo.py +1144 -0
  106. rucio/core/quarantined_replica.py +224 -0
  107. rucio/core/replica.py +4483 -0
  108. rucio/core/replica_sorter.py +362 -0
  109. rucio/core/request.py +3091 -0
  110. rucio/core/rse.py +2079 -0
  111. rucio/core/rse_counter.py +185 -0
  112. rucio/core/rse_expression_parser.py +459 -0
  113. rucio/core/rse_selector.py +304 -0
  114. rucio/core/rule.py +4484 -0
  115. rucio/core/rule_grouping.py +1620 -0
  116. rucio/core/scope.py +181 -0
  117. rucio/core/subscription.py +362 -0
  118. rucio/core/topology.py +490 -0
  119. rucio/core/trace.py +375 -0
  120. rucio/core/transfer.py +1531 -0
  121. rucio/core/vo.py +169 -0
  122. rucio/core/volatile_replica.py +151 -0
  123. rucio/daemons/__init__.py +13 -0
  124. rucio/daemons/abacus/__init__.py +13 -0
  125. rucio/daemons/abacus/account.py +116 -0
  126. rucio/daemons/abacus/collection_replica.py +124 -0
  127. rucio/daemons/abacus/rse.py +117 -0
  128. rucio/daemons/atropos/__init__.py +13 -0
  129. rucio/daemons/atropos/atropos.py +242 -0
  130. rucio/daemons/auditor/__init__.py +289 -0
  131. rucio/daemons/auditor/hdfs.py +97 -0
  132. rucio/daemons/auditor/srmdumps.py +355 -0
  133. rucio/daemons/automatix/__init__.py +13 -0
  134. rucio/daemons/automatix/automatix.py +304 -0
  135. rucio/daemons/badreplicas/__init__.py +13 -0
  136. rucio/daemons/badreplicas/minos.py +322 -0
  137. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  138. rucio/daemons/badreplicas/necromancer.py +196 -0
  139. rucio/daemons/bb8/__init__.py +13 -0
  140. rucio/daemons/bb8/bb8.py +353 -0
  141. rucio/daemons/bb8/common.py +759 -0
  142. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  143. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  144. rucio/daemons/cache/__init__.py +13 -0
  145. rucio/daemons/cache/consumer.py +133 -0
  146. rucio/daemons/common.py +405 -0
  147. rucio/daemons/conveyor/__init__.py +13 -0
  148. rucio/daemons/conveyor/common.py +562 -0
  149. rucio/daemons/conveyor/finisher.py +529 -0
  150. rucio/daemons/conveyor/poller.py +394 -0
  151. rucio/daemons/conveyor/preparer.py +205 -0
  152. rucio/daemons/conveyor/receiver.py +179 -0
  153. rucio/daemons/conveyor/stager.py +133 -0
  154. rucio/daemons/conveyor/submitter.py +403 -0
  155. rucio/daemons/conveyor/throttler.py +532 -0
  156. rucio/daemons/follower/__init__.py +13 -0
  157. rucio/daemons/follower/follower.py +101 -0
  158. rucio/daemons/hermes/__init__.py +13 -0
  159. rucio/daemons/hermes/hermes.py +534 -0
  160. rucio/daemons/judge/__init__.py +13 -0
  161. rucio/daemons/judge/cleaner.py +159 -0
  162. rucio/daemons/judge/evaluator.py +185 -0
  163. rucio/daemons/judge/injector.py +162 -0
  164. rucio/daemons/judge/repairer.py +154 -0
  165. rucio/daemons/oauthmanager/__init__.py +13 -0
  166. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  167. rucio/daemons/reaper/__init__.py +13 -0
  168. rucio/daemons/reaper/dark_reaper.py +282 -0
  169. rucio/daemons/reaper/reaper.py +739 -0
  170. rucio/daemons/replicarecoverer/__init__.py +13 -0
  171. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  172. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  173. rucio/daemons/rsedecommissioner/config.py +81 -0
  174. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  175. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  176. rucio/daemons/rsedecommissioner/profiles/generic.py +452 -0
  177. rucio/daemons/rsedecommissioner/profiles/types.py +93 -0
  178. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  179. rucio/daemons/storage/__init__.py +13 -0
  180. rucio/daemons/storage/consistency/__init__.py +13 -0
  181. rucio/daemons/storage/consistency/actions.py +848 -0
  182. rucio/daemons/tracer/__init__.py +13 -0
  183. rucio/daemons/tracer/kronos.py +511 -0
  184. rucio/daemons/transmogrifier/__init__.py +13 -0
  185. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  186. rucio/daemons/undertaker/__init__.py +13 -0
  187. rucio/daemons/undertaker/undertaker.py +137 -0
  188. rucio/db/__init__.py +13 -0
  189. rucio/db/sqla/__init__.py +52 -0
  190. rucio/db/sqla/constants.py +206 -0
  191. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  192. rucio/db/sqla/migrate_repo/env.py +110 -0
  193. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  194. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  195. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  196. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  197. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  198. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  199. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  200. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  201. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  202. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  203. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  204. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  205. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  206. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  207. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  208. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  209. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  210. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  211. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  212. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  213. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  214. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  215. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  216. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  217. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  218. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  219. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  220. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  221. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  222. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  223. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  224. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  225. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  226. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  227. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  228. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  229. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  230. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  231. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  232. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  233. rucio/db/sqla/migrate_repo/versions/30d5206e9cad_increase_oauthrequest_redirect_msg_.py +37 -0
  234. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  235. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  236. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  237. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  238. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  239. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  240. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  241. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  242. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  243. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  244. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  245. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  246. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  247. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  248. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  249. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  251. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  252. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  253. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  254. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  255. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  256. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  257. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  258. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  259. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  260. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  261. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  262. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  263. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  264. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  265. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  266. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  267. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  268. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  269. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  270. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  271. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  272. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  273. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  274. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  275. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  276. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  278. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  279. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  280. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  281. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  282. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  283. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  284. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  285. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  286. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  287. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  288. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  289. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  290. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  291. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  292. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  293. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  294. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  295. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  296. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  297. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  298. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  299. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  300. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  301. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  302. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  303. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  304. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  305. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  306. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  307. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  308. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  309. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  310. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  311. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  312. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  313. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  314. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  316. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  317. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  318. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  319. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  320. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  321. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  322. rucio/db/sqla/models.py +1743 -0
  323. rucio/db/sqla/sautils.py +55 -0
  324. rucio/db/sqla/session.py +529 -0
  325. rucio/db/sqla/types.py +206 -0
  326. rucio/db/sqla/util.py +543 -0
  327. rucio/gateway/__init__.py +13 -0
  328. rucio/gateway/account.py +345 -0
  329. rucio/gateway/account_limit.py +363 -0
  330. rucio/gateway/authentication.py +381 -0
  331. rucio/gateway/config.py +227 -0
  332. rucio/gateway/credential.py +70 -0
  333. rucio/gateway/did.py +987 -0
  334. rucio/gateway/dirac.py +83 -0
  335. rucio/gateway/exporter.py +60 -0
  336. rucio/gateway/heartbeat.py +76 -0
  337. rucio/gateway/identity.py +189 -0
  338. rucio/gateway/importer.py +46 -0
  339. rucio/gateway/lifetime_exception.py +121 -0
  340. rucio/gateway/lock.py +153 -0
  341. rucio/gateway/meta_conventions.py +98 -0
  342. rucio/gateway/permission.py +74 -0
  343. rucio/gateway/quarantined_replica.py +79 -0
  344. rucio/gateway/replica.py +538 -0
  345. rucio/gateway/request.py +330 -0
  346. rucio/gateway/rse.py +632 -0
  347. rucio/gateway/rule.py +437 -0
  348. rucio/gateway/scope.py +100 -0
  349. rucio/gateway/subscription.py +280 -0
  350. rucio/gateway/vo.py +126 -0
  351. rucio/rse/__init__.py +96 -0
  352. rucio/rse/protocols/__init__.py +13 -0
  353. rucio/rse/protocols/bittorrent.py +194 -0
  354. rucio/rse/protocols/cache.py +111 -0
  355. rucio/rse/protocols/dummy.py +100 -0
  356. rucio/rse/protocols/gfal.py +708 -0
  357. rucio/rse/protocols/globus.py +243 -0
  358. rucio/rse/protocols/http_cache.py +82 -0
  359. rucio/rse/protocols/mock.py +123 -0
  360. rucio/rse/protocols/ngarc.py +209 -0
  361. rucio/rse/protocols/posix.py +250 -0
  362. rucio/rse/protocols/protocol.py +361 -0
  363. rucio/rse/protocols/rclone.py +365 -0
  364. rucio/rse/protocols/rfio.py +145 -0
  365. rucio/rse/protocols/srm.py +338 -0
  366. rucio/rse/protocols/ssh.py +414 -0
  367. rucio/rse/protocols/storm.py +195 -0
  368. rucio/rse/protocols/webdav.py +594 -0
  369. rucio/rse/protocols/xrootd.py +302 -0
  370. rucio/rse/rsemanager.py +881 -0
  371. rucio/rse/translation.py +260 -0
  372. rucio/tests/__init__.py +13 -0
  373. rucio/tests/common.py +280 -0
  374. rucio/tests/common_server.py +149 -0
  375. rucio/transfertool/__init__.py +13 -0
  376. rucio/transfertool/bittorrent.py +200 -0
  377. rucio/transfertool/bittorrent_driver.py +50 -0
  378. rucio/transfertool/bittorrent_driver_qbittorrent.py +134 -0
  379. rucio/transfertool/fts3.py +1600 -0
  380. rucio/transfertool/fts3_plugins.py +152 -0
  381. rucio/transfertool/globus.py +201 -0
  382. rucio/transfertool/globus_library.py +181 -0
  383. rucio/transfertool/mock.py +89 -0
  384. rucio/transfertool/transfertool.py +221 -0
  385. rucio/vcsversion.py +11 -0
  386. rucio/version.py +45 -0
  387. rucio/web/__init__.py +13 -0
  388. rucio/web/rest/__init__.py +13 -0
  389. rucio/web/rest/flaskapi/__init__.py +13 -0
  390. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  391. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  392. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  393. rucio/web/rest/flaskapi/v1/accounts.py +1103 -0
  394. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  395. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  396. rucio/web/rest/flaskapi/v1/common.py +426 -0
  397. rucio/web/rest/flaskapi/v1/config.py +304 -0
  398. rucio/web/rest/flaskapi/v1/credentials.py +213 -0
  399. rucio/web/rest/flaskapi/v1/dids.py +2340 -0
  400. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  401. rucio/web/rest/flaskapi/v1/export.py +75 -0
  402. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  403. rucio/web/rest/flaskapi/v1/identities.py +285 -0
  404. rucio/web/rest/flaskapi/v1/import.py +132 -0
  405. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  406. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  407. rucio/web/rest/flaskapi/v1/main.py +91 -0
  408. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  409. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  410. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  411. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  412. rucio/web/rest/flaskapi/v1/redirect.py +366 -0
  413. rucio/web/rest/flaskapi/v1/replicas.py +1894 -0
  414. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  415. rucio/web/rest/flaskapi/v1/rses.py +2250 -0
  416. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  417. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  418. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  419. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  420. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  421. rucio/web/rest/flaskapi/v1/traces.py +137 -0
  422. rucio/web/rest/flaskapi/v1/types.py +20 -0
  423. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  424. rucio/web/rest/main.py +18 -0
  425. rucio/web/rest/metrics.py +27 -0
  426. rucio/web/rest/ping.py +27 -0
  427. rucio-37.0.0rc1.data/data/rucio/etc/alembic.ini.template +71 -0
  428. rucio-37.0.0rc1.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  429. rucio-37.0.0rc1.data/data/rucio/etc/globus-config.yml.template +5 -0
  430. rucio-37.0.0rc1.data/data/rucio/etc/ldap.cfg.template +30 -0
  431. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  432. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  433. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  434. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  435. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  436. rucio-37.0.0rc1.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  437. rucio-37.0.0rc1.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  438. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.atlas.client.template +43 -0
  439. rucio-37.0.0rc1.data/data/rucio/etc/rucio.cfg.template +241 -0
  440. rucio-37.0.0rc1.data/data/rucio/etc/rucio_multi_vo.cfg.template +217 -0
  441. rucio-37.0.0rc1.data/data/rucio/requirements.server.txt +297 -0
  442. rucio-37.0.0rc1.data/data/rucio/tools/bootstrap.py +34 -0
  443. rucio-37.0.0rc1.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  444. rucio-37.0.0rc1.data/data/rucio/tools/reset_database.py +40 -0
  445. rucio-37.0.0rc1.data/scripts/rucio +133 -0
  446. rucio-37.0.0rc1.data/scripts/rucio-abacus-account +74 -0
  447. rucio-37.0.0rc1.data/scripts/rucio-abacus-collection-replica +46 -0
  448. rucio-37.0.0rc1.data/scripts/rucio-abacus-rse +78 -0
  449. rucio-37.0.0rc1.data/scripts/rucio-admin +97 -0
  450. rucio-37.0.0rc1.data/scripts/rucio-atropos +60 -0
  451. rucio-37.0.0rc1.data/scripts/rucio-auditor +206 -0
  452. rucio-37.0.0rc1.data/scripts/rucio-automatix +50 -0
  453. rucio-37.0.0rc1.data/scripts/rucio-bb8 +57 -0
  454. rucio-37.0.0rc1.data/scripts/rucio-cache-client +141 -0
  455. rucio-37.0.0rc1.data/scripts/rucio-cache-consumer +42 -0
  456. rucio-37.0.0rc1.data/scripts/rucio-conveyor-finisher +58 -0
  457. rucio-37.0.0rc1.data/scripts/rucio-conveyor-poller +66 -0
  458. rucio-37.0.0rc1.data/scripts/rucio-conveyor-preparer +37 -0
  459. rucio-37.0.0rc1.data/scripts/rucio-conveyor-receiver +44 -0
  460. rucio-37.0.0rc1.data/scripts/rucio-conveyor-stager +76 -0
  461. rucio-37.0.0rc1.data/scripts/rucio-conveyor-submitter +139 -0
  462. rucio-37.0.0rc1.data/scripts/rucio-conveyor-throttler +104 -0
  463. rucio-37.0.0rc1.data/scripts/rucio-dark-reaper +53 -0
  464. rucio-37.0.0rc1.data/scripts/rucio-dumper +160 -0
  465. rucio-37.0.0rc1.data/scripts/rucio-follower +44 -0
  466. rucio-37.0.0rc1.data/scripts/rucio-hermes +54 -0
  467. rucio-37.0.0rc1.data/scripts/rucio-judge-cleaner +89 -0
  468. rucio-37.0.0rc1.data/scripts/rucio-judge-evaluator +137 -0
  469. rucio-37.0.0rc1.data/scripts/rucio-judge-injector +44 -0
  470. rucio-37.0.0rc1.data/scripts/rucio-judge-repairer +44 -0
  471. rucio-37.0.0rc1.data/scripts/rucio-kronos +44 -0
  472. rucio-37.0.0rc1.data/scripts/rucio-minos +53 -0
  473. rucio-37.0.0rc1.data/scripts/rucio-minos-temporary-expiration +50 -0
  474. rucio-37.0.0rc1.data/scripts/rucio-necromancer +120 -0
  475. rucio-37.0.0rc1.data/scripts/rucio-oauth-manager +63 -0
  476. rucio-37.0.0rc1.data/scripts/rucio-reaper +83 -0
  477. rucio-37.0.0rc1.data/scripts/rucio-replica-recoverer +248 -0
  478. rucio-37.0.0rc1.data/scripts/rucio-rse-decommissioner +66 -0
  479. rucio-37.0.0rc1.data/scripts/rucio-storage-consistency-actions +74 -0
  480. rucio-37.0.0rc1.data/scripts/rucio-transmogrifier +77 -0
  481. rucio-37.0.0rc1.data/scripts/rucio-undertaker +76 -0
  482. rucio-37.0.0rc1.dist-info/METADATA +92 -0
  483. rucio-37.0.0rc1.dist-info/RECORD +487 -0
  484. rucio-37.0.0rc1.dist-info/WHEEL +5 -0
  485. rucio-37.0.0rc1.dist-info/licenses/AUTHORS.rst +100 -0
  486. rucio-37.0.0rc1.dist-info/licenses/LICENSE +201 -0
  487. rucio-37.0.0rc1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,217 @@
1
+ # Copyright 2020-2022 CERN
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ # Authors:
16
+ # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
17
+ # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
18
+ # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
19
+ # - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
20
+ # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
21
+
22
+ [common]
23
+ logdir = /var/log/rucio
24
+ loglevel = DEBUG
25
+ mailtemplatedir=/opt/rucio/etc/mail_templates
26
+ multi_vo = True
27
+
28
+ [client]
29
+ rucio_host = https://rucio-server-prod.cern.ch:443
30
+ auth_host = https://rucio-auth-prod.cern.ch:443
31
+ auth_type = userpass
32
+ #auth_type = gss
33
+ #auth_type = x509_proxy
34
+ #auth_type = ssh
35
+ username = ddmlab
36
+ password = secret
37
+ ca_cert = /opt/rucio/etc/web/ca.crt
38
+ #client_cert = /opt/rucio/etc/web/client.crt
39
+ #client_key = /opt/rucio/etc/web/client.key
40
+ #client_x509_proxy = $X509_USER_PROXY
41
+ #ssh_private_key = $HOME/.ssh/id_rsa
42
+ account = root
43
+ vo = tst
44
+ request_retries = 3
45
+ protocol_stat_retries = 6
46
+
47
+ [upload]
48
+ #transfer_timeout = 3600
49
+
50
+ [download]
51
+ #transfer_timeout = 3600
52
+
53
+ [core]
54
+ geoip_licence_key = LICENCEKEYGOESHERE # Get a free licence key at https://www.maxmind.com/en/geolite2/signup
55
+ default_mail_from = spamspamspam@cern.ch
56
+
57
+ [database]
58
+ default = sqlite:////tmp/rucio.db
59
+ #default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
60
+ #default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
61
+ #schema=atlas_rucio # only for cern oracle
62
+ #default = mysql+pymysql://rucio:rucio@localhost/rucio
63
+ #default = postgresql+psycopg://rucio:rucio@localhost/rucio
64
+ pool_recycle=3600
65
+ echo=0
66
+ pool_reset_on_return=rollback
67
+
68
+ [bootstrap]
69
+ # Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest()
70
+ userpass_identity = ddmlab
71
+ userpass_pwd = 2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583
72
+ userpass_email = ph-adp-ddm-lab@cern.ch
73
+
74
+ # Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt
75
+ x509_identity = emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH
76
+ x509_email = ph-adp-ddm-lab@cern.ch
77
+
78
+ # Default DDMLAB cern account
79
+ gss_identity = ddmlab@CERN.CH
80
+ gss_email = ph-adp-ddm-lab@cern.ch
81
+
82
+ # Default DDMLAB ssh key
83
+ ssh_identity = ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydnVy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab
84
+ ssh_email = ph-adp-ddm-lab@cern.ch
85
+
86
+ # Default DDMLAB SAML Name ID
87
+ saml_id = ddmlab
88
+ saml_email = ph-adp-ddm-lab@cern.ch
89
+
90
+ [monitor]
91
+ carbon_server = rucio-graphite-int.cern.ch
92
+ carbon_port = 8125
93
+ user_scope = your_username
94
+ enable_metrics = False
95
+ metrics_port = 8080
96
+
97
+ [conveyor]
98
+ scheme = srm,gsiftp,root,http,https
99
+ transfertool = fts3
100
+ ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
101
+ cacert = /opt/rucio/etc/web/ca.crt
102
+ usercert = /opt/rucio/tools/x509up
103
+
104
+ [messaging-fts3]
105
+ port = 61123
106
+ ssl_key_file = /home/mario/.ssh/hostkey.pem
107
+ ssl_cert_file = /home/mario/.ssh/hostcert.pem
108
+ destination = /topic/transfer.fts_monitoring_queue_state
109
+ brokers = dashb-test-mb.cern.ch
110
+ voname = atlas
111
+
112
+ [messaging-hermes]
113
+ username = ____
114
+ password = ____
115
+ port = 61023
116
+ nonssl_port = 61013
117
+ use_ssl = True
118
+ ssl_key_file = /etc/grid-security/hostkey.pem
119
+ ssl_cert_file = /etc/grid-security/hostcert.pem
120
+ destination = /topic/rucio.events
121
+ brokers = localhost
122
+ voname = atlas
123
+ email_from = Rucio <spamspamspam@cern.ch>
124
+ email_test = spamspamspam@cern.ch
125
+
126
+ [transmogrifier]
127
+ maxdids = 100000
128
+
129
+ [accounts]
130
+ # These are accounts that can write into scopes owned by another account
131
+ special_accounts = panda, tier0
132
+
133
+ [trace]
134
+ tracedir = /var/log/rucio/trace
135
+ brokers=localhost
136
+ port=61013
137
+ username = _________
138
+ password = _________
139
+ topic = /topic/rucio.tracer
140
+
141
+ [tracer-kronos]
142
+ brokers=localhost
143
+ port=61013
144
+ ssl_key_file = /etc/grid-security/hostkey.pem
145
+ ssl_cert_file = /etc/grid-security/hostcert.pem
146
+ queue = /queue/Consumer.kronos.rucio.tracer
147
+ prefetch_size = 10
148
+ chunksize = 10
149
+ subscription_id = rucio-tracer-listener
150
+ use_ssl = False
151
+ reconnect_attempts = 100
152
+ excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch
153
+ username = _________
154
+ password = _________
155
+ dataset_wait = 60
156
+
157
+ [injector]
158
+ file = /opt/rucio/tools/test.file.1000
159
+ bytes = 1000
160
+ md5 = fd21ce524a9e45060fd3f62c4ef6a386
161
+ adler32 = 52590737
162
+ #file = /opt/rucio/tools/test.file.1G
163
+ #bytes = 1000000000
164
+ #md5sum = 1a3031f43cc0c0cf4f8341920a9faf52
165
+ #adler32 = 5ff2d417
166
+
167
+ [alembic]
168
+ cfg = /opt/rucio/etc/alembic.ini
169
+
170
+ [messaging-cache]
171
+ port = 61023
172
+ ssl_key_file = /etc/grid-security/hostkey.pem
173
+ ssl_cert_file = /etc/grid-security/hostcert.pem
174
+ destination = /topic/rucio.fax
175
+ brokers = localhost
176
+ voname = atlas
177
+ account = cache_mb
178
+
179
+ [test]
180
+ cacert = /opt/rucio/etc/web/ca.crt
181
+ #cacert = /etc/pki/tls/certs/CERN-bundle.pem
182
+ usercert = /opt/rucio/etc/web/usercert.pem
183
+ #usercert = /home/mario/.ssh/usercert_with_key.pem
184
+
185
+ [nagios]
186
+ proxy = /opt/rucio/etc/ddmadmin.proxy.nagios
187
+ rfcproxy = /opt/rucio/etc/ddmusr01.rfc.proxy
188
+ fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446, https://fts3.cern.ch:8446, https://lcgfts3.gridpp.rl.ac.uk:8446, https://fts3-test.gridpp.rl.ac.uk:8446, https://fts.usatlas.bnl.gov:8446
189
+
190
+ [auditor]
191
+ cache = /opt/rucio/auditor-cache
192
+ results = /opt/rucio/auditor-results
193
+
194
+ [policy]
195
+ #package = GenericMultiVORucioPolicy #doesn't exist
196
+ permission = generic_multi_vo
197
+ schema = generic_multi_vo
198
+ lfn2pfn_algorithm_default = hash
199
+ support = hn-atlas-dist-analysis-help@cern.ch
200
+ support_rucio = https://github.com/rucio/rucio/issues/
201
+
202
+ [webui]
203
+ usercert = /opt/rucio/etc/usercert_with_key.pem
204
+
205
+ [credentials]
206
+ gcs = /opt/rucio/etc/google-cloud-storage-test.json
207
+ signature_lifetime = 3600
208
+
209
+ [saml]
210
+ config_path = /opt/rucio/lib/rucio/web/ui/common/saml/
211
+
212
+ [oidc]
213
+ idpsecrets = /opt/rucio/etc/idpsecrets.json
214
+ admin_issuer = wlcg
215
+
216
+ [api]
217
+ endpoints = accountlimits, accounts, config, credentials, dids, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, traces, vos
@@ -0,0 +1,297 @@
1
+ #
2
+ # This file is autogenerated by pip-compile with Python 3.9
3
+ # by the following command:
4
+ #
5
+ # pip-compile --strip-extras requirements.server.in
6
+ #
7
+ aiohappyeyeballs==2.5.0
8
+ # via aiohttp
9
+ aiohttp==3.11.13
10
+ # via geoip2
11
+ aiosignal==1.3.2
12
+ # via aiohttp
13
+ alembic==1.14.1
14
+ # via -r requirements.server.in
15
+ annotated-types==0.7.0
16
+ # via pydantic
17
+ argcomplete==3.5.3
18
+ # via -r requirements.server.in
19
+ async-timeout==5.0.1
20
+ # via
21
+ # aiohttp
22
+ # redis
23
+ attrs==25.1.0
24
+ # via
25
+ # aiohttp
26
+ # jsonschema
27
+ # referencing
28
+ bcrypt==4.3.0
29
+ # via paramiko
30
+ blinker==1.9.0
31
+ # via flask
32
+ boto3==1.37.5
33
+ # via -r requirements.server.in
34
+ botocore==1.37.7
35
+ # via
36
+ # boto3
37
+ # s3transfer
38
+ cachetools==5.5.2
39
+ # via google-auth
40
+ certifi==2025.1.31
41
+ # via
42
+ # elastic-transport
43
+ # requests
44
+ cffi==1.17.1
45
+ # via
46
+ # cryptography
47
+ # pynacl
48
+ charset-normalizer==3.4.1
49
+ # via requests
50
+ click==8.1.8
51
+ # via flask
52
+ cryptography==44.0.2
53
+ # via
54
+ # globus-sdk
55
+ # oic
56
+ # paramiko
57
+ # pyjwt
58
+ # pyspnego
59
+ # requests-kerberos
60
+ cx-oracle==8.3.0
61
+ # via -r requirements.server.in
62
+ decorator==5.2.1
63
+ # via
64
+ # dogpile-cache
65
+ # gssapi
66
+ defusedxml==0.7.1
67
+ # via oic
68
+ dnspython==2.7.0
69
+ # via pymongo
70
+ docopt==0.6.2
71
+ # via stomp-py
72
+ dogpile-cache==1.2.2
73
+ # via -r requirements.server.in
74
+ elastic-transport==8.17.0
75
+ # via elasticsearch
76
+ elasticsearch==8.15.1
77
+ # via -r requirements.server.in
78
+ flask==3.1.0
79
+ # via -r requirements.server.in
80
+ frozenlist==1.5.0
81
+ # via
82
+ # aiohttp
83
+ # aiosignal
84
+ future==1.0.0
85
+ # via pyjwkest
86
+ geoip2==5.0.1
87
+ # via -r requirements.server.in
88
+ globus-sdk==3.41.0
89
+ # via -r requirements.server.in
90
+ google-auth==2.38.0
91
+ # via -r requirements.server.in
92
+ gssapi==1.9.0
93
+ # via pyspnego
94
+ idna==3.10
95
+ # via
96
+ # requests
97
+ # yarl
98
+ importlib-metadata==8.6.1
99
+ # via flask
100
+ isodate==0.7.2
101
+ # via python3-saml
102
+ itsdangerous==2.2.0
103
+ # via flask
104
+ jinja2==3.1.6
105
+ # via flask
106
+ jmespath==1.0.1
107
+ # via
108
+ # boto3
109
+ # botocore
110
+ jsonschema==4.23.0
111
+ # via -r requirements.server.in
112
+ jsonschema-specifications==2024.10.1
113
+ # via jsonschema
114
+ kerberos==1.3.1
115
+ # via -r requirements.server.in
116
+ krb5==0.7.1
117
+ # via pyspnego
118
+ libtorrent==2.0.11
119
+ # via -r requirements.server.in
120
+ lxml==5.3.1
121
+ # via
122
+ # python3-saml
123
+ # xmlsec
124
+ mako==1.3.9
125
+ # via
126
+ # alembic
127
+ # oic
128
+ markdown-it-py==3.0.0
129
+ # via rich
130
+ markupsafe==3.0.2
131
+ # via
132
+ # jinja2
133
+ # mako
134
+ # werkzeug
135
+ maxminddb==2.6.3
136
+ # via geoip2
137
+ mdurl==0.1.2
138
+ # via markdown-it-py
139
+ multidict==6.1.0
140
+ # via
141
+ # aiohttp
142
+ # yarl
143
+ oic==1.7.0
144
+ # via -r requirements.server.in
145
+ packaging==24.1
146
+ # via
147
+ # -r requirements.server.in
148
+ # qbittorrent-api
149
+ paramiko==3.5.1
150
+ # via -r requirements.server.in
151
+ pbr==6.1.1
152
+ # via stevedore
153
+ prometheus-client==0.21.1
154
+ # via -r requirements.server.in
155
+ propcache==0.3.0
156
+ # via
157
+ # aiohttp
158
+ # yarl
159
+ psycopg==3.2.3 ; implementation_name == "cpython"
160
+ # via -r requirements.server.in
161
+ psycopg-binary==3.2.3
162
+ # via psycopg
163
+ psycopg-pool==3.2.4
164
+ # via psycopg
165
+ pyasn1==0.6.1
166
+ # via
167
+ # pyasn1-modules
168
+ # rsa
169
+ pyasn1-modules==0.4.1
170
+ # via google-auth
171
+ pycparser==2.22
172
+ # via cffi
173
+ pycryptodomex==3.21.0
174
+ # via
175
+ # oic
176
+ # pyjwkest
177
+ pydantic==2.10.6
178
+ # via pydantic-settings
179
+ pydantic-core==2.27.2
180
+ # via pydantic
181
+ pydantic-settings==2.8.1
182
+ # via oic
183
+ pygments==2.19.1
184
+ # via rich
185
+ pyjwkest==1.4.2
186
+ # via oic
187
+ pyjwt==2.10.1
188
+ # via globus-sdk
189
+ pykerberos==1.2.4
190
+ # via -r requirements.server.in
191
+ pymemcache==4.0.0
192
+ # via -r requirements.server.in
193
+ pymongo==4.11.2
194
+ # via -r requirements.server.in
195
+ pymysql==1.1.1
196
+ # via -r requirements.server.in
197
+ pynacl==1.5.0
198
+ # via paramiko
199
+ pyspnego==0.11.2
200
+ # via requests-kerberos
201
+ python-dateutil==2.9.0.post0
202
+ # via
203
+ # -r requirements.server.in
204
+ # botocore
205
+ python-dotenv==1.0.1
206
+ # via pydantic-settings
207
+ python-magic==0.4.27
208
+ # via -r requirements.server.in
209
+ python-swiftclient==4.7.0
210
+ # via -r requirements.server.in
211
+ python3-saml==1.16.0
212
+ # via -r requirements.server.in
213
+ pyyaml==6.0.2
214
+ # via -r requirements.server.in
215
+ qbittorrent-api==2025.2.0
216
+ # via -r requirements.server.in
217
+ redis==5.2.1
218
+ # via -r requirements.server.in
219
+ referencing==0.36.2
220
+ # via
221
+ # jsonschema
222
+ # jsonschema-specifications
223
+ requests==2.32.3
224
+ # via
225
+ # -r requirements.server.in
226
+ # geoip2
227
+ # globus-sdk
228
+ # oic
229
+ # pyjwkest
230
+ # python-swiftclient
231
+ # qbittorrent-api
232
+ # requests-kerberos
233
+ requests-kerberos==0.15.0
234
+ # via -r requirements.server.in
235
+ rich==13.9.4
236
+ # via -r requirements.server.in
237
+ rpds-py==0.23.1
238
+ # via
239
+ # jsonschema
240
+ # referencing
241
+ rsa==4.9
242
+ # via google-auth
243
+ s3transfer==0.11.4
244
+ # via boto3
245
+ six==1.17.0
246
+ # via
247
+ # pyjwkest
248
+ # python-dateutil
249
+ sqlalchemy==2.0.38
250
+ # via
251
+ # -r requirements.server.in
252
+ # alembic
253
+ statsd==4.0.1
254
+ # via -r requirements.server.in
255
+ stevedore==5.4.1
256
+ # via dogpile-cache
257
+ stomp-py==8.2.0
258
+ # via -r requirements.server.in
259
+ tabulate==0.9.0
260
+ # via -r requirements.server.in
261
+ typing-extensions==4.12.2
262
+ # via
263
+ # alembic
264
+ # dogpile-cache
265
+ # globus-sdk
266
+ # multidict
267
+ # psycopg
268
+ # psycopg-pool
269
+ # pydantic
270
+ # pydantic-core
271
+ # referencing
272
+ # rich
273
+ # sqlalchemy
274
+ urllib3==1.26.19
275
+ # via
276
+ # -r requirements.server.in
277
+ # botocore
278
+ # elastic-transport
279
+ # qbittorrent-api
280
+ # requests
281
+ websocket-client==1.8.0
282
+ # via stomp-py
283
+ werkzeug==3.1.3
284
+ # via
285
+ # -r requirements.server.in
286
+ # flask
287
+ xmlsec==1.3.13
288
+ # via
289
+ # -r requirements.server.in
290
+ # python3-saml
291
+ yarl==1.18.3
292
+ # via aiohttp
293
+ zipp==3.21.0
294
+ # via importlib-metadata
295
+
296
+ # The following packages are considered to be unsafe in a requirements file:
297
+ # setuptools
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env python
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Run this once to set up the database.
17
+ # PYTHONPATH=/opt/rucio/.venv/lib/python2.7/site-packages/rucio python tools/bootstrap.py
18
+ #
19
+ # Verify for default SQLite:
20
+ # for i in `sqlite3 /tmp/rucio.db ".tables"`; do echo $i:; sqlite3 /tmp/rucio.db "select * from $i"; echo; done
21
+
22
+ import os.path
23
+ import sys
24
+
25
+ base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
26
+ sys.path.append(base_path)
27
+ os.chdir(base_path)
28
+
29
+ from rucio.db.sqla.util import build_database, create_base_vo, create_root_account # noqa: E402
30
+
31
+ if __name__ == '__main__':
32
+ build_database()
33
+ create_base_vo()
34
+ create_root_account()
@@ -0,0 +1,144 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import argparse
16
+ import json
17
+ import logging
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ try:
23
+ import yaml
24
+ except ImportError:
25
+ yaml = None
26
+ import configparser
27
+
28
+ # Multi-word sections used in kubernetes are slightly different from what rucio expects.
29
+ # Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention.
30
+ multi_word_sections = {
31
+ 'messaging_fts3': 'messaging-fts3',
32
+ 'messaging_cache': 'messaging-cache',
33
+ 'messaging_hermes': 'messaging-hermes',
34
+ 'messaging_hermes2': 'hermes',
35
+ 'nongrid_trace': 'nongrid-trace',
36
+ 'tracer_kronos': 'tracer-kronos',
37
+ }
38
+
39
+
40
+ def load_flat_config(flat_config):
41
+ """
42
+ takes a dict of the form: {"section_option": "value"}
43
+ and converts to {"section": {"option": "value"}
44
+ """
45
+ config_dict = {}
46
+ for flat_key, config_value in flat_config.items():
47
+ section = option = None
48
+ # Try parsing a multi-word section
49
+ for mw_key in multi_word_sections:
50
+ if flat_key.startswith(mw_key + '_'):
51
+ section = mw_key
52
+ option = flat_key[len(mw_key) + 1:]
53
+
54
+ # It didn't match any known multi-word section, assume it's a single word
55
+ if not section:
56
+ section, option = flat_key.split('_', maxsplit=1)
57
+
58
+ config_dict.setdefault(section, {})[option] = config_value
59
+ return config_dict
60
+
61
+
62
+ def fix_multi_word_sections(config_dict):
63
+ return {multi_word_sections.get(section, section): config_for_section for section, config_for_section in config_dict.items()}
64
+
65
+
66
+ def config_len(config_dict):
67
+ return sum(len(option) for _, option in config_dict.items())
68
+
69
+
70
+ def merge_configs(source_file_paths, dest_file_path, use_env=True, logger=logging.log):
71
+ """
72
+ Merge multiple configuration sources into one rucio.cfg.
73
+ On conflicting values, relies on the default python's ConfigParser behavior: the value from last source wins.
74
+ Sources can be .ini, .yaml, or .json files. Json is supported as a compromise solution for easier integration
75
+ with kubernetes (because both python and helm natively support it).
76
+ If use_env=True, env variables starting with RUCIO_CFG_ are also merged as the last (highest priority) source.
77
+ """
78
+
79
+ parser = configparser.ConfigParser()
80
+ for path in source_file_paths:
81
+ path = Path(path)
82
+
83
+ if not path.exists():
84
+ logger(logging.WARNING, "Skipping {}: path doesn't exist".format(path))
85
+ continue
86
+
87
+ if path.is_dir():
88
+ file_paths = sorted(p for p in path.iterdir() if not p.name.startswith(".") and p.is_file())
89
+ else:
90
+ file_paths = [path]
91
+
92
+ for file_path in file_paths:
93
+ try:
94
+ if file_path.suffix == '.json':
95
+ with open(file_path, 'r') as f:
96
+ file_config = fix_multi_word_sections(json.load(f))
97
+ parser.read_dict(file_config)
98
+ elif yaml and file_path.suffix in ['.yaml', '.yml']:
99
+ with open(file_path, 'r') as f:
100
+ file_config = fix_multi_word_sections(yaml.safe_load(f))
101
+ parser.read_dict(file_config)
102
+ elif path.is_file() or file_path.suffix in ['.ini', '.cfg', '.config']:
103
+ local_parser = configparser.ConfigParser()
104
+ local_parser.read(file_path)
105
+ file_config = {section: {option: value for option, value in section_proxy.items()} for section, section_proxy in local_parser.items()}
106
+ else:
107
+ logger(logging.WARNING, "Skipping file {} due to wrong extension".format(file_path))
108
+ continue
109
+
110
+ parser.read_dict(file_config)
111
+ logger(logging.INFO, "Merged {} configuration values from {}".format(config_len(file_config), file_path))
112
+ except Exception as error:
113
+ logger(logging.WARNING, "Skipping file {} due to error: {}".format(file_path, error))
114
+
115
+ if use_env:
116
+ # env variables use the following format: "RUCIO_CFG_{section.substitute('-','_').upper}_{option.substitute('-', '_').upper}"
117
+ env_config = {}
118
+ for env_key, env_value in os.environ.items():
119
+ rucio_cfg_prefix = 'RUCIO_CFG_'
120
+ if not env_key.startswith(rucio_cfg_prefix):
121
+ continue
122
+ env_key = env_key[len(rucio_cfg_prefix):].lower() # convert "RUCIO_CFG_WHATEVER" to "whatever"
123
+ env_config[env_key] = env_value
124
+
125
+ env_config = fix_multi_word_sections(load_flat_config(env_config))
126
+ parser.read_dict(env_config)
127
+ logger(logging.INFO, "Merged {} configuration values from ENV".format(config_len(env_config)))
128
+
129
+ if dest_file_path:
130
+ logger(logging.INFO, "Writing {}".format(dest_file_path))
131
+ with open(dest_file_path, 'w') as dest_file:
132
+ parser.write(dest_file)
133
+ else:
134
+ parser.write(sys.stdout)
135
+
136
+
137
+ logging.getLogger().setLevel(logging.INFO)
138
+ parser = argparse.ArgumentParser(description="Merge multiple rucio configuration sources into one rucio.cfg")
139
+ parser.add_argument("--use-env", action="store_true", default=False, help='Also source config from RUCIO_CFG_* env variables')
140
+ parser.add_argument('-s', '--source', type=str, nargs='*', help='Source config file paths (in .json, .yaml or .ini format)')
141
+ parser.add_argument('-d', '--destination', default=None, help='Destination file path')
142
+ args = parser.parse_args()
143
+
144
+ merge_configs(args.source or [], args.destination, use_env=args.use_env)