rucio 35.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (493) hide show
  1. rucio/__init__.py +17 -0
  2. rucio/alembicrevision.py +15 -0
  3. rucio/client/__init__.py +15 -0
  4. rucio/client/accountclient.py +433 -0
  5. rucio/client/accountlimitclient.py +183 -0
  6. rucio/client/baseclient.py +974 -0
  7. rucio/client/client.py +76 -0
  8. rucio/client/configclient.py +126 -0
  9. rucio/client/credentialclient.py +59 -0
  10. rucio/client/didclient.py +866 -0
  11. rucio/client/diracclient.py +56 -0
  12. rucio/client/downloadclient.py +1785 -0
  13. rucio/client/exportclient.py +44 -0
  14. rucio/client/fileclient.py +50 -0
  15. rucio/client/importclient.py +42 -0
  16. rucio/client/lifetimeclient.py +90 -0
  17. rucio/client/lockclient.py +109 -0
  18. rucio/client/metaconventionsclient.py +140 -0
  19. rucio/client/pingclient.py +44 -0
  20. rucio/client/replicaclient.py +454 -0
  21. rucio/client/requestclient.py +125 -0
  22. rucio/client/rseclient.py +746 -0
  23. rucio/client/ruleclient.py +294 -0
  24. rucio/client/scopeclient.py +90 -0
  25. rucio/client/subscriptionclient.py +173 -0
  26. rucio/client/touchclient.py +82 -0
  27. rucio/client/uploadclient.py +955 -0
  28. rucio/common/__init__.py +13 -0
  29. rucio/common/cache.py +74 -0
  30. rucio/common/config.py +801 -0
  31. rucio/common/constants.py +159 -0
  32. rucio/common/constraints.py +17 -0
  33. rucio/common/didtype.py +189 -0
  34. rucio/common/dumper/__init__.py +335 -0
  35. rucio/common/dumper/consistency.py +452 -0
  36. rucio/common/dumper/data_models.py +318 -0
  37. rucio/common/dumper/path_parsing.py +64 -0
  38. rucio/common/exception.py +1151 -0
  39. rucio/common/extra.py +36 -0
  40. rucio/common/logging.py +420 -0
  41. rucio/common/pcache.py +1408 -0
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +84 -0
  44. rucio/common/schema/__init__.py +150 -0
  45. rucio/common/schema/atlas.py +413 -0
  46. rucio/common/schema/belleii.py +408 -0
  47. rucio/common/schema/domatpc.py +401 -0
  48. rucio/common/schema/escape.py +426 -0
  49. rucio/common/schema/generic.py +433 -0
  50. rucio/common/schema/generic_multi_vo.py +412 -0
  51. rucio/common/schema/icecube.py +406 -0
  52. rucio/common/stomp_utils.py +159 -0
  53. rucio/common/stopwatch.py +55 -0
  54. rucio/common/test_rucio_server.py +148 -0
  55. rucio/common/types.py +403 -0
  56. rucio/common/utils.py +2238 -0
  57. rucio/core/__init__.py +13 -0
  58. rucio/core/account.py +496 -0
  59. rucio/core/account_counter.py +236 -0
  60. rucio/core/account_limit.py +423 -0
  61. rucio/core/authentication.py +620 -0
  62. rucio/core/config.py +456 -0
  63. rucio/core/credential.py +225 -0
  64. rucio/core/did.py +3000 -0
  65. rucio/core/did_meta_plugins/__init__.py +252 -0
  66. rucio/core/did_meta_plugins/did_column_meta.py +331 -0
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +165 -0
  68. rucio/core/did_meta_plugins/filter_engine.py +613 -0
  69. rucio/core/did_meta_plugins/json_meta.py +240 -0
  70. rucio/core/did_meta_plugins/mongo_meta.py +216 -0
  71. rucio/core/did_meta_plugins/postgres_meta.py +316 -0
  72. rucio/core/dirac.py +237 -0
  73. rucio/core/distance.py +187 -0
  74. rucio/core/exporter.py +59 -0
  75. rucio/core/heartbeat.py +363 -0
  76. rucio/core/identity.py +300 -0
  77. rucio/core/importer.py +259 -0
  78. rucio/core/lifetime_exception.py +377 -0
  79. rucio/core/lock.py +576 -0
  80. rucio/core/message.py +282 -0
  81. rucio/core/meta_conventions.py +203 -0
  82. rucio/core/monitor.py +447 -0
  83. rucio/core/naming_convention.py +195 -0
  84. rucio/core/nongrid_trace.py +136 -0
  85. rucio/core/oidc.py +1461 -0
  86. rucio/core/permission/__init__.py +119 -0
  87. rucio/core/permission/atlas.py +1348 -0
  88. rucio/core/permission/belleii.py +1077 -0
  89. rucio/core/permission/escape.py +1078 -0
  90. rucio/core/permission/generic.py +1130 -0
  91. rucio/core/permission/generic_multi_vo.py +1150 -0
  92. rucio/core/quarantined_replica.py +223 -0
  93. rucio/core/replica.py +4158 -0
  94. rucio/core/replica_sorter.py +366 -0
  95. rucio/core/request.py +3089 -0
  96. rucio/core/rse.py +1875 -0
  97. rucio/core/rse_counter.py +186 -0
  98. rucio/core/rse_expression_parser.py +459 -0
  99. rucio/core/rse_selector.py +302 -0
  100. rucio/core/rule.py +4483 -0
  101. rucio/core/rule_grouping.py +1618 -0
  102. rucio/core/scope.py +180 -0
  103. rucio/core/subscription.py +364 -0
  104. rucio/core/topology.py +490 -0
  105. rucio/core/trace.py +375 -0
  106. rucio/core/transfer.py +1517 -0
  107. rucio/core/vo.py +169 -0
  108. rucio/core/volatile_replica.py +150 -0
  109. rucio/daemons/__init__.py +13 -0
  110. rucio/daemons/abacus/__init__.py +13 -0
  111. rucio/daemons/abacus/account.py +116 -0
  112. rucio/daemons/abacus/collection_replica.py +124 -0
  113. rucio/daemons/abacus/rse.py +117 -0
  114. rucio/daemons/atropos/__init__.py +13 -0
  115. rucio/daemons/atropos/atropos.py +242 -0
  116. rucio/daemons/auditor/__init__.py +289 -0
  117. rucio/daemons/auditor/hdfs.py +97 -0
  118. rucio/daemons/auditor/srmdumps.py +355 -0
  119. rucio/daemons/automatix/__init__.py +13 -0
  120. rucio/daemons/automatix/automatix.py +293 -0
  121. rucio/daemons/badreplicas/__init__.py +13 -0
  122. rucio/daemons/badreplicas/minos.py +322 -0
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +171 -0
  124. rucio/daemons/badreplicas/necromancer.py +196 -0
  125. rucio/daemons/bb8/__init__.py +13 -0
  126. rucio/daemons/bb8/bb8.py +353 -0
  127. rucio/daemons/bb8/common.py +759 -0
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +153 -0
  129. rucio/daemons/bb8/t2_background_rebalance.py +153 -0
  130. rucio/daemons/c3po/__init__.py +13 -0
  131. rucio/daemons/c3po/algorithms/__init__.py +13 -0
  132. rucio/daemons/c3po/algorithms/simple.py +134 -0
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +128 -0
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +130 -0
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +294 -0
  136. rucio/daemons/c3po/c3po.py +371 -0
  137. rucio/daemons/c3po/collectors/__init__.py +13 -0
  138. rucio/daemons/c3po/collectors/agis.py +108 -0
  139. rucio/daemons/c3po/collectors/free_space.py +81 -0
  140. rucio/daemons/c3po/collectors/jedi_did.py +57 -0
  141. rucio/daemons/c3po/collectors/mock_did.py +51 -0
  142. rucio/daemons/c3po/collectors/network_metrics.py +71 -0
  143. rucio/daemons/c3po/collectors/workload.py +112 -0
  144. rucio/daemons/c3po/utils/__init__.py +13 -0
  145. rucio/daemons/c3po/utils/dataset_cache.py +50 -0
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +56 -0
  147. rucio/daemons/c3po/utils/expiring_list.py +62 -0
  148. rucio/daemons/c3po/utils/popularity.py +85 -0
  149. rucio/daemons/c3po/utils/timeseries.py +89 -0
  150. rucio/daemons/cache/__init__.py +13 -0
  151. rucio/daemons/cache/consumer.py +197 -0
  152. rucio/daemons/common.py +415 -0
  153. rucio/daemons/conveyor/__init__.py +13 -0
  154. rucio/daemons/conveyor/common.py +562 -0
  155. rucio/daemons/conveyor/finisher.py +529 -0
  156. rucio/daemons/conveyor/poller.py +404 -0
  157. rucio/daemons/conveyor/preparer.py +205 -0
  158. rucio/daemons/conveyor/receiver.py +249 -0
  159. rucio/daemons/conveyor/stager.py +132 -0
  160. rucio/daemons/conveyor/submitter.py +403 -0
  161. rucio/daemons/conveyor/throttler.py +532 -0
  162. rucio/daemons/follower/__init__.py +13 -0
  163. rucio/daemons/follower/follower.py +101 -0
  164. rucio/daemons/hermes/__init__.py +13 -0
  165. rucio/daemons/hermes/hermes.py +774 -0
  166. rucio/daemons/judge/__init__.py +13 -0
  167. rucio/daemons/judge/cleaner.py +159 -0
  168. rucio/daemons/judge/evaluator.py +185 -0
  169. rucio/daemons/judge/injector.py +162 -0
  170. rucio/daemons/judge/repairer.py +154 -0
  171. rucio/daemons/oauthmanager/__init__.py +13 -0
  172. rucio/daemons/oauthmanager/oauthmanager.py +198 -0
  173. rucio/daemons/reaper/__init__.py +13 -0
  174. rucio/daemons/reaper/dark_reaper.py +278 -0
  175. rucio/daemons/reaper/reaper.py +743 -0
  176. rucio/daemons/replicarecoverer/__init__.py +13 -0
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +626 -0
  178. rucio/daemons/rsedecommissioner/__init__.py +13 -0
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +13 -0
  186. rucio/daemons/storage/consistency/__init__.py +13 -0
  187. rucio/daemons/storage/consistency/actions.py +846 -0
  188. rucio/daemons/tracer/__init__.py +13 -0
  189. rucio/daemons/tracer/kronos.py +536 -0
  190. rucio/daemons/transmogrifier/__init__.py +13 -0
  191. rucio/daemons/transmogrifier/transmogrifier.py +762 -0
  192. rucio/daemons/undertaker/__init__.py +13 -0
  193. rucio/daemons/undertaker/undertaker.py +137 -0
  194. rucio/db/__init__.py +13 -0
  195. rucio/db/sqla/__init__.py +52 -0
  196. rucio/db/sqla/constants.py +201 -0
  197. rucio/db/sqla/migrate_repo/__init__.py +13 -0
  198. rucio/db/sqla/migrate_repo/env.py +110 -0
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +70 -0
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +47 -0
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +59 -0
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +43 -0
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +91 -0
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +76 -0
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +43 -0
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +50 -0
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +68 -0
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +40 -0
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +45 -0
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +60 -0
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +40 -0
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +140 -0
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +73 -0
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +74 -0
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +43 -0
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +50 -0
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +134 -0
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +64 -0
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +39 -0
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +64 -0
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +51 -0
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +41 -0
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +43 -0
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +44 -0
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +53 -0
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +38 -0
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +47 -0
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +45 -0
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +45 -0
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +57 -0
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +45 -0
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +69 -0
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +43 -0
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +42 -0
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +47 -0
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +46 -0
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +40 -0
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +67 -0
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +44 -0
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +77 -0
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +60 -0
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +72 -0
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +42 -0
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +65 -0
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +133 -0
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +55 -0
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +76 -0
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +60 -0
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +44 -0
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +43 -0
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +64 -0
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +40 -0
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +43 -0
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +44 -0
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +78 -0
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +41 -0
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +59 -0
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +44 -0
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +43 -0
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +49 -0
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +40 -0
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +63 -0
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +43 -0
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +45 -0
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +43 -0
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +43 -0
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +45 -0
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +47 -0
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +58 -0
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +45 -0
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +106 -0
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +55 -0
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +50 -0
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +47 -0
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +43 -0
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +41 -0
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +91 -0
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +72 -0
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +49 -0
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +43 -0
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +43 -0
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +53 -0
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +45 -0
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +68 -0
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +45 -0
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +94 -0
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +54 -0
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +72 -0
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +76 -0
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +47 -0
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +121 -0
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +59 -0
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +52 -0
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +54 -0
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +64 -0
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +49 -0
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +43 -0
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +91 -0
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +40 -0
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +43 -0
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +143 -0
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +76 -0
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +50 -0
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +72 -0
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +55 -0
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +43 -0
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +65 -0
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +47 -0
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +146 -0
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +104 -0
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +44 -0
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +43 -0
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +103 -0
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +49 -0
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +104 -0
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +29 -0
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +74 -0
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +47 -0
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +43 -0
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +37 -0
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +43 -0
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +43 -0
  327. rucio/db/sqla/models.py +1740 -0
  328. rucio/db/sqla/sautils.py +55 -0
  329. rucio/db/sqla/session.py +498 -0
  330. rucio/db/sqla/types.py +206 -0
  331. rucio/db/sqla/util.py +543 -0
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/gateway/account.py +339 -0
  334. rucio/gateway/account_limit.py +286 -0
  335. rucio/gateway/authentication.py +375 -0
  336. rucio/gateway/config.py +217 -0
  337. rucio/gateway/credential.py +71 -0
  338. rucio/gateway/did.py +970 -0
  339. rucio/gateway/dirac.py +81 -0
  340. rucio/gateway/exporter.py +59 -0
  341. rucio/gateway/heartbeat.py +74 -0
  342. rucio/gateway/identity.py +204 -0
  343. rucio/gateway/importer.py +45 -0
  344. rucio/gateway/lifetime_exception.py +120 -0
  345. rucio/gateway/lock.py +153 -0
  346. rucio/gateway/meta_conventions.py +87 -0
  347. rucio/gateway/permission.py +71 -0
  348. rucio/gateway/quarantined_replica.py +78 -0
  349. rucio/gateway/replica.py +529 -0
  350. rucio/gateway/request.py +321 -0
  351. rucio/gateway/rse.py +600 -0
  352. rucio/gateway/rule.py +417 -0
  353. rucio/gateway/scope.py +99 -0
  354. rucio/gateway/subscription.py +277 -0
  355. rucio/gateway/vo.py +122 -0
  356. rucio/rse/__init__.py +96 -0
  357. rucio/rse/protocols/__init__.py +13 -0
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +122 -0
  360. rucio/rse/protocols/dummy.py +111 -0
  361. rucio/rse/protocols/gfal.py +703 -0
  362. rucio/rse/protocols/globus.py +243 -0
  363. rucio/rse/protocols/gsiftp.py +92 -0
  364. rucio/rse/protocols/http_cache.py +82 -0
  365. rucio/rse/protocols/mock.py +123 -0
  366. rucio/rse/protocols/ngarc.py +209 -0
  367. rucio/rse/protocols/posix.py +250 -0
  368. rucio/rse/protocols/protocol.py +594 -0
  369. rucio/rse/protocols/rclone.py +364 -0
  370. rucio/rse/protocols/rfio.py +136 -0
  371. rucio/rse/protocols/srm.py +338 -0
  372. rucio/rse/protocols/ssh.py +413 -0
  373. rucio/rse/protocols/storm.py +206 -0
  374. rucio/rse/protocols/webdav.py +550 -0
  375. rucio/rse/protocols/xrootd.py +301 -0
  376. rucio/rse/rsemanager.py +764 -0
  377. rucio/tests/__init__.py +13 -0
  378. rucio/tests/common.py +270 -0
  379. rucio/tests/common_server.py +132 -0
  380. rucio/transfertool/__init__.py +13 -0
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +1596 -0
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +201 -0
  387. rucio/transfertool/globus_library.py +181 -0
  388. rucio/transfertool/mock.py +90 -0
  389. rucio/transfertool/transfertool.py +221 -0
  390. rucio/vcsversion.py +11 -0
  391. rucio/version.py +38 -0
  392. rucio/web/__init__.py +13 -0
  393. rucio/web/rest/__init__.py +13 -0
  394. rucio/web/rest/flaskapi/__init__.py +13 -0
  395. rucio/web/rest/flaskapi/authenticated_bp.py +27 -0
  396. rucio/web/rest/flaskapi/v1/__init__.py +13 -0
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +236 -0
  398. rucio/web/rest/flaskapi/v1/accounts.py +1089 -0
  399. rucio/web/rest/flaskapi/v1/archives.py +102 -0
  400. rucio/web/rest/flaskapi/v1/auth.py +1644 -0
  401. rucio/web/rest/flaskapi/v1/common.py +426 -0
  402. rucio/web/rest/flaskapi/v1/config.py +304 -0
  403. rucio/web/rest/flaskapi/v1/credentials.py +212 -0
  404. rucio/web/rest/flaskapi/v1/dids.py +2334 -0
  405. rucio/web/rest/flaskapi/v1/dirac.py +116 -0
  406. rucio/web/rest/flaskapi/v1/export.py +75 -0
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +127 -0
  408. rucio/web/rest/flaskapi/v1/identities.py +261 -0
  409. rucio/web/rest/flaskapi/v1/import.py +132 -0
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +312 -0
  411. rucio/web/rest/flaskapi/v1/locks.py +358 -0
  412. rucio/web/rest/flaskapi/v1/main.py +91 -0
  413. rucio/web/rest/flaskapi/v1/meta_conventions.py +241 -0
  414. rucio/web/rest/flaskapi/v1/metrics.py +36 -0
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +97 -0
  416. rucio/web/rest/flaskapi/v1/ping.py +88 -0
  417. rucio/web/rest/flaskapi/v1/redirect.py +365 -0
  418. rucio/web/rest/flaskapi/v1/replicas.py +1890 -0
  419. rucio/web/rest/flaskapi/v1/requests.py +998 -0
  420. rucio/web/rest/flaskapi/v1/rses.py +2239 -0
  421. rucio/web/rest/flaskapi/v1/rules.py +854 -0
  422. rucio/web/rest/flaskapi/v1/scopes.py +159 -0
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +650 -0
  424. rucio/web/rest/flaskapi/v1/templates/auth_crash.html +80 -0
  425. rucio/web/rest/flaskapi/v1/templates/auth_granted.html +82 -0
  426. rucio/web/rest/flaskapi/v1/traces.py +100 -0
  427. rucio/web/rest/flaskapi/v1/types.py +20 -0
  428. rucio/web/rest/flaskapi/v1/vos.py +278 -0
  429. rucio/web/rest/main.py +18 -0
  430. rucio/web/rest/metrics.py +27 -0
  431. rucio/web/rest/ping.py +27 -0
  432. rucio-35.7.0.data/data/rucio/etc/alembic.ini.template +71 -0
  433. rucio-35.7.0.data/data/rucio/etc/alembic_offline.ini.template +74 -0
  434. rucio-35.7.0.data/data/rucio/etc/globus-config.yml.template +5 -0
  435. rucio-35.7.0.data/data/rucio/etc/ldap.cfg.template +30 -0
  436. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approval_request.tmpl +38 -0
  437. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +4 -0
  438. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_approved_user.tmpl +17 -0
  439. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +6 -0
  440. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_denied_user.tmpl +17 -0
  441. rucio-35.7.0.data/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +19 -0
  442. rucio-35.7.0.data/data/rucio/etc/rse-accounts.cfg.template +25 -0
  443. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.atlas.client.template +42 -0
  444. rucio-35.7.0.data/data/rucio/etc/rucio.cfg.template +257 -0
  445. rucio-35.7.0.data/data/rucio/etc/rucio_multi_vo.cfg.template +234 -0
  446. rucio-35.7.0.data/data/rucio/requirements.server.txt +268 -0
  447. rucio-35.7.0.data/data/rucio/tools/bootstrap.py +34 -0
  448. rucio-35.7.0.data/data/rucio/tools/merge_rucio_configs.py +144 -0
  449. rucio-35.7.0.data/data/rucio/tools/reset_database.py +40 -0
  450. rucio-35.7.0.data/scripts/rucio +2542 -0
  451. rucio-35.7.0.data/scripts/rucio-abacus-account +74 -0
  452. rucio-35.7.0.data/scripts/rucio-abacus-collection-replica +46 -0
  453. rucio-35.7.0.data/scripts/rucio-abacus-rse +78 -0
  454. rucio-35.7.0.data/scripts/rucio-admin +2447 -0
  455. rucio-35.7.0.data/scripts/rucio-atropos +60 -0
  456. rucio-35.7.0.data/scripts/rucio-auditor +205 -0
  457. rucio-35.7.0.data/scripts/rucio-automatix +50 -0
  458. rucio-35.7.0.data/scripts/rucio-bb8 +57 -0
  459. rucio-35.7.0.data/scripts/rucio-c3po +85 -0
  460. rucio-35.7.0.data/scripts/rucio-cache-client +134 -0
  461. rucio-35.7.0.data/scripts/rucio-cache-consumer +42 -0
  462. rucio-35.7.0.data/scripts/rucio-conveyor-finisher +58 -0
  463. rucio-35.7.0.data/scripts/rucio-conveyor-poller +66 -0
  464. rucio-35.7.0.data/scripts/rucio-conveyor-preparer +37 -0
  465. rucio-35.7.0.data/scripts/rucio-conveyor-receiver +43 -0
  466. rucio-35.7.0.data/scripts/rucio-conveyor-stager +76 -0
  467. rucio-35.7.0.data/scripts/rucio-conveyor-submitter +139 -0
  468. rucio-35.7.0.data/scripts/rucio-conveyor-throttler +104 -0
  469. rucio-35.7.0.data/scripts/rucio-dark-reaper +53 -0
  470. rucio-35.7.0.data/scripts/rucio-dumper +160 -0
  471. rucio-35.7.0.data/scripts/rucio-follower +44 -0
  472. rucio-35.7.0.data/scripts/rucio-hermes +54 -0
  473. rucio-35.7.0.data/scripts/rucio-judge-cleaner +89 -0
  474. rucio-35.7.0.data/scripts/rucio-judge-evaluator +137 -0
  475. rucio-35.7.0.data/scripts/rucio-judge-injector +44 -0
  476. rucio-35.7.0.data/scripts/rucio-judge-repairer +44 -0
  477. rucio-35.7.0.data/scripts/rucio-kronos +43 -0
  478. rucio-35.7.0.data/scripts/rucio-minos +53 -0
  479. rucio-35.7.0.data/scripts/rucio-minos-temporary-expiration +50 -0
  480. rucio-35.7.0.data/scripts/rucio-necromancer +120 -0
  481. rucio-35.7.0.data/scripts/rucio-oauth-manager +63 -0
  482. rucio-35.7.0.data/scripts/rucio-reaper +83 -0
  483. rucio-35.7.0.data/scripts/rucio-replica-recoverer +248 -0
  484. rucio-35.7.0.data/scripts/rucio-rse-decommissioner +66 -0
  485. rucio-35.7.0.data/scripts/rucio-storage-consistency-actions +74 -0
  486. rucio-35.7.0.data/scripts/rucio-transmogrifier +77 -0
  487. rucio-35.7.0.data/scripts/rucio-undertaker +76 -0
  488. rucio-35.7.0.dist-info/METADATA +72 -0
  489. rucio-35.7.0.dist-info/RECORD +493 -0
  490. rucio-35.7.0.dist-info/WHEEL +5 -0
  491. rucio-35.7.0.dist-info/licenses/AUTHORS.rst +97 -0
  492. rucio-35.7.0.dist-info/licenses/LICENSE +201 -0
  493. rucio-35.7.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,234 @@
1
+ # Copyright 2020-2022 CERN
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ # Authors:
16
+ # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
17
+ # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
18
+ # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
19
+ # - Radu Carpa <radu.carpa@cern.ch>, 2021-2022
20
+ # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
21
+
22
+ [common]
23
+ logdir = /var/log/rucio
24
+ loglevel = DEBUG
25
+ mailtemplatedir=/opt/rucio/etc/mail_templates
26
+ multi_vo = True
27
+
28
+ [client]
29
+ rucio_host = https://rucio-server-prod.cern.ch:443
30
+ auth_host = https://rucio-auth-prod.cern.ch:443
31
+ auth_type = userpass
32
+ #auth_type = gss
33
+ #auth_type = x509_proxy
34
+ #auth_type = ssh
35
+ username = ddmlab
36
+ password = secret
37
+ ca_cert = /opt/rucio/etc/web/ca.crt
38
+ #client_cert = /opt/rucio/etc/web/client.crt
39
+ #client_key = /opt/rucio/etc/web/client.key
40
+ #client_x509_proxy = $X509_USER_PROXY
41
+ #ssh_private_key = $HOME/.ssh/id_rsa
42
+ account = root
43
+ vo = tst
44
+ request_retries = 3
45
+ protocol_stat_retries = 6
46
+
47
+ [upload]
48
+ #transfer_timeout = 3600
49
+
50
+ [download]
51
+ #transfer_timeout = 3600
52
+
53
+ [core]
54
+ geoip_licence_key = LICENCEKEYGOESHERE # Get a free licence key at https://www.maxmind.com/en/geolite2/signup
55
+ default_mail_from = spamspamspam@cern.ch
56
+
57
+ [database]
58
+ default = sqlite:////tmp/rucio.db
59
+ #default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=______))(ADDRESS=(PROTOCOL=TCP)(HOST=_________)(PORT=_____))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=__________)))
60
+ #default = oracle://_____________:___________@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=127.0.0.1)(PORT=______))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=_____________)))
61
+ #schema=atlas_rucio # only for cern oracle
62
+ #default = mysql+pymysql://rucio:rucio@localhost/rucio
63
+ #default = postgresql://rucio:rucio@localhost/rucio
64
+ pool_recycle=3600
65
+ echo=0
66
+ pool_reset_on_return=rollback
67
+
68
+ [bootstrap]
69
+ # Hardcoded salt = 0, String = secret, Python: hashlib.sha256("0secret").hexdigest()
70
+ userpass_identity = ddmlab
71
+ userpass_pwd = 2ccee6f6dd1bc2269cddd7cd5e47578e98e430539807c36df23fab7dd13e7583
72
+ userpass_email = ph-adp-ddm-lab@cern.ch
73
+
74
+ # Default DDMLAB client certificate from /opt/rucio/etc/web/client.crt
75
+ x509_identity = emailAddress=ph-adp-ddm-lab@cern.ch,CN=DDMLAB Client Certificate,OU=PH-ADP-CO,O=CERN,ST=Geneva,C=CH
76
+ x509_email = ph-adp-ddm-lab@cern.ch
77
+
78
+ # Default DDMLAB cern account
79
+ gss_identity = ddmlab@CERN.CH
80
+ gss_email = ph-adp-ddm-lab@cern.ch
81
+
82
+ # Default DDMLAB ssh key
83
+ ssh_identity = ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq5LySllrQFpPL614sulXQ7wnIr1aGhGtl8b+HCB/0FhMSMTHwSjX78UbfqEorZV16rXrWPgUpvcbp2hqctw6eCbxwqcgu3uGWaeS5A0iWRw7oXUh6ydnVy89zGzX1FJFFDZ+AgiZ3ytp55tg1bjqqhK1OSC0pJxdNe878TRVVo5MLI0S/rZY2UovCSGFaQG2iLj14wz/YqI7NFMUuJFR4e6xmNsOP7fCZ4bGMsmnhR0GmY0dWYTupNiP5WdYXAfKExlnvFLTlDI5Mgh4Z11NraQ8pv4YE1woolYpqOc/IMMBBXFniTT4tC7cgikxWb9ZmFe+r4t6yCDpX4IL8L5GOQ== ddmlab
84
+ ssh_email = ph-adp-ddm-lab@cern.ch
85
+
86
+ # Default DDMLAB SAML Name ID
87
+ saml_id = ddmlab
88
+ saml_email = ph-adp-ddm-lab@cern.ch
89
+
90
+ [monitor]
91
+ carbon_server = rucio-graphite-int.cern.ch
92
+ carbon_port = 8125
93
+ user_scope = your_username
94
+ enable_metrics = False
95
+ metrics_port = 8080
96
+
97
+ [conveyor]
98
+ scheme = srm,gsiftp,root,http,https
99
+ transfertool = fts3
100
+ ftshosts = https://fts3-pilot.cern.ch:8446, https://fts3-pilot.cern.ch:8446
101
+ cacert = /opt/rucio/etc/web/ca.crt
102
+ usercert = /opt/rucio/tools/x509up
103
+
104
+ [messaging-fts3]
105
+ port = 61123
106
+ ssl_key_file = /home/mario/.ssh/hostkey.pem
107
+ ssl_cert_file = /home/mario/.ssh/hostcert.pem
108
+ destination = /topic/transfer.fts_monitoring_queue_state
109
+ brokers = dashb-test-mb.cern.ch
110
+ voname = atlas
111
+
112
+ [messaging-hermes]
113
+ username = ____
114
+ password = ____
115
+ port = 61023
116
+ nonssl_port = 61013
117
+ use_ssl = True
118
+ ssl_key_file = /etc/grid-security/hostkey.pem
119
+ ssl_cert_file = /etc/grid-security/hostcert.pem
120
+ destination = /topic/rucio.events
121
+ brokers = localhost
122
+ voname = atlas
123
+ email_from = Rucio <spamspamspam@cern.ch>
124
+ email_test = spamspamspam@cern.ch
125
+
126
+ [transmogrifier]
127
+ maxdids = 100000
128
+
129
+ [accounts]
130
+ # These are accounts that can write into scopes owned by another account
131
+ special_accounts = panda, tier0
132
+
133
+ [trace]
134
+ tracedir = /var/log/rucio/trace
135
+ brokers=localhost
136
+ port=61013
137
+ username = _________
138
+ password = _________
139
+ topic = /topic/rucio.tracer
140
+
141
+ [tracer-kronos]
142
+ brokers=localhost
143
+ port=61013
144
+ ssl_key_file = /etc/grid-security/hostkey.pem
145
+ ssl_cert_file = /etc/grid-security/hostcert.pem
146
+ queue = /queue/Consumer.kronos.rucio.tracer
147
+ prefetch_size = 10
148
+ chunksize = 10
149
+ subscription_id = rucio-tracer-listener
150
+ use_ssl = False
151
+ reconnect_attempts = 100
152
+ excluded_usrdns = CN=proxy,CN=Robot: Ganga Robot,CN=722147,CN=gangarbt,OU=Users,OU=Organic Units,DC=cern,DC=ch
153
+ username = _________
154
+ password = _________
155
+ dataset_wait = 60
156
+
157
+ [injector]
158
+ file = /opt/rucio/tools/test.file.1000
159
+ bytes = 1000
160
+ md5 = fd21ce524a9e45060fd3f62c4ef6a386
161
+ adler32 = 52590737
162
+ #file = /opt/rucio/tools/test.file.1G
163
+ #bytes = 1000000000
164
+ #md5sum = 1a3031f43cc0c0cf4f8341920a9faf52
165
+ #adler32 = 5ff2d417
166
+
167
+ [alembic]
168
+ cfg = /opt/rucio/etc/alembic.ini
169
+
170
+ [messaging-cache]
171
+ port = 61023
172
+ ssl_key_file = /etc/grid-security/hostkey.pem
173
+ ssl_cert_file = /etc/grid-security/hostcert.pem
174
+ destination = /topic/rucio.fax
175
+ brokers = localhost
176
+ voname = atlas
177
+ account = cache_mb
178
+
179
+ [test]
180
+ cacert = /opt/rucio/etc/web/ca.crt
181
+ #cacert = /etc/pki/tls/certs/CERN-bundle.pem
182
+ usercert = /opt/rucio/etc/web/usercert.pem
183
+ #usercert = /home/mario/.ssh/usercert_with_key.pem
184
+
185
+ [nagios]
186
+ proxy = /opt/rucio/etc/ddmadmin.proxy.nagios
187
+ rfcproxy = /opt/rucio/etc/ddmusr01.rfc.proxy
188
+ fts_servers = https://fts3-pilot.cern.ch:8446, https://fts3-devel.cern.ch:8446, https://fts3.cern.ch:8446, https://lcgfts3.gridpp.rl.ac.uk:8446, https://fts3-test.gridpp.rl.ac.uk:8446, https://fts.usatlas.bnl.gov:8446
189
+
190
+ [auditor]
191
+ cache = /opt/rucio/auditor-cache
192
+ results = /opt/rucio/auditor-results
193
+
194
+ [c3po]
195
+ placement_algorithm = t2_free_space
196
+ elastic_url = http://aianalytics01.cern.ch:9200
197
+ redis_host = localhost
198
+ redis_port = 6379
199
+
200
+ [c3po-popularity]
201
+ elastic_url = http://rucio-logger-prod-01.cern.ch:9200
202
+
203
+ [c3po-site-mapper]
204
+ panda_url = http://atlas-agis-api.cern.ch/request/pandaqueue/query/list/?json
205
+ ddm_url = http://atlas-agis-api.cern.ch/request/ddmendpoint/query/list/?json
206
+
207
+ [c3po-workload]
208
+ panda_url = http://bigpanda.cern.ch/jobs/?category=analysis&jobstatus=running
209
+ window = 604800
210
+
211
+ [policy]
212
+ #package = GenericMultiVORucioPolicy #doesn't exist
213
+ permission = generic_multi_vo
214
+ schema = generic_multi_vo
215
+ lfn2pfn_algorithm_default = hash
216
+ support = hn-atlas-dist-analysis-help@cern.ch
217
+ support_rucio = https://github.com/rucio/rucio/issues/
218
+
219
+ [webui]
220
+ usercert = /opt/rucio/etc/usercert_with_key.pem
221
+
222
+ [credentials]
223
+ gcs = /opt/rucio/etc/google-cloud-storage-test.json
224
+ signature_lifetime = 3600
225
+
226
+ [saml]
227
+ config_path = /opt/rucio/lib/rucio/web/ui/common/saml/
228
+
229
+ [oidc]
230
+ idpsecrets = /opt/rucio/etc/idpsecrets.json
231
+ admin_issuer = wlcg
232
+
233
+ [api]
234
+ endpoints = accountlimits, accounts, config, credentials, dids, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, traces, vos
@@ -0,0 +1,268 @@
1
+ #
2
+ # This file is autogenerated by pip-compile with Python 3.9
3
+ # by the following command:
4
+ #
5
+ # pip-compile requirements.server.in
6
+ #
7
+ aiohttp==3.9.5
8
+ # via geoip2
9
+ aiosignal==1.3.1
10
+ # via aiohttp
11
+ alembic==1.13.2
12
+ # via -r requirements.server.in
13
+ annotated-types==0.7.0
14
+ # via pydantic
15
+ argcomplete==3.4.0
16
+ # via -r requirements.server.in
17
+ async-timeout==4.0.3
18
+ # via
19
+ # aiohttp
20
+ # redis
21
+ attrs==23.2.0
22
+ # via
23
+ # aiohttp
24
+ # jsonschema
25
+ # referencing
26
+ bcrypt==4.1.3
27
+ # via paramiko
28
+ blinker==1.8.2
29
+ # via flask
30
+ boto3==1.34.142
31
+ # via -r requirements.server.in
32
+ botocore==1.34.143
33
+ # via
34
+ # boto3
35
+ # s3transfer
36
+ cachetools==5.3.3
37
+ # via google-auth
38
+ certifi==2024.7.4
39
+ # via requests
40
+ cffi==1.16.0
41
+ # via
42
+ # cryptography
43
+ # pynacl
44
+ charset-normalizer==3.3.2
45
+ # via requests
46
+ click==8.1.7
47
+ # via flask
48
+ cryptography==43.0.1
49
+ # via
50
+ # globus-sdk
51
+ # oic
52
+ # paramiko
53
+ # pyjwt
54
+ # pyspnego
55
+ # requests-kerberos
56
+ cx-oracle==8.3.0
57
+ # via -r requirements.server.in
58
+ decorator==5.1.1
59
+ # via
60
+ # dogpile-cache
61
+ # gssapi
62
+ defusedxml==0.7.1
63
+ # via oic
64
+ dnspython==2.6.1
65
+ # via pymongo
66
+ docopt==0.6.2
67
+ # via stomp-py
68
+ dogpile-cache==1.2.2
69
+ # via -r requirements.server.in
70
+ flask==3.0.3
71
+ # via -r requirements.server.in
72
+ frozenlist==1.4.1
73
+ # via
74
+ # aiohttp
75
+ # aiosignal
76
+ future==1.0.0
77
+ # via pyjwkest
78
+ geoip2==4.8.0
79
+ # via -r requirements.server.in
80
+ globus-sdk==3.41.0
81
+ # via -r requirements.server.in
82
+ google-auth==2.32.0
83
+ # via -r requirements.server.in
84
+ greenlet==3.0.3
85
+ # via sqlalchemy
86
+ gssapi==1.8.3
87
+ # via pyspnego
88
+ idna==3.7
89
+ # via
90
+ # requests
91
+ # yarl
92
+ importlib-metadata==8.0.0
93
+ # via flask
94
+ isodate==0.6.1
95
+ # via python3-saml
96
+ itsdangerous==2.2.0
97
+ # via flask
98
+ jinja2==3.1.4
99
+ # via flask
100
+ jmespath==1.0.1
101
+ # via
102
+ # boto3
103
+ # botocore
104
+ jsonschema==4.23.0
105
+ # via -r requirements.server.in
106
+ jsonschema-specifications==2023.12.1
107
+ # via jsonschema
108
+ kerberos==1.3.1
109
+ # via -r requirements.server.in
110
+ krb5==0.5.1
111
+ # via pyspnego
112
+ libtorrent==2.0.9
113
+ # via -r requirements.server.in
114
+ lxml==5.2.2
115
+ # via
116
+ # python3-saml
117
+ # xmlsec
118
+ mako==1.3.5
119
+ # via
120
+ # alembic
121
+ # oic
122
+ markupsafe==2.1.5
123
+ # via
124
+ # jinja2
125
+ # mako
126
+ # werkzeug
127
+ maxminddb==2.6.2
128
+ # via geoip2
129
+ multidict==6.0.5
130
+ # via
131
+ # aiohttp
132
+ # yarl
133
+ oic==1.7.0
134
+ # via -r requirements.server.in
135
+ packaging==24.1
136
+ # via qbittorrent-api
137
+ paramiko==3.4.0
138
+ # via -r requirements.server.in
139
+ pbr==6.0.0
140
+ # via stevedore
141
+ prometheus-client==0.20.0
142
+ # via -r requirements.server.in
143
+ psycopg2-binary==2.9.9
144
+ # via -r requirements.server.in
145
+ pyasn1==0.6.0
146
+ # via
147
+ # pyasn1-modules
148
+ # rsa
149
+ pyasn1-modules==0.4.0
150
+ # via google-auth
151
+ pycparser==2.22
152
+ # via cffi
153
+ pycryptodomex==3.20.0
154
+ # via
155
+ # oic
156
+ # pyjwkest
157
+ pydantic==2.8.2
158
+ # via pydantic-settings
159
+ pydantic-core==2.20.1
160
+ # via pydantic
161
+ pydantic-settings==2.3.4
162
+ # via oic
163
+ pyjwkest==1.4.2
164
+ # via oic
165
+ pyjwt[crypto]==2.8.0
166
+ # via globus-sdk
167
+ pykerberos==1.2.4
168
+ # via -r requirements.server.in
169
+ pymemcache==4.0.0
170
+ # via -r requirements.server.in
171
+ pymongo==4.8.0
172
+ # via -r requirements.server.in
173
+ pymysql==1.1.1
174
+ # via -r requirements.server.in
175
+ pynacl==1.5.0
176
+ # via paramiko
177
+ pyspnego[kerberos]==0.11.0
178
+ # via requests-kerberos
179
+ python-dateutil==2.9.0.post0
180
+ # via
181
+ # -r requirements.server.in
182
+ # botocore
183
+ python-dotenv==1.0.1
184
+ # via pydantic-settings
185
+ python-magic==0.4.27
186
+ # via -r requirements.server.in
187
+ python-swiftclient==4.6.0
188
+ # via -r requirements.server.in
189
+ python3-saml==1.16.0
190
+ # via -r requirements.server.in
191
+ pyyaml==6.0.1
192
+ # via -r requirements.server.in
193
+ qbittorrent-api==2023.11.57
194
+ # via -r requirements.server.in
195
+ redis==5.0.7
196
+ # via -r requirements.server.in
197
+ referencing==0.35.1
198
+ # via
199
+ # jsonschema
200
+ # jsonschema-specifications
201
+ requests==2.32.3
202
+ # via
203
+ # -r requirements.server.in
204
+ # geoip2
205
+ # globus-sdk
206
+ # oic
207
+ # pyjwkest
208
+ # python-swiftclient
209
+ # qbittorrent-api
210
+ # requests-kerberos
211
+ requests-kerberos==0.15.0
212
+ # via -r requirements.server.in
213
+ rpds-py==0.19.0
214
+ # via
215
+ # jsonschema
216
+ # referencing
217
+ rsa==4.9
218
+ # via google-auth
219
+ s3transfer==0.10.2
220
+ # via boto3
221
+ six==1.16.0
222
+ # via
223
+ # isodate
224
+ # pyjwkest
225
+ # python-dateutil
226
+ sqlalchemy==2.0.31
227
+ # via
228
+ # -r requirements.server.in
229
+ # alembic
230
+ statsd==4.0.1
231
+ # via -r requirements.server.in
232
+ stevedore==5.2.0
233
+ # via dogpile-cache
234
+ stomp-py==8.1.2
235
+ # via -r requirements.server.in
236
+ tabulate==0.9.0
237
+ # via -r requirements.server.in
238
+ typing-extensions==4.12.2
239
+ # via
240
+ # alembic
241
+ # dogpile-cache
242
+ # globus-sdk
243
+ # pydantic
244
+ # pydantic-core
245
+ # sqlalchemy
246
+ urllib3==1.26.19
247
+ # via
248
+ # -r requirements.server.in
249
+ # botocore
250
+ # qbittorrent-api
251
+ # requests
252
+ websocket-client==1.8.0
253
+ # via stomp-py
254
+ werkzeug==3.0.3
255
+ # via
256
+ # -r requirements.server.in
257
+ # flask
258
+ xmlsec==1.3.13
259
+ # via
260
+ # -r requirements.server.in
261
+ # python3-saml
262
+ yarl==1.9.4
263
+ # via aiohttp
264
+ zipp==3.19.2
265
+ # via importlib-metadata
266
+
267
+ # The following packages are considered to be unsafe in a requirements file:
268
+ # setuptools
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env python
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Run this once to set up the database.
17
+ # PYTHONPATH=/opt/rucio/.venv/lib/python2.7/site-packages/rucio python tools/bootstrap.py
18
+ #
19
+ # Verify for default SQLite:
20
+ # for i in `sqlite3 /tmp/rucio.db ".tables"`; do echo $i:; sqlite3 /tmp/rucio.db "select * from $i"; echo; done
21
+
22
+ import os.path
23
+ import sys
24
+
25
+ base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
26
+ sys.path.append(base_path)
27
+ os.chdir(base_path)
28
+
29
+ from rucio.db.sqla.util import build_database, create_base_vo, create_root_account # noqa: E402
30
+
31
+ if __name__ == '__main__':
32
+ build_database()
33
+ create_base_vo()
34
+ create_root_account()
@@ -0,0 +1,144 @@
1
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import argparse
16
+ import json
17
+ import logging
18
+ import os
19
+ import sys
20
+ from pathlib import Path
21
+
22
+ try:
23
+ import yaml
24
+ except ImportError:
25
+ yaml = None
26
+ import configparser
27
+
28
+ # Multi-word sections used in kubernetes are slightly different from what rucio expects.
29
+ # Usually, it's just a .replace('-', '_'), but not for hermes2, which doesn't follow any convention.
30
+ multi_word_sections = {
31
+ 'messaging_fts3': 'messaging-fts3',
32
+ 'messaging_cache': 'messaging-cache',
33
+ 'messaging_hermes': 'messaging-hermes',
34
+ 'messaging_hermes2': 'hermes',
35
+ 'nongrid_trace': 'nongrid-trace',
36
+ 'tracer_kronos': 'tracer-kronos',
37
+ }
38
+
39
+
40
+ def load_flat_config(flat_config):
41
+ """
42
+ takes a dict of the form: {"section_option": "value"}
43
+ and converts to {"section": {"option": "value"}
44
+ """
45
+ config_dict = {}
46
+ for flat_key, config_value in flat_config.items():
47
+ section = option = None
48
+ # Try parsing a multi-word section
49
+ for mw_key in multi_word_sections:
50
+ if flat_key.startswith(mw_key + '_'):
51
+ section = mw_key
52
+ option = flat_key[len(mw_key) + 1:]
53
+
54
+ # It didn't match any known multi-word section, assume it's a single word
55
+ if not section:
56
+ section, option = flat_key.split('_', maxsplit=1)
57
+
58
+ config_dict.setdefault(section, {})[option] = config_value
59
+ return config_dict
60
+
61
+
62
+ def fix_multi_word_sections(config_dict):
63
+ return {multi_word_sections.get(section, section): config_for_section for section, config_for_section in config_dict.items()}
64
+
65
+
66
+ def config_len(config_dict):
67
+ return sum(len(option) for _, option in config_dict.items())
68
+
69
+
70
+ def merge_configs(source_file_paths, dest_file_path, use_env=True, logger=logging.log):
71
+ """
72
+ Merge multiple configuration sources into one rucio.cfg.
73
+ On conflicting values, relies on the default python's ConfigParser behavior: the value from last source wins.
74
+ Sources can be .ini, .yaml, or .json files. Json is supported as a compromise solution for easier integration
75
+ with kubernetes (because both python and helm natively support it).
76
+ If use_env=True, env variables starting with RUCIO_CFG_ are also merged as the last (highest priority) source.
77
+ """
78
+
79
+ parser = configparser.ConfigParser()
80
+ for path in source_file_paths:
81
+ path = Path(path)
82
+
83
+ if not path.exists():
84
+ logger(logging.WARNING, "Skipping {}: path doesn't exist".format(path))
85
+ continue
86
+
87
+ if path.is_dir():
88
+ file_paths = sorted(p for p in path.iterdir() if not p.name.startswith(".") and p.is_file())
89
+ else:
90
+ file_paths = [path]
91
+
92
+ for file_path in file_paths:
93
+ try:
94
+ if file_path.suffix == '.json':
95
+ with open(file_path, 'r') as f:
96
+ file_config = fix_multi_word_sections(json.load(f))
97
+ parser.read_dict(file_config)
98
+ elif yaml and file_path.suffix in ['.yaml', '.yml']:
99
+ with open(file_path, 'r') as f:
100
+ file_config = fix_multi_word_sections(yaml.safe_load(f))
101
+ parser.read_dict(file_config)
102
+ elif path.is_file() or file_path.suffix in ['.ini', '.cfg', '.config']:
103
+ local_parser = configparser.ConfigParser()
104
+ local_parser.read(file_path)
105
+ file_config = {section: {option: value for option, value in section_proxy.items()} for section, section_proxy in local_parser.items()}
106
+ else:
107
+ logger(logging.WARNING, "Skipping file {} due to wrong extension".format(file_path))
108
+ continue
109
+
110
+ parser.read_dict(file_config)
111
+ logger(logging.INFO, "Merged {} configuration values from {}".format(config_len(file_config), file_path))
112
+ except Exception as error:
113
+ logger(logging.WARNING, "Skipping file {} due to error: {}".format(file_path, error))
114
+
115
+ if use_env:
116
+ # env variables use the following format: "RUCIO_CFG_{section.substitute('-','_').upper}_{option.substitute('-', '_').upper}"
117
+ env_config = {}
118
+ for env_key, env_value in os.environ.items():
119
+ rucio_cfg_prefix = 'RUCIO_CFG_'
120
+ if not env_key.startswith(rucio_cfg_prefix):
121
+ continue
122
+ env_key = env_key[len(rucio_cfg_prefix):].lower() # convert "RUCIO_CFG_WHATEVER" to "whatever"
123
+ env_config[env_key] = env_value
124
+
125
+ env_config = fix_multi_word_sections(load_flat_config(env_config))
126
+ parser.read_dict(env_config)
127
+ logger(logging.INFO, "Merged {} configuration values from ENV".format(config_len(env_config)))
128
+
129
+ if dest_file_path:
130
+ logger(logging.INFO, "Writing {}".format(dest_file_path))
131
+ with open(dest_file_path, 'w') as dest_file:
132
+ parser.write(dest_file)
133
+ else:
134
+ parser.write(sys.stdout)
135
+
136
+
137
+ logging.getLogger().setLevel(logging.INFO)
138
+ parser = argparse.ArgumentParser(description="Merge multiple rucio configuration sources into one rucio.cfg")
139
+ parser.add_argument("--use-env", action="store_true", default=False, help='Also source config from RUCIO_CFG_* env variables')
140
+ parser.add_argument('-s', '--source', type=str, nargs='*', help='Source config file paths (in .json, .yaml or .ini format)')
141
+ parser.add_argument('-d', '--destination', default=None, help='Destination file path')
142
+ args = parser.parse_args()
143
+
144
+ merge_configs(args.source or [], args.destination, use_env=args.use_env)