rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,7 +20,7 @@
21
20
  import argparse
22
21
  import json
23
22
  import os
24
- import random
23
+ import secrets
25
24
  import ssl
26
25
  import sys
27
26
 
@@ -80,7 +79,7 @@ def cache_operation(args):
80
79
  conn.set_ssl(key_file=args.ssl_key_file, cert_file=args.ssl_cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
81
80
 
82
81
  conn.connect()
83
- message = {'id': int(random.random() * 1000), 'payload': payload}
82
+ message = {'id': secrets.randbelow(1001), 'payload': payload}
84
83
  conn.send(destination=args.destination, body=json.dumps(message), id='rucio-cache-messaging', ack='auto', headers={'vo': 'atlas'})
85
84
  conn.disconnect()
86
85
 
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,7 +27,7 @@ def get_parser():
28
27
  """
29
28
  Returns the argparse parser.
30
29
  """
31
- parser = argparse.ArgumentParser(description="Conveyor is a group of daemons to manage file transfers. The conveyor-finisher is the resposible to update Rucio internal state after the transfer has finished.")
30
+ parser = argparse.ArgumentParser(description="Conveyor is a group of daemons to manage file transfers. The conveyor-finisher is the responsible to update Rucio internal state after the transfer has finished.")
32
31
  parser.add_argument("--run-once", action="store_true", default=False,
33
32
  help='One iteration only')
34
33
  parser.add_argument("--total-threads", action="store", default=1, type=int,
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -35,7 +34,7 @@ Upload a file and create a replication rule::
35
34
  $ rucio add-rule mock:file 1 MOCK2
36
35
  $ rucio-admin rse add-distance MOCK2 MOCK --distance 1
37
36
 
38
- The rule should replicate the file from RSE MOCK to RSE MOCK2. Therefor a distance between these RSEs is needed.
37
+ The rule should replicate the file from RSE MOCK to RSE MOCK2. Therefore a distance between these RSEs is needed.
39
38
 
40
39
  Check transfer requests for the DID::
41
40
 
@@ -57,7 +56,7 @@ Check again the transfer requests for the DID::
57
56
  session.get_session().query(models.Request).filter_by(scope='mock', name='file').first()
58
57
  # {'request_type': TRANSFER, 'state': SUBMITTED', ...}
59
58
 
60
- A tranfer request got created by executing the transfer. Depending on the transfer submission, the request state can be different. In this example the transfer got submitted successfully.
59
+ A transfer request got created by executing the transfer. Depending on the transfer submission, the request state can be different. In this example the transfer got submitted successfully.
61
60
 
62
61
  When run in multi-VO mode, by default the daemon will run on RSEs from all VOs::
63
62
 
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,11 +17,11 @@ import argparse
18
17
  import datetime
19
18
  import logging
20
19
  import sys
20
+
21
21
  import tabulate
22
22
 
23
23
  import rucio.common.dumper.consistency as consistency
24
24
  import rucio.common.dumper.data_models as data_models
25
-
26
25
  from rucio.common.dumper import error
27
26
 
28
27
  logger = logging.getLogger('rucio-dumper')
@@ -96,9 +95,10 @@ if __name__ == "__main__":
96
95
  record_type = data_models.CompleteDataset
97
96
  elif args.subcommand == 'dump-replicas':
98
97
  record_type = data_models.Replica
99
- else:
100
- assert args.subcommand in consistency.subcommands
98
+ elif args.subcommand in consistency.subcommands:
101
99
  record_type = consistency.Consistency
100
+ else:
101
+ raise ValueError("Record type not set! Cannot dump files.")
102
102
 
103
103
  if 'filter' in args and args.filter:
104
104
  user_filter = data_models.Filter(args.filter, record_type).match
@@ -114,8 +114,8 @@ if __name__ == "__main__":
114
114
  fields = set(record_type.get_fieldnames())
115
115
 
116
116
  if 'group_by' in args and args.group_by:
117
- # FIXME: Better checking
118
- assert args.group_by in fields
117
+ if args.group_by not in fields:
118
+ raise ValueError("Group_by option %s not supported." % args.group_by)
119
119
  fields = [args.group_by]
120
120
  data_iter = data
121
121
  data_dict = {}
@@ -126,11 +126,12 @@ if __name__ == "__main__":
126
126
  data_dict[key].append(record)
127
127
 
128
128
  if 'sum' in args and args.sum:
129
- assert args.sum in set(record_type.get_fieldnames())
129
+ if args.sum not in set(record_type.get_fieldnames()):
130
+ raise ValueError("Sum option %s for record type %s not supported.\nChoice from %s" % (args.sum, args.record_type, set(record_type.get_fieldnames())))
130
131
  fields.append(args.sum)
131
132
  data = []
132
133
  for key, value in data_dict.items():
133
- total = sum((getattr(x, args.sum) for x in value))
134
+ total = sum(getattr(x, args.sum) for x in value)
134
135
  # FIXME: Ugly hack to have some result
135
136
  setattr(value[0], args.sum, total)
136
137
  data.append(value[0])
@@ -139,12 +140,12 @@ if __name__ == "__main__":
139
140
 
140
141
  if args.fields:
141
142
  _show_fields = args.fields.split(',')
142
- if not all((f in fields for f in _show_fields)):
143
+ if not all(f in fields for f in _show_fields):
143
144
  error('Invalid field in --fields argument')
144
145
  fields = _show_fields
145
146
  elif args.hide:
146
147
  _hide_fields = args.hide.split(',')
147
- if not all((f in fields for f in _hide_fields)):
148
+ if not all(f in fields for f in _hide_fields):
148
149
  error('Invalid field in --hide argument')
149
150
  fields = [f for f in fields if f not in _hide_fields]
150
151
 
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -114,11 +113,11 @@ Check the replicas for the DID mock:file::
114
113
  from rucio.db.sqla import session, models
115
114
  session.get_session().query(models.RSEFileAssociation).filter_by(name='file', scope='mock').first()
116
115
  // [{'name': 'file','lock_cnt': 1, 'state': COPYING, 'scope': 'mock', 'rse_id': 'f81f366593754c01b0c340fa5ea0ab90'},
117
- // {'scope': 'mock', 'rse_id': '1330d5daee37474c88ba888101d7b859', 'name': 'file', 'state': AVAIABLE, 'lock_cnt': 1}]
116
+ // {'scope': 'mock', 'rse_id': '1330d5daee37474c88ba888101d7b859', 'name': 'file', 'state': AVAILABLE, 'lock_cnt': 1}]
118
117
 
119
118
  The DID mock:file has now two replicas with one lock each.
120
119
  As the file replica is attached to the dataset and the rule for the dataset specifies another RSE MOCK instead of the upload RSE, it has to be replicated to this RSE.
121
- Therefor a second replica in state COPYING got created on RSE MOCK.
120
+ Therefore a second replica in state COPYING got created on RSE MOCK.
122
121
  ''', formatter_class=argparse.RawDescriptionHelpFormatter)
123
122
  parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only')
124
123
  parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads for this process')
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +12,8 @@
13
12
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
13
  # See the License for the specific language governing permissions and
15
14
  # limitations under the License.
16
-
17
15
  """
18
- Kronos is a daemon the consumer tracer messages and update the replica atime accordingly.
16
+ Kronos is a daemon that consumes tracer messages and updates the replica atime accordingly.
19
17
  """
20
18
 
21
19
  import argparse
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +30,7 @@ def get_parser():
31
30
  parser = argparse.ArgumentParser(description="The Necromancer daemon is responsible for managing bad replicas. If a replica that got declared bad has other replicas, it will try to recover it by requesting a new transfer. If there are no replicas anymore, then the file gets marked as lost.", epilog='''
32
31
  Lost replica:
33
32
  In this example the file gets uploaded and will only have this replica as there are no replication rules. If it gets declared bad, there will be no replica to recover from.
34
- Therefor the replica gets marked as lost.
33
+ Therefore the replica gets marked as lost.
35
34
 
36
35
  Upload a file::
37
36
 
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +14,7 @@
15
14
  # limitations under the License.
16
15
 
17
16
  """
18
- OAuth Manager is a daemon which is reponsible for:
17
+ OAuth Manager is a daemon which is responsible for:
19
18
  - deletion of expired access tokens (in case there is a valid refresh token, expired access tokens will be kept until refresh_token expires as well.)
20
19
  - deletion of expired OAuth session parameters
21
20
  - refreshing access tokens via their refresh tokens.
@@ -33,7 +32,7 @@ def get_parser():
33
32
  """
34
33
  parser = argparse.ArgumentParser(description='''
35
34
 
36
- OAuth Manager is a daemon which is reponsible for:
35
+ OAuth Manager is a daemon which is responsible for:
37
36
  - deletion of expired access tokens (in case there is a valid refresh token,
38
37
  expired access tokens will be kept until refresh_token expires as well.)
39
38
  - deletion of expired OAuth session parameters
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -68,7 +67,7 @@ $ id0=`uuidgen`
68
67
  $ id1=`uuidgen`
69
68
  $ id2=`uuidgen`
70
69
  $ id3=`uuidgen`
71
- $ echo "file available on MOCK_RECOVERY and decalred suspicious on MOCK_SUSPICIOUS (11 times)" > /tmp/file_available_suspicious'_'$id1
70
+ $ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS (11 times)" > /tmp/file_available_suspicious'_'$id1
72
71
  $ echo "file available on MOCK_RECOVERY and declared suspicious on MOCK_SUSPICIOUS (11 times) and 1 time bad/deleted/lost on MOCK_SUSPICIOUS" > /tmp/file_available_suspicious_and_bad'_'$id2
73
72
  $ echo "file declared as unavailable on MOCK_RECOVERY and declared as suspicious 11 times on MOCK_SUSPICIOUS" > /tmp/file_notavailable_suspicious'_'$id3
74
73
 
@@ -184,8 +183,8 @@ Terminal output:
184
183
 
185
184
  2019-02-19 14:39:24,114 709 INFO replica_recoverer[0/0]: ready to query replicas at RSEs like *MOCK*, declared as suspicious in the last 3 days at least 10 times and which are available on other RSEs.
186
185
  2019-02-19 14:39:24,124 709 INFO replica_recoverer[0/0]: suspicious replica query took 0.0101511478424 seconds, total of 1 replicas were found. [{'scope': 'mock', 'cnt': 11L, 'name': 'file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e', 'rse': 'MOCK_SUSPICIOUS'}]
187
- 2019-02-19 14:39:24,125 709 INFO replica_recoverer[0/0]: looking for replica surls.
188
- 2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: found 1/1 surls (took 0.035572052002 seconds) - declaring them as bad replicas now.
186
+ 2019-02-19 14:39:24,125 709 INFO replica_recoverer[0/0]: looking for replica pfns.
187
+ 2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: found 1/1 pfns (took 0.035572052002 seconds) - declaring them as bad replicas now.
189
188
  2019-02-19 14:39:24,160 709 INFO replica_recoverer[0/0]: ready to declare 1 bad replica(s) on MOCK_SUSPICIOUS: ['file://localhost:0/tmp/rucio_rse1/mock/1f/6b/file_available_suspicious_5180be3e-4ebc-4c34-b528-efbfd09f067e'].
190
189
  2019-02-19 14:39:24,188 709 INFO replica_recoverer[0/0]: finished declaring bad replicas on MOCK_SUSPICIOUS.
191
190
  2019-02-19 14:39:24,192 709 INFO replica_recoverer[0/0]: graceful stop done
@@ -228,11 +227,11 @@ Note that attempting the use the ``--vos`` argument when in single-VO mode will
228
227
  $ rucio-replica-recoverer --run-once --vos abc xyz
229
228
  2020-07-28 15:21:33,349 5488 WARNING Ignoring argument vos, this is only applicable in a multi-VO setup.
230
229
  ''', formatter_class=argparse.RawDescriptionHelpFormatter) # NOQA: E501
231
- parser.add_argument("--nattempts", action="store", default=10, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 10.')
232
- parser.add_argument("--younger-than", action="store", default=3, help='Consider all file replicas logged in bad_replicas table since speicified number of younger-than days. Default value is 3.')
230
+ parser.add_argument("--nattempts", action="store", default=5, type=int, help='Minimum count of suspicious file replica appearance in bad_replicas table. Default value is 5.')
231
+ parser.add_argument("--younger-than", action="store", default=5, type=int, help='Consider all file replicas logged in bad_replicas table since specified number of younger-than days. Default value is 5.')
233
232
  parser.add_argument('--vos', nargs='+', type=str, help='Optional list of VOs to consider. Only used in multi-VO mode.')
234
233
  parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only.')
235
- parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
234
+ parser.add_argument("--limit-suspicious-files-on-rse", action="store", default=5, type=int, help='Maximum number of suspicious replicas on an RSE before that RSE is considered problematic and the suspicious replicas on that RSE are declared "TEMPORARY_UNAVAILABLE". Default value is 5.')
236
235
  parser.add_argument('--json-file-name', action="store", default="/opt/rucio/etc/suspicious_replica_recoverer.json", type=str, help='Name of the json file that that contains the policies which will be used by the suspicious replica recoverer.')
237
236
  parser.add_argument('--sleep-time', action="store", default=3600, type=int, help='Concurrency control: Thread sleep time after each chunk of work.')
238
237
  parser.add_argument('--active-mode', action="store_true", default=False, help='If NOT specified, the daemon will run without taking any actions on any files. In either case, the log file will be produced normally.')
@@ -0,0 +1,66 @@
1
+ #!python
2
+ # Copyright European Organization for Nuclear Research (CERN) since 2012
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """
17
+ RSE-Decommissioner is a daemon that clears RSEs scheduled for decommissioning.
18
+ The actual operations (e.g. just delete all replicas, move the replicas to
19
+ specific RSEs, etc.) that are performed on the RSEs depend on the
20
+ "decommissioning profile", which must be specified for each RSE through RSE
21
+ attributes.
22
+ """
23
+
24
+ import argparse
25
+ import signal
26
+
27
+ from rucio.daemons.rsedecommissioner.rse_decommissioner import run, stop
28
+
29
+
30
+ def get_parser() -> argparse.ArgumentParser:
31
+ """Construct and return the argparse parser."""
32
+
33
+ parser = argparse.ArgumentParser(
34
+ description=__doc__,
35
+ epilog="""\
36
+ To mark an RSE for decommissioning
37
+ ----------------------------------
38
+
39
+ Clearing with the generic "delete all" profile:
40
+
41
+ $ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
42
+ --value profile=generic_delete
43
+
44
+ Clearing with the generic "move all" profile:
45
+
46
+ $ rucio-admin rse set-attribute --rse MOCK_RSE --key decommission \
47
+ --value profile=generic_move,destination=ANOTHER_RSE""",
48
+ formatter_class=argparse.RawDescriptionHelpFormatter
49
+ )
50
+
51
+ parser.add_argument('--run-once', dest='run_once', action='store_true',
52
+ help='One iteration only.')
53
+ parser.add_argument('--sleep-time', dest='sleep_time', action='store',
54
+ default=86400, type=int,
55
+ help='Concurrency control: thread sleep time after each'
56
+ ' chunk of work')
57
+ return parser
58
+
59
+
60
+ if __name__ == '__main__':
61
+ signal.signal(signal.SIGTERM, stop)
62
+ args = get_parser().parse_args()
63
+ try:
64
+ run(once=args.run_once, sleep_time=args.sleep_time)
65
+ except KeyboardInterrupt:
66
+ stop()
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1,5 +1,4 @@
1
1
  #!python
2
- # -*- coding: utf-8 -*-
3
2
  # Copyright European Organization for Nuclear Research (CERN) since 2012
4
3
  #
5
4
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,7 +27,7 @@ def get_parser():
28
27
  """
29
28
  Returns the argparse parser.
30
29
  """
31
- parser = argparse.ArgumentParser(description="The Undertaker deamon is responsible for managing expired DIDs. It deletes DIDs, but not replicas by checking if there are DIDs where the 'expired_at' date property is older than the current timestamp.", epilog='''
30
+ parser = argparse.ArgumentParser(description="The Undertaker daemon is responsible for managing expired DIDs. It deletes DIDs, but not replicas by checking if there are DIDs where the 'expired_at' date property is older than the current timestamp.", epilog='''
32
31
  Create a DID that is already expired by setting its lifetime to -1::
33
32
 
34
33
  $ python
@@ -0,0 +1,72 @@
1
+ Metadata-Version: 2.4
2
+ Name: rucio
3
+ Version: 35.8.0
4
+ Summary: Rucio Package
5
+ Home-page: https://rucio.cern.ch/
6
+ Author: Rucio
7
+ Author-email: rucio-dev@cern.ch
8
+ License: Apache License, Version 2.0
9
+ Classifier: Development Status :: 5 - Production/Stable
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Intended Audience :: Information Technology
12
+ Classifier: Intended Audience :: System Administrators
13
+ Classifier: Operating System :: POSIX :: Linux
14
+ Classifier: Natural Language :: English
15
+ Classifier: Programming Language :: Python
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Environment :: No Input/Output (Daemon)
20
+ Requires-Python: >=3.9, <4
21
+ License-File: LICENSE
22
+ License-File: AUTHORS.rst
23
+ Requires-Dist: alembic==1.13.2
24
+ Requires-Dist: argcomplete==3.4.0
25
+ Requires-Dist: boto3==1.34.142
26
+ Requires-Dist: dogpile-cache==1.2.2
27
+ Requires-Dist: flask==3.0.3
28
+ Requires-Dist: geoip2==4.8.0
29
+ Requires-Dist: google-auth==2.32.0
30
+ Requires-Dist: jsonschema==4.23.0
31
+ Requires-Dist: oic==1.7.0
32
+ Requires-Dist: paramiko==3.4.1
33
+ Requires-Dist: prometheus-client==0.20.0
34
+ Requires-Dist: pymemcache==4.0.0
35
+ Requires-Dist: python-dateutil==2.9.0.post0
36
+ Requires-Dist: python-magic==0.4.27
37
+ Requires-Dist: redis==5.0.7
38
+ Requires-Dist: requests==2.32.3
39
+ Requires-Dist: sqlalchemy==2.0.31
40
+ Requires-Dist: statsd==4.0.1
41
+ Requires-Dist: stomp-py==8.1.2
42
+ Requires-Dist: tabulate==0.9.0
43
+ Requires-Dist: urllib3==1.26.19
44
+ Provides-Extra: oracle
45
+ Requires-Dist: cx-oracle==8.3.0; extra == "oracle"
46
+ Provides-Extra: mongo
47
+ Requires-Dist: pymongo==4.8.0; extra == "mongo"
48
+ Provides-Extra: postgresql
49
+ Requires-Dist: psycopg2-binary==2.9.9; extra == "postgresql"
50
+ Provides-Extra: mysql
51
+ Requires-Dist: pymysql==1.1.1; extra == "mysql"
52
+ Provides-Extra: kerberos
53
+ Requires-Dist: kerberos==1.3.1; extra == "kerberos"
54
+ Requires-Dist: pykerberos==1.2.4; extra == "kerberos"
55
+ Requires-Dist: requests-kerberos==0.15.0; extra == "kerberos"
56
+ Provides-Extra: globus
57
+ Requires-Dist: globus-sdk==3.41.0; extra == "globus"
58
+ Requires-Dist: pyyaml==6.0.1; extra == "globus"
59
+ Provides-Extra: saml
60
+ Requires-Dist: python3-saml==1.16.0; extra == "saml"
61
+ Provides-Extra: dev
62
+ Requires-Dist: pyyaml==6.0.1; extra == "dev"
63
+ Dynamic: author
64
+ Dynamic: author-email
65
+ Dynamic: classifier
66
+ Dynamic: home-page
67
+ Dynamic: license
68
+ Dynamic: license-file
69
+ Dynamic: provides-extra
70
+ Dynamic: requires-dist
71
+ Dynamic: requires-python
72
+ Dynamic: summary