rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/core/rule.py CHANGED
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,53 +14,74 @@
15
14
 
16
15
  import json
17
16
  import logging
18
- from configparser import NoOptionError
17
+ from collections.abc import Callable, Iterator, Sequence
18
+ from configparser import NoOptionError, NoSectionError
19
19
  from copy import deepcopy
20
20
  from datetime import datetime, timedelta
21
21
  from os import path
22
22
  from re import match
23
23
  from string import Template
24
- from typing import Any, Optional
25
- from typing import TYPE_CHECKING
26
-
27
- from dogpile.cache.api import NO_VALUE
28
- from sqlalchemy import select, update
29
- from sqlalchemy.exc import IntegrityError, StatementError
30
- from sqlalchemy.orm.exc import NoResultFound
24
+ from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union
25
+
26
+ from dogpile.cache.api import NoValue
27
+ from sqlalchemy import delete, desc, select, update
28
+ from sqlalchemy.exc import (
29
+ IntegrityError,
30
+ NoResultFound, # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.exc.NoResultFound.html
31
+ StatementError,
32
+ )
31
33
  from sqlalchemy.sql import func
32
- from sqlalchemy.sql.expression import and_, or_, true, null, tuple_, false
34
+ from sqlalchemy.sql.expression import and_, false, null, or_, true, tuple_
33
35
 
34
36
  import rucio.core.did
35
37
  import rucio.core.lock # import get_replica_locks, get_files_and_replica_locks_of_dataset
36
38
  import rucio.core.replica # import get_and_lock_file_replicas, get_and_lock_file_replicas_for_dataset
37
39
  from rucio.common.cache import make_region_memcached
38
40
  from rucio.common.config import config_get
39
- from rucio.common.exception import (InvalidRSEExpression, InvalidReplicationRule, InsufficientAccountLimit,
40
- DataIdentifierNotFound, RuleNotFound, InputValidationError, RSEOverQuota,
41
- ReplicationRuleCreationTemporaryFailed, InsufficientTargetRSEs, RucioException,
42
- InvalidRuleWeight, StagingAreaRuleRequiresLifetime, DuplicateRule,
43
- InvalidObject, RSEWriteBlocked, RuleReplaceFailed, RequestNotFound,
44
- ManualRuleApprovalBlocked, UnsupportedOperation, UndefinedPolicy, InvalidValueForKey,
45
- InvalidSourceReplicaExpression)
46
- from rucio.common.policy import policy_filter, get_scratchdisk_lifetime
41
+ from rucio.common.constants import RseAttr
42
+ from rucio.common.exception import (
43
+ DataIdentifierNotFound,
44
+ DuplicateRule,
45
+ InputValidationError,
46
+ InsufficientAccountLimit,
47
+ InsufficientTargetRSEs,
48
+ InvalidObject,
49
+ InvalidReplicationRule,
50
+ InvalidRSEExpression,
51
+ InvalidRuleWeight,
52
+ InvalidSourceReplicaExpression,
53
+ InvalidValueForKey,
54
+ ManualRuleApprovalBlocked,
55
+ ReplicationRuleCreationTemporaryFailed,
56
+ RequestNotFound,
57
+ RSEOverQuota,
58
+ RSEWriteBlocked,
59
+ RucioException,
60
+ RuleNotFound,
61
+ RuleReplaceFailed,
62
+ StagingAreaRuleRequiresLifetime,
63
+ UndefinedPolicy,
64
+ UnsupportedOperation,
65
+ )
66
+ from rucio.common.plugins import PolicyPackageAlgorithms
67
+ from rucio.common.policy import get_scratchdisk_lifetime, policy_filter
47
68
  from rucio.common.schema import validate_schema
48
- from rucio.common.types import InternalScope, InternalAccount
49
- from rucio.common.utils import str_to_date, sizefmt, chunks
50
- from rucio.core import account_counter, rse_counter, request as request_core, transfer as transfer_core
51
- from rucio.core.account import get_account
52
- from rucio.core.account import has_account_attribute
69
+ from rucio.common.types import DIDDict, InternalAccount, InternalScope, LoggerFunction, RuleDict
70
+ from rucio.common.utils import chunks, sizefmt, str_to_date
71
+ from rucio.core import account_counter, rse_counter
72
+ from rucio.core import request as request_core
73
+ from rucio.core import transfer as transfer_core
74
+ from rucio.core.account import get_account, has_account_attribute
53
75
  from rucio.core.lifetime_exception import define_eol
54
76
  from rucio.core.message import add_message
55
77
  from rucio.core.monitor import MetricManager
56
- from rucio.core.rse import get_rse_name, list_rse_attributes, get_rse, get_rse_usage
78
+ from rucio.core.rse import get_rse, get_rse_name, get_rse_usage, list_rse_attributes
57
79
  from rucio.core.rse_expression_parser import parse_expression
58
80
  from rucio.core.rse_selector import RSESelector
59
- from rucio.core.rule_grouping import apply_rule_grouping, repair_stuck_locks_and_apply_rule_grouping, create_transfer_dict, apply_rule
60
- from rucio.db.sqla import models, filter_thread_work
61
- from rucio.db.sqla.constants import (LockState, ReplicaState, RuleState, RuleGrouping,
62
- DIDAvailability, DIDReEvaluation, DIDType, BadFilesStatus,
63
- RequestType, RuleNotification, OBSOLETE, RSEType)
64
- from rucio.db.sqla.session import read_session, transactional_session, stream_session
81
+ from rucio.core.rule_grouping import apply_rule, apply_rule_grouping, create_transfer_dict, repair_stuck_locks_and_apply_rule_grouping
82
+ from rucio.db.sqla import filter_thread_work, models
83
+ from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, DIDAvailability, DIDReEvaluation, DIDType, LockState, ReplicaState, RequestType, RSEType, RuleGrouping, RuleNotification, RuleState
84
+ from rucio.db.sqla.session import read_session, stream_session, transactional_session
65
85
 
66
86
  if TYPE_CHECKING:
67
87
  from sqlalchemy.orm import Session
@@ -69,13 +89,106 @@ if TYPE_CHECKING:
69
89
 
70
90
  REGION = make_region_memcached(expiration_time=900)
71
91
  METRICS = MetricManager(module=__name__)
92
+ AutoApproveT = TypeVar('AutoApproveT', bound='AutoApprove')
93
+
94
+
95
+ class AutoApprove(PolicyPackageAlgorithms):
96
+ """
97
+ Handle automatic approval algorithms for replication rules
98
+ """
99
+
100
+ _algorithm_type = 'auto_approve'
101
+
102
+ def __init__(self, rule: models.ReplicationRule, did: models.DataIdentifier, session: 'Session') -> None:
103
+ super().__init__()
104
+ self.rule = rule
105
+ self.did = did
106
+ self.session = session
107
+ self.register("default", self.default)
108
+
109
+ def evaluate(self) -> bool:
110
+ """
111
+ Evaluate the auto-approve algorithm
112
+ """
113
+ return self.get_configured_algorithm()(self.rule, self.did, self.session)
114
+
115
+ @classmethod
116
+ def get_configured_algorithm(cls: type[AutoApproveT]) -> Callable[[models.ReplicationRule, models.DataIdentifier, 'Session'], bool]:
117
+ """
118
+ Get the configured auto-approve algorithm
119
+ """
120
+ try:
121
+ configured_algorithm: str = str(config_get('rules', cls._algorithm_type, default='default'))
122
+ except (NoOptionError, NoSectionError, RuntimeError):
123
+ configured_algorithm = 'default'
124
+
125
+ return super()._get_one_algorithm(cls._algorithm_type, configured_algorithm)
126
+
127
+ @classmethod
128
+ def register(cls: type[AutoApproveT], name: str, fn_auto_approve: Callable[[models.ReplicationRule, models.DataIdentifier, 'Session'], bool]) -> None:
129
+ """
130
+ Register a new auto-approve algorithm
131
+ """
132
+ algorithm_dict = {name: fn_auto_approve}
133
+ super()._register(cls._algorithm_type, algorithm_dict)
134
+
135
+ @staticmethod
136
+ def default(rule: models.ReplicationRule, did: models.DataIdentifier, session: 'Session') -> bool:
137
+ """
138
+ Default auto-approve algorithm
139
+ """
140
+ rse_expression = rule['rse_expression']
141
+ vo = rule['account'].vo
142
+
143
+ rses = parse_expression(rse_expression, filter_={'vo': vo}, session=session)
144
+
145
+ auto_approve = False
146
+ # Block manual approval for multi-rse rules
147
+ if len(rses) > 1:
148
+ raise InvalidReplicationRule('Ask approval is not allowed for rules with multiple RSEs')
149
+ if len(rses) == 1 and not did.is_open and did.bytes is not None and did.length is not None:
150
+ # This rule can be considered for auto-approval:
151
+ rse_attr = list_rse_attributes(rse_id=rses[0]['id'], session=session)
152
+ auto_approve = False
153
+ if RseAttr.AUTO_APPROVE_BYTES in rse_attr and RseAttr.AUTO_APPROVE_FILES in rse_attr:
154
+ if did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)) and did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES)):
155
+ auto_approve = True
156
+ elif did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES, -1)):
157
+ auto_approve = True
158
+ elif did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES, -1)):
159
+ auto_approve = True
160
+
161
+ return auto_approve
72
162
 
73
163
 
74
164
  @transactional_session
75
- def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime, locked, subscription_id,
76
- source_replica_expression=None, activity='User Subscriptions', notify=None, purge_replicas=False,
77
- ignore_availability=False, comment=None, ask_approval=False, asynchronous=False, ignore_account_limit=False,
78
- priority=3, delay_injection=None, split_container=False, meta=None, *, session: "Session", logger=logging.log):
165
+ def add_rule(
166
+ dids: Sequence[DIDDict],
167
+ account: InternalAccount,
168
+ copies: int,
169
+ rse_expression: str,
170
+ grouping: Literal['ALL', 'DATASET', 'NONE'],
171
+ weight: Optional[str],
172
+ lifetime: Optional[int],
173
+ locked: bool,
174
+ subscription_id: Optional[str],
175
+ source_replica_expression: Optional[str] = None,
176
+ activity: str = 'User Subscriptions',
177
+ notify: Optional[Literal['Y', 'N', 'C', 'P']] = None,
178
+ purge_replicas: bool = False,
179
+ ignore_availability: bool = False,
180
+ comment: Optional[str] = None,
181
+ ask_approval: bool = False,
182
+ asynchronous: bool = False,
183
+ ignore_account_limit: bool = False,
184
+ priority: int = 3,
185
+ delay_injection: Optional[int] = None,
186
+ split_container: bool = False,
187
+ meta: Optional[dict[str, Any]] = None,
188
+ *,
189
+ session: "Session",
190
+ logger: LoggerFunction = logging.log
191
+ ) -> list[str]:
79
192
  """
80
193
  Adds a replication rule for every did in dids
81
194
 
@@ -114,7 +227,7 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
114
227
 
115
228
  rule_ids = []
116
229
 
117
- grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
230
+ grouping_value = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
118
231
 
119
232
  with METRICS.timer('add_rule.total'):
120
233
  # 1. Resolve the rse_expression into a list of RSE-ids
@@ -143,14 +256,14 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
143
256
  # Block manual approval if RSE does not allow it
144
257
  if ask_approval:
145
258
  for rse in rses:
146
- if list_rse_attributes(rse_id=rse['id'], session=session).get('block_manual_approval', False):
259
+ if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.BLOCK_MANUAL_APPROVAL, False):
147
260
  raise ManualRuleApprovalBlocked()
148
261
 
149
262
  if source_replica_expression:
150
263
  try:
151
264
  source_rses = parse_expression(source_replica_expression, filter_={'vo': vo}, session=session)
152
- except InvalidRSEExpression:
153
- raise InvalidSourceReplicaExpression
265
+ except InvalidRSEExpression as exc:
266
+ raise InvalidSourceReplicaExpression from exc
154
267
  else:
155
268
  source_rses = []
156
269
 
@@ -160,43 +273,69 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
160
273
 
161
274
  expires_at = datetime.utcnow() + timedelta(seconds=lifetime) if lifetime is not None else None
162
275
 
163
- notify = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(notify, RuleNotification.NO)
276
+ notify_value = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(notify or '', RuleNotification.NO)
164
277
 
165
278
  for elem in dids:
166
279
  # 3. Get the did
167
280
  with METRICS.timer('add_rule.get_did'):
168
281
  try:
169
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == elem['scope'],
170
- models.DataIdentifier.name == elem['name']).one()
171
- except NoResultFound:
172
- raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
282
+ stmt = select(
283
+ models.DataIdentifier
284
+ ).where(
285
+ and_(models.DataIdentifier.scope == elem['scope'],
286
+ models.DataIdentifier.name == elem['name'])
287
+ )
288
+ did = session.execute(stmt).scalar_one()
289
+ except NoResultFound as exc:
290
+ raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
173
291
  except TypeError as error:
174
- raise InvalidObject(error.args)
292
+ raise InvalidObject(error.args) from error # https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/raise-missing-from.html
175
293
 
176
294
  # 3.1 If the did is a constituent, relay the rule to the archive
177
295
  if did.did_type == DIDType.FILE and did.constituent:
178
296
  # Check if a single replica of this DID exists; Do not put rule on file if there are only replicas on TAPE
179
- replica_cnt = session.query(models.RSEFileAssociation).join(models.RSE, models.RSEFileAssociation.rse_id == models.RSE.id)\
180
- .filter(models.RSEFileAssociation.scope == did.scope,
181
- models.RSEFileAssociation.name == did.name,
182
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
183
- models.RSE.rse_type != RSEType.TAPE).count()
297
+ stmt = select(
298
+ func.count()
299
+ ).select_from(
300
+ models.RSEFileAssociation
301
+ ).join(
302
+ models.RSE,
303
+ models.RSEFileAssociation.rse_id == models.RSE.id
304
+ ).where(
305
+ and_(models.RSEFileAssociation.scope == did.scope,
306
+ models.RSEFileAssociation.name == did.name,
307
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
308
+ models.RSE.rse_type != RSEType.TAPE)
309
+ )
310
+ replica_cnt = session.execute(stmt).scalar()
311
+
184
312
  if replica_cnt == 0: # Put the rule on the archive
185
- archive = session.query(models.ConstituentAssociation).join(models.RSEFileAssociation,
186
- and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
187
- models.ConstituentAssociation.name == models.RSEFileAssociation.name))\
188
- .filter(models.ConstituentAssociation.child_scope == did.scope,
189
- models.ConstituentAssociation.child_name == did.name).first()
313
+ stmt = select(
314
+ models.ConstituentAssociation
315
+ ).join(
316
+ models.RSEFileAssociation,
317
+ and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
318
+ models.ConstituentAssociation.name == models.RSEFileAssociation.name)
319
+ ).where(
320
+ and_(models.ConstituentAssociation.child_scope == did.scope,
321
+ models.ConstituentAssociation.child_name == did.name)
322
+ )
323
+ archive = session.execute(stmt).scalars().first()
190
324
  if archive is not None:
191
325
  elem['name'] = archive.name
192
326
  elem['scope'] = archive.scope
193
327
  try:
194
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == elem['scope'],
195
- models.DataIdentifier.name == elem['name']).one()
196
- except NoResultFound:
197
- raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
328
+ stmt = select(
329
+ models.DataIdentifier
330
+ ).where(
331
+ and_(models.DataIdentifier.scope == elem['scope'],
332
+ models.DataIdentifier.name == elem['name'])
333
+ )
334
+ did = session.execute(stmt).scalar_one()
335
+ except NoResultFound as exc:
336
+ raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
198
337
  except TypeError as error:
199
- raise InvalidObject(error.args)
338
+ raise InvalidObject(error.args) from error
200
339
  else: # Put the rule on the constituent directly
201
340
  pass
202
341
 
@@ -219,13 +358,13 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
219
358
  copies=copies,
220
359
  rse_expression=rse_expression,
221
360
  locked=locked,
222
- grouping=grouping,
361
+ grouping=grouping_value,
223
362
  expires_at=expires_at,
224
363
  weight=weight,
225
364
  source_replica_expression=source_replica_expression,
226
365
  activity=activity,
227
366
  subscription_id=subscription_id,
228
- notification=notify,
367
+ notification=notify_value,
229
368
  purge_replicas=purge_replicas,
230
369
  ignore_availability=ignore_availability,
231
370
  comments=comment,
@@ -243,31 +382,20 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
243
382
  or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
244
383
  or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
245
384
  or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]):
246
- raise DuplicateRule(error.args[0])
247
- raise InvalidReplicationRule(error.args[0])
385
+ raise DuplicateRule(error.args[0]) from error
386
+ raise InvalidReplicationRule(error.args[0]) from error
248
387
  rule_ids.append(new_rule.id)
249
388
 
250
389
  if ask_approval:
251
390
  new_rule.state = RuleState.WAITING_APPROVAL
252
- # Block manual approval for multi-rse rules
253
- if len(rses) > 1:
254
- raise InvalidReplicationRule('Ask approval is not allowed for rules with multiple RSEs')
255
- if len(rses) == 1 and not did.is_open and did.bytes is not None and did.length is not None:
256
- # This rule can be considered for auto-approval:
257
- rse_attr = list_rse_attributes(rse_id=rses[0]['id'], session=session)
258
- auto_approve = False
259
- if 'auto_approve_bytes' in rse_attr and 'auto_approve_files' in rse_attr:
260
- if did.bytes < int(rse_attr.get('auto_approve_bytes')) and did.length < int(rse_attr.get('auto_approve_bytes')):
261
- auto_approve = True
262
- elif did.bytes < int(rse_attr.get('auto_approve_bytes', -1)):
263
- auto_approve = True
264
- elif did.length < int(rse_attr.get('auto_approve_files', -1)):
265
- auto_approve = True
266
- if auto_approve:
267
- logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
268
- logger(logging.DEBUG, "Created rule %s for injection", str(new_rule.id))
269
- approve_rule(rule_id=new_rule.id, notify_approvers=False, session=session)
270
- continue
391
+ # Use the new rule as the argument here
392
+ auto_approver = AutoApprove(new_rule, did, session=session)
393
+ if auto_approver.evaluate():
394
+ logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
395
+ logger(logging.DEBUG, "Created rule %s for injection", str(new_rule.id))
396
+ approve_rule(rule_id=new_rule.id, notify_approvers=False, session=session)
397
+ continue
398
+
271
399
  logger(logging.DEBUG, "Created rule %s in waiting for approval", str(new_rule.id))
272
400
  __create_rule_approval_email(rule=new_rule, session=session)
273
401
  continue
@@ -297,17 +425,31 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
297
425
  try:
298
426
  apply_rule(did, new_rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
299
427
  except IntegrityError as error:
300
- raise ReplicationRuleCreationTemporaryFailed(error.args[0])
428
+ raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
301
429
 
302
430
  if new_rule.locks_stuck_cnt > 0:
303
431
  new_rule.state = RuleState.STUCK
304
432
  new_rule.error = 'MissingSourceReplica'
305
433
  if new_rule.grouping != RuleGrouping.NONE:
306
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.STUCK})
434
+ stmt = update(
435
+ models.DatasetLock
436
+ ).where(
437
+ models.DatasetLock.rule_id == new_rule.id
438
+ ).values({
439
+ models.DatasetLock.state: LockState.STUCK
440
+ })
441
+ session.execute(stmt)
307
442
  elif new_rule.locks_replicating_cnt == 0:
308
443
  new_rule.state = RuleState.OK
309
444
  if new_rule.grouping != RuleGrouping.NONE:
310
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.OK})
445
+ stmt = update(
446
+ models.DatasetLock
447
+ ).where(
448
+ models.DatasetLock.rule_id == new_rule.id
449
+ ).values({
450
+ models.DatasetLock.state: LockState.OK
451
+ })
452
+ session.execute(stmt)
311
453
  session.flush()
312
454
  if new_rule.notification == RuleNotification.YES:
313
455
  generate_email_for_rule_ok_notification(rule=new_rule, session=session)
@@ -315,7 +457,14 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
315
457
  else:
316
458
  new_rule.state = RuleState.REPLICATING
317
459
  if new_rule.grouping != RuleGrouping.NONE:
318
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.REPLICATING})
460
+ stmt = update(
461
+ models.DatasetLock
462
+ ).where(
463
+ models.DatasetLock.rule_id == new_rule.id
464
+ ).values({
465
+ models.DatasetLock.state: LockState.REPLICATING
466
+ })
467
+ session.execute(stmt)
319
468
 
320
469
  # Add rule to History
321
470
  insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
@@ -327,13 +476,19 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
327
476
 
328
477
 
329
478
  @transactional_session
330
- def add_rules(dids, rules, *, session: "Session", logger=logging.log):
479
+ def add_rules(
480
+ dids: Sequence[DIDDict],
481
+ rules: Sequence[RuleDict],
482
+ *,
483
+ session: "Session",
484
+ logger: LoggerFunction = logging.log
485
+ ) -> dict[tuple[InternalScope, str], list[str]]:
331
486
  """
332
487
  Adds a list of replication rules to every did in dids
333
488
 
334
489
  :params dids: List of data identifiers.
335
490
  :param rules: List of dictionaries defining replication rules.
336
- {account, copies, rse_expression, grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notifiy, purge_replicas}
491
+ {account, copies, rse_expression, grouping, weight, lifetime, locked, subscription_id, source_replica_expression, activity, notify, purge_replicas}
337
492
  :param session: The database session in use.
338
493
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
339
494
  :returns: Dictionary (scope, name) with list of created rule ids
@@ -368,37 +523,61 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
368
523
  # 2. Get the did
369
524
  with METRICS.timer('add_rules.get_did'):
370
525
  try:
371
- did = session.query(models.DataIdentifier).filter(
372
- models.DataIdentifier.scope == elem['scope'],
373
- models.DataIdentifier.name == elem['name']).one()
374
- except NoResultFound:
375
- raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
526
+ stmt = select(
527
+ models.DataIdentifier
528
+ ).where(
529
+ and_(models.DataIdentifier.scope == elem['scope'],
530
+ models.DataIdentifier.name == elem['name'])
531
+ )
532
+ did = session.execute(stmt).scalar_one()
533
+ except NoResultFound as exc:
534
+ raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
376
535
  except TypeError as error:
377
- raise InvalidObject(error.args)
536
+ raise InvalidObject(error.args) from error
378
537
 
379
538
  # 2.1 If the did is a constituent, relay the rule to the archive
380
539
  if did.did_type == DIDType.FILE and did.constituent: # Check if a single replica of this DID exists
381
- replica_cnt = session.query(models.RSEFileAssociation).join(models.RSE, models.RSEFileAssociation.rse_id == models.RSE.id)\
382
- .filter(models.RSEFileAssociation.scope == did.scope,
383
- models.RSEFileAssociation.name == did.name,
384
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
385
- models.RSE.rse_type != RSEType.TAPE).count()
540
+ stmt = select(
541
+ func.count()
542
+ ).select_from(
543
+ models.RSEFileAssociation
544
+ ).join(
545
+ models.RSE,
546
+ models.RSEFileAssociation.rse_id == models.RSE.id
547
+ ).where(
548
+ and_(models.RSEFileAssociation.scope == did.scope,
549
+ models.RSEFileAssociation.name == did.name,
550
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
551
+ models.RSE.rse_type != RSEType.TAPE)
552
+ )
553
+ replica_cnt = session.execute(stmt).scalar()
386
554
  if replica_cnt == 0: # Put the rule on the archive
387
- archive = session.query(models.ConstituentAssociation).join(models.RSEFileAssociation,
388
- and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
389
- models.ConstituentAssociation.name == models.RSEFileAssociation.name))\
390
- .filter(models.ConstituentAssociation.child_scope == did.scope,
391
- models.ConstituentAssociation.child_name == did.name).first()
555
+ stmt = select(
556
+ models.ConstituentAssociation
557
+ ).join(
558
+ models.RSEFileAssociation,
559
+ and_(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
560
+ models.ConstituentAssociation.name == models.RSEFileAssociation.name)
561
+ ).where(
562
+ and_(models.ConstituentAssociation.child_scope == did.scope,
563
+ models.ConstituentAssociation.child_name == did.name)
564
+ )
565
+ archive = session.execute(stmt).scalars().first()
392
566
  if archive is not None:
393
567
  elem['name'] = archive.name
394
568
  elem['scope'] = archive.scope
395
569
  try:
396
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == elem['scope'],
397
- models.DataIdentifier.name == elem['name']).one()
398
- except NoResultFound:
399
- raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
570
+ stmt = select(
571
+ models.DataIdentifier
572
+ ).where(
573
+ and_(models.DataIdentifier.scope == elem['scope'],
574
+ models.DataIdentifier.name == elem['name'])
575
+ )
576
+ did = session.execute(stmt).scalar_one()
577
+ except NoResultFound as exc:
578
+ raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name'])) from exc
400
579
  except TypeError as error:
401
- raise InvalidObject(error.args)
580
+ raise InvalidObject(error.args) from error
402
581
  else: # Put the rule on the constituent directly
403
582
  pass
404
583
 
@@ -445,7 +624,7 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
445
624
  # Block manual approval if RSE does not allow it
446
625
  if rule.get('ask_approval', False):
447
626
  for rse in rses:
448
- if list_rse_attributes(rse_id=rse['id'], session=session).get('block_manual_approval', False):
627
+ if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.BLOCK_MANUAL_APPROVAL, False):
449
628
  raise ManualRuleApprovalBlocked()
450
629
 
451
630
  if rule.get('source_replica_expression'):
@@ -459,11 +638,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
459
638
 
460
639
  # 4. Create the replication rule
461
640
  with METRICS.timer('add_rules.create_rule'):
462
- grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(rule.get('grouping'), RuleGrouping.DATASET)
641
+ grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(str(rule.get('grouping')), RuleGrouping.DATASET)
463
642
 
464
- expires_at = datetime.utcnow() + timedelta(seconds=rule.get('lifetime')) if rule.get('lifetime') is not None else None
643
+ rule_lifetime: Optional[int] = rule.get('lifetime')
644
+ expires_at: Optional[datetime] = datetime.utcnow() + timedelta(seconds=rule_lifetime) if rule_lifetime is not None else None
465
645
 
466
- notify = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS}.get(rule.get('notify'), RuleNotification.NO)
646
+ notify = {'Y': RuleNotification.YES, 'C': RuleNotification.CLOSE, 'P': RuleNotification.PROGRESS, None: RuleNotification.NO}.get(rule.get('notify'))
467
647
 
468
648
  if rule.get('meta') is not None:
469
649
  try:
@@ -498,10 +678,10 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
498
678
  new_rule.save(session=session)
499
679
  except IntegrityError as error:
500
680
  if match('.*ORA-00001.*', str(error.args[0])):
501
- raise DuplicateRule(error.args[0])
681
+ raise DuplicateRule(error.args[0]) from error
502
682
  elif str(error.args[0]) == '(IntegrityError) UNIQUE constraint failed: rules.scope, rules.name, rules.account, rules.rse_expression, rules.copies':
503
- raise DuplicateRule(error.args[0])
504
- raise InvalidReplicationRule(error.args[0])
683
+ raise DuplicateRule(error.args[0]) from error
684
+ raise InvalidReplicationRule(error.args[0]) from error
505
685
 
506
686
  rule_ids[(did.scope, did.name)].append(new_rule.id)
507
687
 
@@ -514,12 +694,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
514
694
  # This rule can be considered for auto-approval:
515
695
  rse_attr = list_rse_attributes(rse_id=rses[0]['id'], session=session)
516
696
  auto_approve = False
517
- if 'auto_approve_bytes' in rse_attr and 'auto_approve_files' in rse_attr:
518
- if did.bytes < int(rse_attr.get('auto_approve_bytes')) and did.length < int(rse_attr.get('auto_approve_bytes')):
697
+ if RseAttr.AUTO_APPROVE_BYTES in rse_attr and RseAttr.AUTO_APPROVE_FILES in rse_attr:
698
+ if did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)) and did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES)):
519
699
  auto_approve = True
520
- elif did.bytes < int(rse_attr.get('auto_approve_bytes', -1)):
700
+ elif did.bytes < int(rse_attr.get(RseAttr.AUTO_APPROVE_BYTES, -1)):
521
701
  auto_approve = True
522
- elif did.length < int(rse_attr.get('auto_approve_files', -1)):
702
+ elif did.length < int(rse_attr.get(RseAttr.AUTO_APPROVE_FILES, -1)):
523
703
  auto_approve = True
524
704
  if auto_approve:
525
705
  logger(logging.DEBUG, "Auto approving rule %s", str(new_rule.id))
@@ -557,17 +737,31 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
557
737
  source_rses=[rse['id'] for rse in source_rses],
558
738
  session=session)
559
739
  except IntegrityError as error:
560
- raise ReplicationRuleCreationTemporaryFailed(error.args[0])
740
+ raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
561
741
 
562
742
  if new_rule.locks_stuck_cnt > 0:
563
743
  new_rule.state = RuleState.STUCK
564
744
  new_rule.error = 'MissingSourceReplica'
565
745
  if new_rule.grouping != RuleGrouping.NONE:
566
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.STUCK})
746
+ stmt = update(
747
+ models.DatasetLock
748
+ ).where(
749
+ models.DatasetLock.rule_id == new_rule.id
750
+ ).values({
751
+ models.DatasetLock.state: LockState.STUCK
752
+ })
753
+ session.execute(stmt)
567
754
  elif new_rule.locks_replicating_cnt == 0:
568
755
  new_rule.state = RuleState.OK
569
756
  if new_rule.grouping != RuleGrouping.NONE:
570
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.OK})
757
+ stmt = update(
758
+ models.DatasetLock
759
+ ).where(
760
+ models.DatasetLock.rule_id == new_rule.id
761
+ ).values({
762
+ models.DatasetLock.state: LockState.OK
763
+ })
764
+ session.execute(stmt)
571
765
  session.flush()
572
766
  if new_rule.notification == RuleNotification.YES:
573
767
  generate_email_for_rule_ok_notification(rule=new_rule, session=session)
@@ -575,7 +769,14 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
575
769
  else:
576
770
  new_rule.state = RuleState.REPLICATING
577
771
  if new_rule.grouping != RuleGrouping.NONE:
578
- session.query(models.DatasetLock).filter_by(rule_id=new_rule.id).update({'state': LockState.REPLICATING})
772
+ stmt = update(
773
+ models.DatasetLock
774
+ ).where(
775
+ models.DatasetLock.rule_id == new_rule.id
776
+ ).values({
777
+ models.DatasetLock.state: LockState.REPLICATING
778
+ })
779
+ session.execute(stmt)
579
780
 
580
781
  # Add rule to History
581
782
  insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
@@ -586,7 +787,12 @@ def add_rules(dids, rules, *, session: "Session", logger=logging.log):
586
787
 
587
788
 
588
789
  @transactional_session
589
- def inject_rule(rule_id, *, session: "Session", logger=logging.log):
790
+ def inject_rule(
791
+ rule_id: str,
792
+ *,
793
+ session: "Session",
794
+ logger: LoggerFunction = logging.log
795
+ ) -> None:
590
796
  """
591
797
  Inject a replication rule.
592
798
 
@@ -598,9 +804,16 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
598
804
  """
599
805
 
600
806
  try:
601
- rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == rule_id).with_for_update(nowait=True).one()
602
- except NoResultFound:
603
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
807
+ stmt = select(
808
+ models.ReplicationRule
809
+ ).where(
810
+ models.ReplicationRule.id == rule_id
811
+ ).with_for_update(
812
+ nowait=True
813
+ )
814
+ rule = session.execute(stmt).scalar_one()
815
+ except NoResultFound as exc:
816
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
604
817
 
605
818
  # Check if rule will expire in the next 5 minutes:
606
819
  if rule.child_rule_id is None and rule.expires_at is not None and rule.expires_at < datetime.utcnow() + timedelta(seconds=300):
@@ -615,7 +828,13 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
615
828
  # Remove duplicates from the list of dictionaries
616
829
  dids = [dict(t) for t in {tuple(d.items()) for d in dids}]
617
830
  # Remove dids which already have a similar rule
618
- dids = [did for did in dids if session.query(models.ReplicationRule).filter_by(scope=did['scope'], name=did['name'], account=rule.account, rse_expression=rule.rse_expression).count() == 0]
831
+ stmt = select(
832
+ models.ReplicationRule.id
833
+ ).where(
834
+ and_(models.ReplicationRule.account == rule.account,
835
+ models.ReplicationRule.rse_expression == rule.rse_expression)
836
+ )
837
+ dids = [did for did in dids if session.execute(stmt.where(and_(models.ReplicationRule.scope == did['scope'], models.ReplicationRule.name == did['name']))).scalar_one_or_none() is None]
619
838
  if rule.expires_at:
620
839
  lifetime = (rule.expires_at - datetime.utcnow()).days * 24 * 3600 + (rule.expires_at - datetime.utcnow()).seconds
621
840
  else:
@@ -664,29 +883,48 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
664
883
  # 3. Get the did
665
884
  with METRICS.timer('inject_rule.get_did'):
666
885
  try:
667
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == rule.scope,
668
- models.DataIdentifier.name == rule.name).one()
669
- except NoResultFound:
670
- raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (rule.scope, rule.name))
886
+ stmt = select(
887
+ models.DataIdentifier
888
+ ).where(
889
+ and_(models.DataIdentifier.scope == rule.scope,
890
+ models.DataIdentifier.name == rule.name)
891
+ )
892
+ did = session.execute(stmt).scalar_one()
893
+ except NoResultFound as exc:
894
+ raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (rule.scope, rule.name)) from exc
671
895
  except TypeError as error:
672
- raise InvalidObject(error.args)
896
+ raise InvalidObject(error.args) from error
673
897
 
674
898
  # 4. Apply the rule
675
899
  with METRICS.timer('inject_rule.apply_rule'):
676
900
  try:
677
901
  apply_rule(did, rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
678
902
  except IntegrityError as error:
679
- raise ReplicationRuleCreationTemporaryFailed(error.args[0])
903
+ raise ReplicationRuleCreationTemporaryFailed(error.args[0]) from error
680
904
 
681
905
  if rule.locks_stuck_cnt > 0:
682
906
  rule.state = RuleState.STUCK
683
907
  rule.error = 'MissingSourceReplica'
684
908
  if rule.grouping != RuleGrouping.NONE:
685
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
909
+ stmt = update(
910
+ models.DatasetLock
911
+ ).where(
912
+ models.DatasetLock.rule_id == rule.id
913
+ ).values({
914
+ models.DatasetLock.state: LockState.STUCK
915
+ })
916
+ session.execute(stmt)
686
917
  elif rule.locks_replicating_cnt == 0:
687
918
  rule.state = RuleState.OK
688
919
  if rule.grouping != RuleGrouping.NONE:
689
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.OK})
920
+ stmt = update(
921
+ models.DatasetLock
922
+ ).where(
923
+ models.DatasetLock.rule_id == rule.id
924
+ ).values({
925
+ models.DatasetLock.state: LockState.OK
926
+ })
927
+ session.execute(stmt)
690
928
  session.flush()
691
929
  if rule.notification == RuleNotification.YES:
692
930
  generate_email_for_rule_ok_notification(rule=rule, session=session)
@@ -696,7 +934,14 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
696
934
  else:
697
935
  rule.state = RuleState.REPLICATING
698
936
  if rule.grouping != RuleGrouping.NONE:
699
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.REPLICATING})
937
+ stmt = update(
938
+ models.DatasetLock
939
+ ).where(
940
+ models.DatasetLock.rule_id == rule.id
941
+ ).values({
942
+ models.DatasetLock.state: LockState.REPLICATING
943
+ })
944
+ session.execute(stmt)
700
945
 
701
946
  # Add rule to History
702
947
  insert_rule_history(rule=rule, recent=True, longterm=True, session=session)
@@ -705,7 +950,11 @@ def inject_rule(rule_id, *, session: "Session", logger=logging.log):
705
950
 
706
951
 
707
952
  @stream_session
708
- def list_rules(filters={}, *, session: "Session"):
953
+ def list_rules(
954
+ filters: Optional[dict[str, Any]] = None,
955
+ *,
956
+ session: "Session"
957
+ ) -> Iterator[dict[str, Any]]:
709
958
  """
710
959
  List replication rules.
711
960
 
@@ -724,7 +973,7 @@ def list_rules(filters={}, *, session: "Session"):
724
973
  models.ReplicationRule.name == models.DataIdentifier.name
725
974
  )
726
975
  )
727
- if filters:
976
+ if filters is not None:
728
977
  for (key, value) in filters.items():
729
978
  if key in ['account', 'scope']:
730
979
  if '*' in value.internal:
@@ -763,12 +1012,16 @@ def list_rules(filters={}, *, session: "Session"):
763
1012
  d = rule.to_dict()
764
1013
  d['bytes'] = data_identifier_bytes
765
1014
  yield d
766
- except StatementError:
767
- raise RucioException('Badly formatted input (IDs?)')
1015
+ except StatementError as exc:
1016
+ raise RucioException('Badly formatted input (IDs?)') from exc
768
1017
 
769
1018
 
770
1019
  @stream_session
771
- def list_rule_history(rule_id, *, session: "Session"):
1020
+ def list_rule_history(
1021
+ rule_id: str,
1022
+ *,
1023
+ session: "Session"
1024
+ ) -> Iterator[dict[str, Any]]:
772
1025
  """
773
1026
  List the rule history of a rule.
774
1027
 
@@ -777,21 +1030,32 @@ def list_rule_history(rule_id, *, session: "Session"):
777
1030
  :raises: RucioException
778
1031
  """
779
1032
 
780
- query = session.query(models.ReplicationRuleHistoryRecent.updated_at,
781
- models.ReplicationRuleHistoryRecent.state,
782
- models.ReplicationRuleHistoryRecent.locks_ok_cnt,
783
- models.ReplicationRuleHistoryRecent.locks_stuck_cnt,
784
- models.ReplicationRuleHistoryRecent.locks_replicating_cnt).filter_by(id=rule_id).order_by(models.ReplicationRuleHistoryRecent.updated_at)
1033
+ stmt = select(
1034
+ models.ReplicationRuleHistory.updated_at,
1035
+ models.ReplicationRuleHistory.state,
1036
+ models.ReplicationRuleHistory.locks_ok_cnt,
1037
+ models.ReplicationRuleHistory.locks_stuck_cnt,
1038
+ models.ReplicationRuleHistory.locks_replicating_cnt
1039
+ ).where(
1040
+ models.ReplicationRuleHistory.id == rule_id
1041
+ ).order_by(
1042
+ models.ReplicationRuleHistory.updated_at
1043
+ )
785
1044
 
786
1045
  try:
787
- for rule in query.yield_per(5):
788
- yield {'updated_at': rule[0], 'state': rule[1], 'locks_ok_cnt': rule[2], 'locks_stuck_cnt': rule[3], 'locks_replicating_cnt': rule[4]}
789
- except StatementError:
790
- raise RucioException('Badly formatted input (IDs?)')
1046
+ for rule in session.execute(stmt).yield_per(5):
1047
+ yield rule._asdict()
1048
+ except StatementError as exc:
1049
+ raise RucioException('Badly formatted input (IDs?)') from exc
791
1050
 
792
1051
 
793
1052
  @stream_session
794
- def list_rule_full_history(scope, name, *, session: "Session"):
1053
+ def list_rule_full_history(
1054
+ scope: InternalScope,
1055
+ name: str,
1056
+ *,
1057
+ session: "Session"
1058
+ ) -> Iterator[dict[str, Any]]:
795
1059
  """
796
1060
  List the rule history of a DID.
797
1061
 
@@ -801,26 +1065,36 @@ def list_rule_full_history(scope, name, *, session: "Session"):
801
1065
  :raises: RucioException
802
1066
  """
803
1067
 
804
- query = session.query(models.ReplicationRuleHistory.id,
805
- models.ReplicationRuleHistory.created_at,
806
- models.ReplicationRuleHistory.updated_at,
807
- models.ReplicationRuleHistory.rse_expression,
808
- models.ReplicationRuleHistory.state,
809
- models.ReplicationRuleHistory.account,
810
- models.ReplicationRuleHistory.locks_ok_cnt,
811
- models.ReplicationRuleHistory.locks_stuck_cnt,
812
- models.ReplicationRuleHistory.locks_replicating_cnt).\
813
- with_hint(models.ReplicationRuleHistory, "INDEX(RULES_HISTORY_SCOPENAME_IDX)", 'oracle').\
814
- filter(models.ReplicationRuleHistory.scope == scope, models.ReplicationRuleHistory.name == name).\
815
- order_by(models.ReplicationRuleHistory.created_at, models.ReplicationRuleHistory.updated_at)
816
-
817
- for rule in query.yield_per(5):
818
- yield {'rule_id': rule[0], 'created_at': rule[1], 'updated_at': rule[2], 'rse_expression': rule[3], 'state': rule[4],
819
- 'account': rule[5], 'locks_ok_cnt': rule[6], 'locks_stuck_cnt': rule[7], 'locks_replicating_cnt': rule[8]}
1068
+ stmt = select(
1069
+ models.ReplicationRuleHistory.id.label('rule_id'),
1070
+ models.ReplicationRuleHistory.created_at,
1071
+ models.ReplicationRuleHistory.updated_at,
1072
+ models.ReplicationRuleHistory.rse_expression,
1073
+ models.ReplicationRuleHistory.state,
1074
+ models.ReplicationRuleHistory.account,
1075
+ models.ReplicationRuleHistory.locks_ok_cnt,
1076
+ models.ReplicationRuleHistory.locks_stuck_cnt,
1077
+ models.ReplicationRuleHistory.locks_replicating_cnt
1078
+ ).with_hint(
1079
+ models.ReplicationRuleHistory, 'INDEX(RULES_HISTORY_SCOPENAME_IDX)', 'oracle'
1080
+ ).where(
1081
+ and_(models.ReplicationRuleHistory.scope == scope,
1082
+ models.ReplicationRuleHistory.name == name)
1083
+ ).order_by(
1084
+ models.ReplicationRuleHistory.created_at,
1085
+ models.ReplicationRuleHistory.updated_at
1086
+ )
1087
+ for rule in session.execute(stmt).yield_per(5):
1088
+ yield rule._asdict()
820
1089
 
821
1090
 
822
1091
  @stream_session
823
- def list_associated_rules_for_file(scope, name, *, session: "Session"):
1092
+ def list_associated_rules_for_file(
1093
+ scope: InternalScope,
1094
+ name: str,
1095
+ *,
1096
+ session: "Session"
1097
+ ) -> Iterator[dict[str, Any]]:
824
1098
  """
825
1099
  List replication rules a file is affected from.
826
1100
 
@@ -829,22 +1103,44 @@ def list_associated_rules_for_file(scope, name, *, session: "Session"):
829
1103
  :param session: The database session in use.
830
1104
  :raises: RucioException
831
1105
  """
832
-
833
- rucio.core.did.get_did(scope=scope, name=name, session=session) # Check if the did acually exists
834
- query = session.query(models.ReplicationRule).\
835
- with_hint(models.ReplicaLock, "INDEX(LOCKS LOCKS_PK)", 'oracle').\
836
- join(models.ReplicaLock, models.ReplicationRule.id == models.ReplicaLock.rule_id).\
837
- filter(models.ReplicaLock.scope == scope, models.ReplicaLock.name == name).distinct()
1106
+ rucio.core.did.get_did(scope=scope, name=name, session=session) # Check if the did actually exists
1107
+ stmt = select(
1108
+ models.ReplicationRule,
1109
+ models.DataIdentifier.bytes
1110
+ ).distinct(
1111
+ ).join(
1112
+ models.ReplicaLock,
1113
+ models.ReplicationRule.id == models.ReplicaLock.rule_id
1114
+ ).join(
1115
+ models.DataIdentifier,
1116
+ and_(models.ReplicationRule.scope == models.DataIdentifier.scope,
1117
+ models.ReplicationRule.name == models.DataIdentifier.name)
1118
+ ).with_hint(
1119
+ models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
1120
+ ).where(
1121
+ and_(models.ReplicaLock.scope == scope,
1122
+ models.ReplicaLock.name == name)
1123
+ )
838
1124
  try:
839
- for rule in query.yield_per(5):
840
- yield rule.to_dict()
841
- except StatementError:
842
- raise RucioException('Badly formatted input (IDs?)')
1125
+ for rule, data_identifier_bytes in session.execute(stmt).yield_per(5):
1126
+ d = rule.to_dict()
1127
+ d['bytes'] = data_identifier_bytes
1128
+ yield d
1129
+ except StatementError as exc:
1130
+ raise RucioException('Badly formatted input (IDs?)') from exc
843
1131
 
844
1132
 
845
1133
  @transactional_session
846
- def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, nowait=False, *, session: "Session",
847
- ignore_rule_lock=False):
1134
+ def delete_rule(
1135
+ rule_id: str,
1136
+ purge_replicas: Optional[bool] = None,
1137
+ soft: bool = False,
1138
+ delete_parent: bool = False,
1139
+ nowait: bool = False,
1140
+ *,
1141
+ session: "Session",
1142
+ ignore_rule_lock: bool = False
1143
+ ) -> None:
848
1144
  """
849
1145
  Delete a replication rule.
850
1146
 
@@ -861,11 +1157,16 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
861
1157
 
862
1158
  with METRICS.timer('delete_rule.total'):
863
1159
  try:
864
- rule = session.query(models.ReplicationRule)\
865
- .filter(models.ReplicationRule.id == rule_id)\
866
- .with_for_update(nowait=nowait).one()
867
- except NoResultFound:
868
- raise RuleNotFound('No rule with the id %s found' % rule_id)
1160
+ stmt = select(
1161
+ models.ReplicationRule
1162
+ ).where(
1163
+ models.ReplicationRule.id == rule_id
1164
+ ).with_for_update(
1165
+ nowait=nowait
1166
+ )
1167
+ rule = session.execute(stmt).scalar_one()
1168
+ except NoResultFound as exc:
1169
+ raise RuleNotFound('No rule with the id %s found' % rule_id) from exc
869
1170
  if rule.locked and not ignore_rule_lock:
870
1171
  raise UnsupportedOperation('The replication rule is locked and has to be unlocked before it can be deleted.')
871
1172
 
@@ -885,15 +1186,21 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
885
1186
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
886
1187
  return
887
1188
 
888
- locks = session.query(models.ReplicaLock)\
889
- .filter(models.ReplicaLock.rule_id == rule_id)\
890
- .with_for_update(nowait=nowait).yield_per(100)
1189
+ stmt = select(
1190
+ models.ReplicaLock
1191
+ ).where(
1192
+ models.ReplicaLock.rule_id == rule_id
1193
+ ).with_for_update(
1194
+ nowait=nowait
1195
+ )
1196
+ results = session.execute(stmt).yield_per(100)
891
1197
 
892
1198
  # Remove locks, set tombstone if applicable
893
1199
  transfers_to_delete = [] # [{'scope': , 'name':, 'rse_id':}]
894
1200
  account_counter_decreases = {} # {'rse_id': [file_size, file_size, file_size]}
895
1201
 
896
- for lock in locks:
1202
+ for result in results:
1203
+ lock = result[0]
897
1204
  if __delete_lock_and_update_replica(lock=lock, purge_replicas=rule.purge_replicas,
898
1205
  nowait=nowait, session=session):
899
1206
  transfers_to_delete.append({'scope': lock.scope, 'name': lock.name, 'rse_id': lock.rse_id})
@@ -902,9 +1209,14 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
902
1209
  account_counter_decreases[lock.rse_id].append(lock.bytes)
903
1210
 
904
1211
  # Delete the DatasetLocks
905
- session.query(models.DatasetLock)\
906
- .filter(models.DatasetLock.rule_id == rule_id)\
907
- .delete(synchronize_session=False)
1212
+ stmt = delete(
1213
+ models.DatasetLock
1214
+ ).where(
1215
+ models.DatasetLock.rule_id == rule_id
1216
+ ).execution_options(
1217
+ synchronize_session=False
1218
+ )
1219
+ session.execute(stmt)
908
1220
 
909
1221
  # Decrease account_counters
910
1222
  for rse_id in account_counter_decreases.keys():
@@ -927,7 +1239,12 @@ def delete_rule(rule_id, purge_replicas=None, soft=False, delete_parent=False, n
927
1239
 
928
1240
 
929
1241
  @transactional_session
930
- def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1242
+ def repair_rule(
1243
+ rule_id: str,
1244
+ *,
1245
+ session: "Session",
1246
+ logger: LoggerFunction = logging.log
1247
+ ) -> None:
931
1248
  """
932
1249
  Repair a STUCK replication rule.
933
1250
 
@@ -937,8 +1254,8 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
937
1254
  """
938
1255
 
939
1256
  # Rule error cases:
940
- # (A) A rule get's an exception on rule-creation. This can only be the MissingSourceReplica exception.
941
- # (B) A rule get's an error when re-evaluated: InvalidRSEExpression, InvalidRuleWeight, InsufficientTargetRSEs, RSEWriteBlocked
1257
+ # (A) A rule gets an exception on rule-creation. This can only be the MissingSourceReplica exception.
1258
+ # (B) A rule gets an error when re-evaluated: InvalidRSEExpression, InvalidRuleWeight, InsufficientTargetRSEs, RSEWriteBlocked
942
1259
  # InsufficientAccountLimit. The re-evaluation has to be done again and potential missing locks have to be
943
1260
  # created.
944
1261
  # (C) Transfers fail and mark locks (and the rule) as STUCK. All STUCK locks have to be repaired.
@@ -946,7 +1263,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
946
1263
 
947
1264
  # start_time = time.time()
948
1265
  try:
949
- rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == rule_id).with_for_update(nowait=True).one()
1266
+ stmt = select(
1267
+ models.ReplicationRule
1268
+ ).where(
1269
+ models.ReplicationRule.id == rule_id
1270
+ ).with_for_update(
1271
+ nowait=True
1272
+ )
1273
+ rule = session.execute(stmt).scalar_one()
950
1274
  rule.updated_at = datetime.utcnow()
951
1275
 
952
1276
  # Check if rule is longer than 2 weeks in STUCK
@@ -978,7 +1302,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
978
1302
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
979
1303
  # Try to update the DatasetLocks
980
1304
  if rule.grouping != RuleGrouping.NONE:
981
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
1305
+ stmt = update(
1306
+ models.DatasetLock
1307
+ ).where(
1308
+ models.DatasetLock.rule_id == rule.id
1309
+ ).values({
1310
+ models.DatasetLock.state: LockState.STUCK
1311
+ })
1312
+ session.execute(stmt)
982
1313
  logger(logging.DEBUG, '%s while repairing rule %s', str(error), rule_id)
983
1314
  return
984
1315
 
@@ -998,7 +1329,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
998
1329
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
999
1330
  # Try to update the DatasetLocks
1000
1331
  if rule.grouping != RuleGrouping.NONE:
1001
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
1332
+ stmt = update(
1333
+ models.DatasetLock
1334
+ ).where(
1335
+ models.DatasetLock.rule_id == rule.id
1336
+ ).values({
1337
+ models.DatasetLock.state: LockState.STUCK
1338
+ })
1339
+ session.execute(stmt)
1002
1340
  logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
1003
1341
  return
1004
1342
 
@@ -1007,19 +1345,32 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1007
1345
  rule.locks_ok_cnt = 0
1008
1346
  rule.locks_replicating_cnt = 0
1009
1347
  rule.locks_stuck_cnt = 0
1010
- rule_counts = session.query(models.ReplicaLock.state, func.count(models.ReplicaLock.state)).filter(models.ReplicaLock.rule_id == rule.id).group_by(models.ReplicaLock.state).all()
1348
+ stmt = select(
1349
+ models.ReplicaLock.state,
1350
+ func.count(models.ReplicaLock.state).label('state_counter')
1351
+ ).where(
1352
+ models.ReplicaLock.rule_id == rule.id
1353
+ ).group_by(
1354
+ models.ReplicaLock.state
1355
+ )
1356
+ rule_counts = session.execute(stmt).all()
1011
1357
  for count in rule_counts:
1012
- if count[0] == LockState.OK:
1013
- rule.locks_ok_cnt = count[1]
1014
- elif count[0] == LockState.REPLICATING:
1015
- rule.locks_replicating_cnt = count[1]
1016
- elif count[0] == LockState.STUCK:
1017
- rule.locks_stuck_cnt = count[1]
1358
+ if count.state == LockState.OK:
1359
+ rule.locks_ok_cnt = count.state_counter
1360
+ elif count.state == LockState.REPLICATING:
1361
+ rule.locks_replicating_cnt = count.state_counter
1362
+ elif count.state == LockState.STUCK:
1363
+ rule.locks_stuck_cnt = count.state_counter
1018
1364
  logger(logging.DEBUG, "Finished resetting counters for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
1019
1365
 
1020
1366
  # Get the did
1021
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == rule.scope,
1022
- models.DataIdentifier.name == rule.name).one()
1367
+ stmt = select(
1368
+ models.DataIdentifier
1369
+ ).where(
1370
+ and_(models.DataIdentifier.scope == rule.scope,
1371
+ models.DataIdentifier.name == rule.name)
1372
+ )
1373
+ did = session.execute(stmt).scalar_one()
1023
1374
 
1024
1375
  # Detect if there is something wrong with the dataset and
1025
1376
  # make the decisison on soft or hard repair.
@@ -1064,7 +1415,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1064
1415
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
1065
1416
  # Try to update the DatasetLocks
1066
1417
  if rule.grouping != RuleGrouping.NONE:
1067
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
1418
+ stmt = update(
1419
+ models.DatasetLock
1420
+ ).where(
1421
+ models.DatasetLock.rule_id == rule.id
1422
+ ).values({
1423
+ models.DatasetLock.state: LockState.STUCK
1424
+ })
1425
+ session.execute(stmt)
1068
1426
  logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
1069
1427
  return
1070
1428
 
@@ -1074,11 +1432,7 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1074
1432
  if hard_repair:
1075
1433
  __find_surplus_locks_and_remove_them(datasetfiles=datasetfiles,
1076
1434
  locks=locks,
1077
- replicas=replicas,
1078
- source_replicas=source_replicas,
1079
- rseselector=rseselector,
1080
1435
  rule=rule,
1081
- source_rses=[rse['id'] for rse in source_rses],
1082
1436
  session=session)
1083
1437
 
1084
1438
  session.flush()
@@ -1101,13 +1455,32 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1101
1455
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
1102
1456
  # Try to update the DatasetLocks
1103
1457
  if rule.grouping != RuleGrouping.NONE:
1104
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
1458
+ stmt = update(
1459
+ models.DatasetLock
1460
+ ).where(
1461
+ models.DatasetLock.rule_id == rule.id
1462
+ ).values({
1463
+ models.DatasetLock.state: LockState.STUCK
1464
+ })
1465
+ session.execute(stmt)
1105
1466
  logger(logging.DEBUG, '%s while repairing rule %s', type(error).__name__, rule_id)
1106
1467
  return
1107
1468
 
1108
1469
  # Delete Datasetlocks which are not relevant anymore
1109
- validated_datasetlock_rse_ids = [rse_id[0] for rse_id in session.query(models.ReplicaLock.rse_id).filter(models.ReplicaLock.rule_id == rule.id).group_by(models.ReplicaLock.rse_id).all()]
1110
- dataset_locks = session.query(models.DatasetLock).filter_by(rule_id=rule.id).all()
1470
+ stmt = select(
1471
+ models.ReplicaLock.rse_id
1472
+ ).distinct(
1473
+ ).where(
1474
+ models.ReplicaLock.rule_id == rule.id
1475
+ )
1476
+ validated_datasetlock_rse_ids = session.execute(stmt).scalars().all()
1477
+
1478
+ stmt = select(
1479
+ models.DatasetLock
1480
+ ).where(
1481
+ models.DatasetLock.rule_id == rule.id
1482
+ )
1483
+ dataset_locks = session.execute(stmt).scalars().all()
1111
1484
  for dataset_lock in dataset_locks:
1112
1485
  if dataset_lock.rse_id not in validated_datasetlock_rse_ids:
1113
1486
  dataset_lock.delete(session=session)
@@ -1119,7 +1492,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1119
1492
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
1120
1493
  # Try to update the DatasetLocks
1121
1494
  if rule.grouping != RuleGrouping.NONE:
1122
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
1495
+ stmt = update(
1496
+ models.DatasetLock
1497
+ ).where(
1498
+ models.DatasetLock.rule_id == rule.id
1499
+ ).values({
1500
+ models.DatasetLock.state: LockState.STUCK
1501
+ })
1502
+ session.execute(stmt)
1123
1503
  # TODO: Increase some kind of Stuck Counter here, The rule should at some point be SUSPENDED
1124
1504
  return
1125
1505
 
@@ -1133,7 +1513,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1133
1513
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
1134
1514
  # Try to update the DatasetLocks
1135
1515
  if rule.grouping != RuleGrouping.NONE:
1136
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.REPLICATING})
1516
+ stmt = update(
1517
+ models.DatasetLock
1518
+ ).where(
1519
+ models.DatasetLock.rule_id == rule.id
1520
+ ).values({
1521
+ models.DatasetLock.state: LockState.REPLICATING
1522
+ })
1523
+ session.execute(stmt)
1137
1524
  return
1138
1525
 
1139
1526
  rule.state = RuleState.OK
@@ -1143,7 +1530,14 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1143
1530
  logger(logging.INFO, 'Rule %s [%d/%d/%d] state=OK', str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
1144
1531
 
1145
1532
  if rule.grouping != RuleGrouping.NONE:
1146
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.OK})
1533
+ stmt = update(
1534
+ models.DatasetLock
1535
+ ).where(
1536
+ models.DatasetLock.rule_id == rule.id
1537
+ ).values({
1538
+ models.DatasetLock.state: LockState.OK
1539
+ })
1540
+ session.execute(stmt)
1147
1541
  session.flush()
1148
1542
  if rule.notification == RuleNotification.YES:
1149
1543
  generate_email_for_rule_ok_notification(rule=rule, session=session)
@@ -1159,7 +1553,11 @@ def repair_rule(rule_id, *, session: "Session", logger=logging.log):
1159
1553
 
1160
1554
 
1161
1555
  @read_session
1162
- def get_rule(rule_id, *, session: "Session"):
1556
+ def get_rule(
1557
+ rule_id: str,
1558
+ *,
1559
+ session: "Session"
1560
+ ) -> dict[str, Any]:
1163
1561
  """
1164
1562
  Get a specific replication rule.
1165
1563
 
@@ -1169,16 +1567,26 @@ def get_rule(rule_id, *, session: "Session"):
1169
1567
  """
1170
1568
 
1171
1569
  try:
1172
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
1570
+ stmt = select(
1571
+ models.ReplicationRule
1572
+ ).where(
1573
+ models.ReplicationRule.id == rule_id
1574
+ )
1575
+ rule = session.execute(stmt).scalar_one()
1173
1576
  return rule.to_dict()
1174
- except NoResultFound:
1175
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
1176
- except StatementError:
1177
- raise RucioException('Badly formatted rule id (%s)' % (rule_id))
1577
+ except NoResultFound as exc:
1578
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
1579
+ except StatementError as exc:
1580
+ raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
1178
1581
 
1179
1582
 
1180
1583
  @transactional_session
1181
- def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") -> None:
1584
+ def update_rule(
1585
+ rule_id: str,
1586
+ options: dict[str, Any],
1587
+ *,
1588
+ session: "Session"
1589
+ ) -> None:
1182
1590
  """
1183
1591
  Update a rules options.
1184
1592
 
@@ -1291,9 +1699,9 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
1291
1699
  locktype
1292
1700
  ).where(
1293
1701
  locktype.rule_id == rule.id
1294
- ).values(
1295
- account=options['account']
1296
- )
1702
+ ).values({
1703
+ locktype.account: options['account']
1704
+ })
1297
1705
  session.execute(query)
1298
1706
 
1299
1707
  # Update counters
@@ -1361,18 +1769,18 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
1361
1769
  ).where(
1362
1770
  models.ReplicationRule.id == rid,
1363
1771
  models.ReplicationRule.state != RuleState.SUSPENDED
1364
- ).values(
1365
- state=RuleState.STUCK
1366
- )
1772
+ ).values({
1773
+ models.ReplicationRule.state: RuleState.STUCK
1774
+ })
1367
1775
  session.execute(query)
1368
1776
 
1369
1777
  query = update(
1370
1778
  models.DatasetLock
1371
1779
  ).where(
1372
1780
  models.DatasetLock.rule_id == rid
1373
- ).values(
1374
- state=LockState.STUCK
1375
- )
1781
+ ).values({
1782
+ models.DatasetLock.state: LockState.STUCK
1783
+ })
1376
1784
  session.execute(query)
1377
1785
 
1378
1786
  if options['state'].lower() == 'suspended':
@@ -1387,18 +1795,18 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
1387
1795
  ).where(
1388
1796
  models.ReplicaLock.rule_id == rule.id,
1389
1797
  models.ReplicaLock.state == LockState.REPLICATING
1390
- ).values(
1391
- state=LockState.STUCK
1392
- )
1798
+ ).values({
1799
+ models.ReplicaLock.state: LockState.STUCK
1800
+ })
1393
1801
  session.execute(query)
1394
1802
 
1395
1803
  query = update(
1396
1804
  models.DatasetLock
1397
1805
  ).where(
1398
1806
  models.DatasetLock.rule_id == rule_id
1399
- ).values(
1400
- state=LockState.STUCK
1401
- )
1807
+ ).values({
1808
+ models.DatasetLock.state: LockState.STUCK
1809
+ })
1402
1810
  session.execute(query)
1403
1811
 
1404
1812
  elif key == 'cancel_requests':
@@ -1409,8 +1817,8 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
1409
1817
  rule.priority = options[key]
1410
1818
  transfers_to_update = request_core.update_requests_priority(priority=options[key], filter_={'rule_id': rule_id}, session=session)
1411
1819
  transfer_core.update_transfer_priority(transfers_to_update)
1412
- except Exception:
1413
- raise UnsupportedOperation('The FTS Requests are already in a final state.')
1820
+ except Exception as exc:
1821
+ raise UnsupportedOperation('The FTS Requests are already in a final state.') from exc
1414
1822
 
1415
1823
  elif key == 'child_rule_id':
1416
1824
  # Check if the child rule has the same scope/name as the parent rule
@@ -1470,17 +1878,23 @@ def update_rule(rule_id: str, options: dict[str, Any], *, session: "Session") ->
1470
1878
  or match('.*IntegrityError.*UNIQUE constraint failed.*', str(error.args[0])) \
1471
1879
  or match('.*1062.*Duplicate entry.*for key.*', str(error.args[0])) \
1472
1880
  or match('.*IntegrityError.*columns? .*not unique.*', str(error.args[0])):
1473
- raise DuplicateRule(error.args[0])
1881
+ raise DuplicateRule(error.args[0]) from error
1474
1882
  else:
1475
1883
  raise error
1476
- except NoResultFound:
1477
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
1478
- except StatementError as e:
1479
- raise RucioException(f"A StatementError occurred while processing rule {rule_id}") from e
1884
+ except NoResultFound as exc:
1885
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
1886
+ except StatementError as exc:
1887
+ raise RucioException(f"A StatementError occurred while processing rule {rule_id}") from exc
1480
1888
 
1481
1889
 
1482
1890
  @transactional_session
1483
- def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session"):
1891
+ def reduce_rule(
1892
+ rule_id: str,
1893
+ copies: int,
1894
+ exclude_expression: Optional[str] = None,
1895
+ *,
1896
+ session: "Session"
1897
+ ) -> str:
1484
1898
  """
1485
1899
  Reduce the number of copies for a rule by atomically replacing the rule.
1486
1900
 
@@ -1491,7 +1905,12 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
1491
1905
  :raises: RuleReplaceFailed, RuleNotFound
1492
1906
  """
1493
1907
  try:
1494
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
1908
+ stmt = select(
1909
+ models.ReplicationRule
1910
+ ).where(
1911
+ models.ReplicationRule.id == rule_id
1912
+ )
1913
+ rule = session.execute(stmt).scalar_one()
1495
1914
 
1496
1915
  if copies >= rule.copies:
1497
1916
  raise RuleReplaceFailed('Copies of the new rule must be smaller than the old rule.')
@@ -1531,7 +1950,12 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
1531
1950
 
1532
1951
  session.flush()
1533
1952
 
1534
- new_rule = session.query(models.ReplicationRule).filter_by(id=new_rule_id[0]).one()
1953
+ stmt = select(
1954
+ models.ReplicationRule
1955
+ ).where(
1956
+ models.ReplicationRule.id == new_rule_id[0]
1957
+ )
1958
+ new_rule = session.execute(stmt).scalar_one()
1535
1959
 
1536
1960
  if new_rule.state != RuleState.OK:
1537
1961
  raise RuleReplaceFailed('The replacement of the rule failed.')
@@ -1541,12 +1965,18 @@ def reduce_rule(rule_id, copies, exclude_expression=None, *, session: "Session")
1541
1965
 
1542
1966
  return new_rule_id[0]
1543
1967
 
1544
- except NoResultFound:
1545
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
1968
+ except NoResultFound as exc:
1969
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
1546
1970
 
1547
1971
 
1548
1972
  @transactional_session
1549
- def move_rule(rule_id: str, rse_expression: str, override: Optional[dict[str, Any]] = None, *, session: "Session"):
1973
+ def move_rule(
1974
+ rule_id: str,
1975
+ rse_expression: str,
1976
+ override: Optional[dict[str, Any]] = None,
1977
+ *,
1978
+ session: "Session"
1979
+ ) -> str:
1550
1980
  """
1551
1981
  Move a replication rule to another RSE and, once done, delete the original one.
1552
1982
 
@@ -1559,7 +1989,12 @@ def move_rule(rule_id: str, rse_expression: str, override: Optional[dict[str, An
1559
1989
  override = override or {}
1560
1990
 
1561
1991
  try:
1562
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
1992
+ stmt = select(
1993
+ models.ReplicationRule
1994
+ ).where(
1995
+ models.ReplicationRule.id == rule_id
1996
+ )
1997
+ rule = session.execute(stmt).scalar_one()
1563
1998
 
1564
1999
  if rule.child_rule_id:
1565
2000
  raise RuleReplaceFailed('The rule must not have a child rule.')
@@ -1608,14 +2043,20 @@ def move_rule(rule_id: str, rse_expression: str, override: Optional[dict[str, An
1608
2043
 
1609
2044
  return new_rule_id[0]
1610
2045
 
1611
- except StatementError:
1612
- raise RucioException('Badly formatted rule id (%s)' % (rule_id))
1613
- except NoResultFound:
1614
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
2046
+ except StatementError as exc:
2047
+ raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
2048
+ except NoResultFound as exc:
2049
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
1615
2050
 
1616
2051
 
1617
2052
  @transactional_session
1618
- def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
2053
+ def re_evaluate_did(
2054
+ scope: InternalScope,
2055
+ name: str,
2056
+ rule_evaluation_action: DIDReEvaluation,
2057
+ *,
2058
+ session: "Session"
2059
+ ) -> None:
1619
2060
  """
1620
2061
  Re-Evaluates a did.
1621
2062
 
@@ -1627,10 +2068,15 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
1627
2068
  """
1628
2069
 
1629
2070
  try:
1630
- did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == scope,
1631
- models.DataIdentifier.name == name).one()
1632
- except NoResultFound:
1633
- raise DataIdentifierNotFound()
2071
+ stmt = select(
2072
+ models.DataIdentifier
2073
+ ).where(
2074
+ and_(models.DataIdentifier.scope == scope,
2075
+ models.DataIdentifier.name == name)
2076
+ )
2077
+ did = session.execute(stmt).scalar_one()
2078
+ except NoResultFound as exc:
2079
+ raise DataIdentifierNotFound() from exc
1634
2080
 
1635
2081
  if rule_evaluation_action == DIDReEvaluation.ATTACH:
1636
2082
  __evaluate_did_attach(did, session=session)
@@ -1639,13 +2085,16 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
1639
2085
 
1640
2086
  # Update size and length of did
1641
2087
  if session.bind.dialect.name == 'oracle':
1642
- stmt = session.query(func.sum(models.DataIdentifierAssociation.bytes),
1643
- func.count(1)).\
1644
- with_hint(models.DataIdentifierAssociation,
1645
- "index(CONTENTS CONTENTS_PK)", 'oracle').\
1646
- filter(models.DataIdentifierAssociation.scope == scope,
1647
- models.DataIdentifierAssociation.name == name)
1648
- for bytes_, length in stmt:
2088
+ stmt = select(
2089
+ func.sum(models.DataIdentifierAssociation.bytes),
2090
+ func.count(1)
2091
+ ).with_hint(
2092
+ models.DataIdentifierAssociation, 'INDEX(CONTENTS CONTENTS_PK)', 'oracle'
2093
+ ).where(
2094
+ and_(models.DataIdentifierAssociation.scope == scope,
2095
+ models.DataIdentifierAssociation.name == name)
2096
+ )
2097
+ for bytes_, length in session.execute(stmt):
1649
2098
  did.bytes = bytes_
1650
2099
  did.length = length
1651
2100
 
@@ -1657,7 +2106,14 @@ def re_evaluate_did(scope, name, rule_evaluation_action, *, session: "Session"):
1657
2106
 
1658
2107
 
1659
2108
  @read_session
1660
- def get_updated_dids(total_workers, worker_number, limit=100, blocked_dids=[], *, session: "Session"):
2109
+ def get_updated_dids(
2110
+ total_workers: int,
2111
+ worker_number: int,
2112
+ limit: int = 100,
2113
+ blocked_dids: Optional[Sequence[tuple[str, str]]] = None,
2114
+ *,
2115
+ session: "Session"
2116
+ ) -> list[tuple[str, InternalScope, str, DIDReEvaluation]]:
1661
2117
  """
1662
2118
  Get updated dids.
1663
2119
 
@@ -1667,21 +2123,23 @@ def get_updated_dids(total_workers, worker_number, limit=100, blocked_dids=[], *
1667
2123
  :param blocked_dids: Blocked dids to filter.
1668
2124
  :param session: Database session in use.
1669
2125
  """
1670
- query = session.query(models.UpdatedDID.id,
1671
- models.UpdatedDID.scope,
1672
- models.UpdatedDID.name,
1673
- models.UpdatedDID.rule_evaluation_action)
1674
-
1675
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
2126
+ blocked_dids = blocked_dids or []
2127
+ stmt = select(
2128
+ models.UpdatedDID.id,
2129
+ models.UpdatedDID.scope,
2130
+ models.UpdatedDID.name,
2131
+ models.UpdatedDID.rule_evaluation_action
2132
+ )
2133
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
1676
2134
 
1677
2135
  # Remove blocked dids from query, but only do the first 30 ones, not to overload the query
1678
2136
  if blocked_dids:
1679
2137
  chunk = list(chunks(blocked_dids, 30))[0]
1680
- query = query.filter(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
2138
+ stmt = stmt.where(tuple_(models.UpdatedDID.scope, models.UpdatedDID.name).notin_(chunk))
1681
2139
 
1682
2140
  if limit:
1683
- fetched_dids = query.order_by(models.UpdatedDID.created_at).limit(limit).all()
1684
- filtered_dids = [did for did in fetched_dids if (did.scope, did.name) not in blocked_dids]
2141
+ fetched_dids = session.execute(stmt.order_by(models.UpdatedDID.created_at).limit(limit)).all()
2142
+ filtered_dids = [did._tuple() for did in fetched_dids if (did.scope, did.name) not in blocked_dids]
1685
2143
  if len(fetched_dids) == limit and not filtered_dids:
1686
2144
  return get_updated_dids(total_workers=total_workers,
1687
2145
  worker_number=worker_number,
@@ -1691,11 +2149,23 @@ def get_updated_dids(total_workers, worker_number, limit=100, blocked_dids=[], *
1691
2149
  else:
1692
2150
  return filtered_dids
1693
2151
  else:
1694
- return [did for did in query.order_by(models.UpdatedDID.created_at).all() if (did.scope, did.name) not in blocked_dids]
2152
+ return [did._tuple() for did in session.execute(stmt.order_by(models.UpdatedDID.created_at)).all() if (did.scope, did.name) not in blocked_dids]
1695
2153
 
1696
2154
 
1697
2155
  @read_session
1698
- def get_rules_beyond_eol(date_check, worker_number, total_workers, *, session: "Session"):
2156
+ def get_rules_beyond_eol(
2157
+ date_check: datetime,
2158
+ worker_number: int,
2159
+ total_workers: int, *,
2160
+ session: "Session"
2161
+ ) -> list[tuple[InternalScope,
2162
+ str,
2163
+ str,
2164
+ bool,
2165
+ str,
2166
+ Optional[datetime],
2167
+ Optional[datetime],
2168
+ InternalAccount]]:
1699
2169
  """
1700
2170
  Get rules which have eol_at before a certain date.
1701
2171
 
@@ -1704,22 +2174,32 @@ def get_rules_beyond_eol(date_check, worker_number, total_workers, *, session: "
1704
2174
  :param total_workers: Number of total workers.
1705
2175
  :param session: Database session in use.
1706
2176
  """
1707
- query = session.query(models.ReplicationRule.scope,
1708
- models.ReplicationRule.name,
1709
- models.ReplicationRule.rse_expression,
1710
- models.ReplicationRule.locked,
1711
- models.ReplicationRule.id,
1712
- models.ReplicationRule.eol_at,
1713
- models.ReplicationRule.expires_at,
1714
- models.ReplicationRule.account).\
1715
- filter(models.ReplicationRule.eol_at < date_check)
2177
+ stmt = select(
2178
+ models.ReplicationRule.scope,
2179
+ models.ReplicationRule.name,
2180
+ models.ReplicationRule.rse_expression,
2181
+ models.ReplicationRule.locked,
2182
+ models.ReplicationRule.id,
2183
+ models.ReplicationRule.eol_at,
2184
+ models.ReplicationRule.expires_at,
2185
+ models.ReplicationRule.account
2186
+ ).where(
2187
+ models.ReplicationRule.eol_at < date_check
2188
+ )
1716
2189
 
1717
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
1718
- return [rule for rule in query.all()]
2190
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
2191
+ return [row._tuple() for row in session.execute(stmt).all()]
1719
2192
 
1720
2193
 
1721
2194
  @read_session
1722
- def get_expired_rules(total_workers, worker_number, limit=100, blocked_rules=[], *, session: "Session"):
2195
+ def get_expired_rules(
2196
+ total_workers: int,
2197
+ worker_number: int,
2198
+ limit: int = 100,
2199
+ blocked_rules: Optional[Sequence[str]] = None,
2200
+ *,
2201
+ session: "Session"
2202
+ ) -> list[tuple[str, str]]:
1723
2203
  """
1724
2204
  Get expired rules.
1725
2205
 
@@ -1730,18 +2210,26 @@ def get_expired_rules(total_workers, worker_number, limit=100, blocked_rules=[],
1730
2210
  :param session: Database session in use.
1731
2211
  """
1732
2212
 
1733
- query = session.query(models.ReplicationRule.id, models.ReplicationRule.rse_expression).filter(models.ReplicationRule.expires_at < datetime.utcnow(),
1734
- models.ReplicationRule.locked == false(),
1735
- models.ReplicationRule.child_rule_id == None).\
1736
- with_hint(models.ReplicationRule, "index(rules RULES_EXPIRES_AT_IDX)", 'oracle').\
1737
- order_by(models.ReplicationRule.expires_at) # NOQA
1738
-
1739
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
2213
+ blocked_rules = blocked_rules or []
2214
+ stmt = select(
2215
+ models.ReplicationRule.id,
2216
+ models.ReplicationRule.rse_expression
2217
+ ).with_hint(
2218
+ models.ReplicationRule, 'INDEX(RULES RULES_EXPIRES_AT_IDX)', 'oracle'
2219
+ ).where(
2220
+ and_(models.ReplicationRule.expires_at < datetime.utcnow(),
2221
+ models.ReplicationRule.locked == false(),
2222
+ models.ReplicationRule.child_rule_id == null())
2223
+ ).order_by(
2224
+ models.ReplicationRule.expires_at
2225
+ )
2226
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
1740
2227
 
1741
2228
  if limit:
1742
- fetched_rules = query.limit(limit).all()
1743
- filtered_rules = [rule for rule in fetched_rules if rule[0] not in blocked_rules]
1744
- if len(fetched_rules) == limit and not filtered_rules:
2229
+ stmt = stmt.limit(limit)
2230
+ result = session.execute(stmt).all()
2231
+ filtered_rules = [rule._tuple() for rule in result if rule.id not in blocked_rules]
2232
+ if len(result) == limit and not filtered_rules:
1745
2233
  return get_expired_rules(total_workers=total_workers,
1746
2234
  worker_number=worker_number,
1747
2235
  limit=None,
@@ -1750,11 +2238,18 @@ def get_expired_rules(total_workers, worker_number, limit=100, blocked_rules=[],
1750
2238
  else:
1751
2239
  return filtered_rules
1752
2240
  else:
1753
- return [rule for rule in query.all() if rule[0] not in blocked_rules]
2241
+ return [rule._tuple() for rule in session.execute(stmt).all() if rule.id not in blocked_rules]
1754
2242
 
1755
2243
 
1756
2244
  @read_session
1757
- def get_injected_rules(total_workers, worker_number, limit=100, blocked_rules=[], *, session: "Session"):
2245
+ def get_injected_rules(
2246
+ total_workers: int,
2247
+ worker_number: int,
2248
+ limit: int = 100,
2249
+ blocked_rules: Optional[Sequence[str]] = None,
2250
+ *,
2251
+ session: "Session"
2252
+ ) -> list[str]:
1758
2253
  """
1759
2254
  Get rules to be injected.
1760
2255
 
@@ -1765,18 +2260,24 @@ def get_injected_rules(total_workers, worker_number, limit=100, blocked_rules=[]
1765
2260
  :param session: Database session in use.
1766
2261
  """
1767
2262
 
1768
- query = session.query(models.ReplicationRule.id).\
1769
- with_hint(models.ReplicationRule, "index(rules RULES_STATE_IDX)", 'oracle').\
1770
- filter(models.ReplicationRule.state == RuleState.INJECT).\
1771
- order_by(models.ReplicationRule.created_at).\
1772
- filter(models.ReplicationRule.created_at <= datetime.utcnow())
1773
-
1774
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
2263
+ blocked_rules = blocked_rules or []
2264
+ stmt = select(
2265
+ models.ReplicationRule.id
2266
+ ).with_hint(
2267
+ models.ReplicationRule, 'INDEX(RULES RULES_STATE_IDX)', 'oracle'
2268
+ ).where(
2269
+ and_(models.ReplicationRule.state == RuleState.INJECT,
2270
+ models.ReplicationRule.created_at <= datetime.utcnow())
2271
+ ).order_by(
2272
+ models.ReplicationRule.created_at
2273
+ )
2274
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
1775
2275
 
1776
2276
  if limit:
1777
- fetched_rules = query.limit(limit).all()
1778
- filtered_rules = [rule for rule in fetched_rules if rule[0] not in blocked_rules]
1779
- if len(fetched_rules) == limit and not filtered_rules:
2277
+ stmt = stmt.limit(limit)
2278
+ result = session.execute(stmt).scalars().all()
2279
+ filtered_rules = [rule for rule in result if rule not in blocked_rules]
2280
+ if len(result) == limit and not filtered_rules:
1780
2281
  return get_injected_rules(total_workers=total_workers,
1781
2282
  worker_number=worker_number,
1782
2283
  limit=None,
@@ -1785,11 +2286,19 @@ def get_injected_rules(total_workers, worker_number, limit=100, blocked_rules=[]
1785
2286
  else:
1786
2287
  return filtered_rules
1787
2288
  else:
1788
- return [rule for rule in query.all() if rule[0] not in blocked_rules]
2289
+ return [rule for rule in session.execute(stmt).scalars().all() if rule not in blocked_rules]
1789
2290
 
1790
2291
 
1791
2292
  @read_session
1792
- def get_stuck_rules(total_workers, worker_number, delta=600, limit=10, blocked_rules=[], *, session: "Session"):
2293
+ def get_stuck_rules(
2294
+ total_workers: int,
2295
+ worker_number: int,
2296
+ delta: int = 600,
2297
+ limit: int = 10,
2298
+ blocked_rules: Optional[Sequence[str]] = None,
2299
+ *,
2300
+ session: "Session"
2301
+ ) -> list[str]:
1793
2302
  """
1794
2303
  Get stuck rules.
1795
2304
 
@@ -1800,21 +2309,27 @@ def get_stuck_rules(total_workers, worker_number, delta=600, limit=10, blocked_r
1800
2309
  :param blocked_rules: Blocked rules to filter out.
1801
2310
  :param session: Database session in use.
1802
2311
  """
1803
- query = session.query(models.ReplicationRule.id).\
1804
- with_hint(models.ReplicationRule, "index(rules RULES_STATE_IDX)", 'oracle').\
1805
- filter(models.ReplicationRule.state == RuleState.STUCK).\
1806
- filter(models.ReplicationRule.updated_at < datetime.utcnow() - timedelta(seconds=delta)).\
1807
- filter(or_(models.ReplicationRule.expires_at == null(),
1808
- models.ReplicationRule.expires_at > datetime.utcnow(),
1809
- models.ReplicationRule.locked == true())).\
1810
- order_by(models.ReplicationRule.updated_at)
1811
-
1812
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
2312
+ blocked_rules = blocked_rules or []
2313
+ stmt = select(
2314
+ models.ReplicationRule.id
2315
+ ).with_hint(
2316
+ models.ReplicationRule, 'INDEX(RULES RULES_STATE_IDX)', 'oracle'
2317
+ ).where(
2318
+ and_(models.ReplicationRule.state == RuleState.STUCK,
2319
+ models.ReplicationRule.updated_at < datetime.utcnow() - timedelta(seconds=delta),
2320
+ or_(models.ReplicationRule.expires_at == null(),
2321
+ models.ReplicationRule.expires_at > datetime.utcnow(),
2322
+ models.ReplicationRule.locked == true()))
2323
+ ).order_by(
2324
+ models.ReplicationRule.updated_at
2325
+ )
2326
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
1813
2327
 
1814
2328
  if limit:
1815
- fetched_rules = query.limit(limit).all()
1816
- filtered_rules = [rule for rule in fetched_rules if rule[0] not in blocked_rules]
1817
- if len(fetched_rules) == limit and not filtered_rules:
2329
+ stmt = stmt.limit(limit)
2330
+ result = session.execute(stmt).scalars().all()
2331
+ filtered_rules = [rule for rule in result if rule not in blocked_rules]
2332
+ if len(result) == limit and not filtered_rules:
1818
2333
  return get_stuck_rules(total_workers=total_workers,
1819
2334
  worker_number=worker_number,
1820
2335
  delta=delta,
@@ -1824,22 +2339,39 @@ def get_stuck_rules(total_workers, worker_number, delta=600, limit=10, blocked_r
1824
2339
  else:
1825
2340
  return filtered_rules
1826
2341
  else:
1827
- return [rule for rule in query.all() if rule[0] not in blocked_rules]
2342
+ return [rule for rule in session.execute(stmt).scalars().all() if rule not in blocked_rules]
1828
2343
 
1829
2344
 
1830
2345
  @transactional_session
1831
- def delete_updated_did(id_, *, session: "Session"):
2346
+ def delete_updated_did(
2347
+ id_: str,
2348
+ *,
2349
+ session: "Session"
2350
+ ) -> None:
1832
2351
  """
1833
2352
  Delete an updated_did by id.
1834
2353
 
1835
2354
  :param id_: Id of the row not to delete.
1836
2355
  :param session: The database session in use.
1837
2356
  """
1838
- session.query(models.UpdatedDID).filter(models.UpdatedDID.id == id_).delete()
2357
+ stmt = delete(
2358
+ models.UpdatedDID
2359
+ ).where(
2360
+ models.UpdatedDID.id == id_
2361
+ )
2362
+ session.execute(stmt)
1839
2363
 
1840
2364
 
1841
2365
  @transactional_session
1842
- def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session: "Session", logger=logging.log):
2366
+ def update_rules_for_lost_replica(
2367
+ scope: InternalScope,
2368
+ name: str,
2369
+ rse_id: str,
2370
+ nowait: bool = False,
2371
+ *,
2372
+ session: "Session",
2373
+ logger: LoggerFunction = logging.log
2374
+ ) -> None:
1843
2375
  """
1844
2376
  Update rules if a file replica is lost.
1845
2377
 
@@ -1851,9 +2383,38 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
1851
2383
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
1852
2384
  """
1853
2385
 
1854
- locks = session.query(models.ReplicaLock).filter(models.ReplicaLock.scope == scope, models.ReplicaLock.name == name, models.ReplicaLock.rse_id == rse_id).with_for_update(nowait=nowait).all()
1855
- replica = session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id == rse_id).with_for_update(nowait=nowait).one()
1856
- requests = session.query(models.Request).filter(models.Request.scope == scope, models.Request.name == name, models.Request.dest_rse_id == rse_id).with_for_update(nowait=nowait).all()
2386
+ stmt = select(
2387
+ models.ReplicaLock
2388
+ ).where(
2389
+ and_(models.ReplicaLock.scope == scope,
2390
+ models.ReplicaLock.name == name,
2391
+ models.ReplicaLock.rse_id == rse_id)
2392
+ ).with_for_update(
2393
+ nowait=nowait
2394
+ )
2395
+ locks = session.execute(stmt).scalars().all()
2396
+
2397
+ stmt = select(
2398
+ models.RSEFileAssociation
2399
+ ).where(
2400
+ and_(models.RSEFileAssociation.scope == scope,
2401
+ models.RSEFileAssociation.name == name,
2402
+ models.RSEFileAssociation.rse_id == rse_id)
2403
+ ).with_for_update(
2404
+ nowait=nowait
2405
+ )
2406
+ replica = session.execute(stmt).scalar_one()
2407
+
2408
+ stmt = select(
2409
+ models.Request
2410
+ ).where(
2411
+ and_(models.Request.scope == scope,
2412
+ models.Request.name == name,
2413
+ models.Request.dest_rse_id == rse_id)
2414
+ ).with_for_update(
2415
+ nowait=nowait
2416
+ )
2417
+ requests = session.execute(stmt).scalars().all()
1857
2418
 
1858
2419
  rse = get_rse_name(rse_id, session=session)
1859
2420
 
@@ -1867,7 +2428,14 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
1867
2428
  session.delete(request)
1868
2429
 
1869
2430
  for lock in locks:
1870
- rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == lock.rule_id).with_for_update(nowait=nowait).one()
2431
+ stmt = select(
2432
+ models.ReplicationRule
2433
+ ).where(
2434
+ models.ReplicationRule.id == lock.rule_id
2435
+ ).with_for_update(
2436
+ nowait=nowait
2437
+ )
2438
+ rule = session.execute(stmt).scalar_one()
1871
2439
  rule_state_before = rule.state
1872
2440
  replica.lock_cnt -= 1
1873
2441
  if lock.state == LockState.OK:
@@ -1884,7 +2452,14 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
1884
2452
  elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
1885
2453
  rule.state = RuleState.OK
1886
2454
  if rule.grouping != RuleGrouping.NONE:
1887
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.OK})
2455
+ stmt = update(
2456
+ models.DatasetLock
2457
+ ).where(
2458
+ models.DatasetLock.rule_id == rule.id
2459
+ ).values({
2460
+ models.DatasetLock.state: LockState.OK
2461
+ })
2462
+ session.execute(stmt)
1888
2463
  session.flush()
1889
2464
  if rule_state_before != RuleState.OK:
1890
2465
  generate_rule_notifications(rule=rule, session=session)
@@ -1902,8 +2477,28 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
1902
2477
 
1903
2478
  replica.tombstone = OBSOLETE
1904
2479
  replica.state = ReplicaState.UNAVAILABLE
1905
- session.query(models.DataIdentifier).filter_by(scope=scope, name=name).update({'availability': DIDAvailability.LOST})
1906
- session.query(models.BadReplicas).filter_by(state=BadFilesStatus.BAD, rse_id=rse_id, scope=scope, name=name).update({'state': BadFilesStatus.LOST, 'updated_at': datetime.utcnow()})
2480
+ stmt = update(
2481
+ models.DataIdentifier
2482
+ ).where(
2483
+ and_(models.DataIdentifier.scope == scope,
2484
+ models.DataIdentifier.name == name)
2485
+ ).values({
2486
+ models.DataIdentifier.availability: DIDAvailability.LOST
2487
+ })
2488
+ session.execute(stmt)
2489
+
2490
+ stmt = update(
2491
+ models.BadReplica
2492
+ ).where(
2493
+ and_(models.BadReplica.scope == scope,
2494
+ models.BadReplica.name == name,
2495
+ models.BadReplica.rse_id == rse_id,
2496
+ models.BadReplica.state == BadFilesStatus.BAD)
2497
+ ).values({
2498
+ models.BadReplica.state: BadFilesStatus.LOST,
2499
+ models.BadReplica.updated_at: datetime.utcnow()
2500
+ })
2501
+ session.execute(stmt)
1907
2502
  for dts in datasets:
1908
2503
  logger(logging.INFO, 'File %s:%s bad at site %s is completely lost from dataset %s:%s. Will be marked as LOST and detached', scope, name, rse, dts['scope'], dts['name'])
1909
2504
  rucio.core.did.detach_dids(scope=dts['scope'], name=dts['name'], dids=[{'scope': scope, 'name': name}], session=session)
@@ -1919,7 +2514,15 @@ def update_rules_for_lost_replica(scope, name, rse_id, nowait=False, *, session:
1919
2514
 
1920
2515
 
1921
2516
  @transactional_session
1922
- def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, *, session: "Session", logger=logging.log):
2517
+ def update_rules_for_bad_replica(
2518
+ scope: InternalScope,
2519
+ name: str,
2520
+ rse_id: str,
2521
+ nowait: bool = False,
2522
+ *,
2523
+ session: "Session",
2524
+ logger: LoggerFunction = logging.log
2525
+ ) -> None:
1923
2526
  """
1924
2527
  Update rules if a file replica is bad and has to be recreated.
1925
2528
 
@@ -1930,15 +2533,40 @@ def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, *, session:
1930
2533
  :param session: The database session in use.
1931
2534
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
1932
2535
  """
2536
+ stmt = select(
2537
+ models.ReplicaLock
2538
+ ).where(
2539
+ and_(models.ReplicaLock.scope == scope,
2540
+ models.ReplicaLock.name == name,
2541
+ models.ReplicaLock.rse_id == rse_id)
2542
+ ).with_for_update(
2543
+ nowait=nowait
2544
+ )
2545
+ locks = session.execute(stmt).scalars().all()
1933
2546
 
1934
- locks = session.query(models.ReplicaLock).filter(models.ReplicaLock.scope == scope, models.ReplicaLock.name == name, models.ReplicaLock.rse_id == rse_id).with_for_update(nowait=nowait).all()
1935
- replica = session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id == rse_id).with_for_update(nowait=nowait).one()
2547
+ stmt = select(
2548
+ models.RSEFileAssociation
2549
+ ).where(
2550
+ and_(models.RSEFileAssociation.scope == scope,
2551
+ models.RSEFileAssociation.name == name,
2552
+ models.RSEFileAssociation.rse_id == rse_id)
2553
+ ).with_for_update(
2554
+ nowait=nowait
2555
+ )
2556
+ replica = session.execute(stmt).scalar_one()
1936
2557
 
1937
2558
  nlock = 0
1938
2559
  datasets = []
1939
2560
  for lock in locks:
1940
2561
  nlock += 1
1941
- rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == lock.rule_id).with_for_update(nowait=nowait).one()
2562
+ stmt = select(
2563
+ models.ReplicationRule
2564
+ ).where(
2565
+ models.ReplicationRule.id == lock.rule_id
2566
+ ).with_for_update(
2567
+ nowait=nowait
2568
+ )
2569
+ rule = session.execute(stmt).scalar_one()
1942
2570
  # If source replica expression exists, we remove it
1943
2571
  if rule.source_replica_expression:
1944
2572
  rule.source_replica_expression = None
@@ -1981,19 +2609,50 @@ def update_rules_for_bad_replica(scope, name, rse_id, nowait=False, *, session:
1981
2609
  else:
1982
2610
  rule.state = RuleState.REPLICATING
1983
2611
  if rule.grouping != RuleGrouping.NONE:
1984
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.REPLICATING})
2612
+ stmt = update(
2613
+ models.DatasetLock
2614
+ ).where(
2615
+ models.DatasetLock.rule_id == rule.id
2616
+ ).values({
2617
+ models.DatasetLock.state: LockState.REPLICATING
2618
+ })
2619
+ session.execute(stmt)
1985
2620
  # Insert rule history
1986
2621
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
1987
2622
  if nlock:
1988
- session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id == rse_id).update({'state': ReplicaState.COPYING})
2623
+ stmt = update(
2624
+ models.RSEFileAssociation
2625
+ ).where(
2626
+ and_(models.RSEFileAssociation.scope == scope,
2627
+ models.RSEFileAssociation.name == name,
2628
+ models.RSEFileAssociation.rse_id == rse_id)
2629
+ ).values({
2630
+ models.RSEFileAssociation.state: ReplicaState.COPYING
2631
+ })
2632
+ session.execute(stmt)
1989
2633
  else:
1990
2634
  logger(logging.INFO, 'File %s:%s at site %s has no locks. Will be deleted now.', scope, name, get_rse_name(rse_id=rse_id, session=session))
1991
2635
  tombstone = OBSOLETE
1992
- session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == scope, models.RSEFileAssociation.name == name, models.RSEFileAssociation.rse_id == rse_id).update({'state': ReplicaState.UNAVAILABLE, 'tombstone': tombstone})
2636
+ stmt = update(
2637
+ models.RSEFileAssociation
2638
+ ).where(
2639
+ and_(models.RSEFileAssociation.scope == scope,
2640
+ models.RSEFileAssociation.name == name,
2641
+ models.RSEFileAssociation.rse_id == rse_id)
2642
+ ).values({
2643
+ models.RSEFileAssociation.state: ReplicaState.UNAVAILABLE,
2644
+ models.RSEFileAssociation.tombstone: tombstone
2645
+ })
2646
+ session.execute(stmt)
1993
2647
 
1994
2648
 
1995
2649
  @transactional_session
1996
- def generate_rule_notifications(rule, replicating_locks_before=None, *, session: "Session"):
2650
+ def generate_rule_notifications(
2651
+ rule: models.ReplicationRule,
2652
+ replicating_locks_before: Optional[int] = None,
2653
+ *,
2654
+ session: "Session"
2655
+ ) -> None:
1997
2656
  """
1998
2657
  Generate (If necessary) a callback for a rule (DATASETLOCK_OK, RULE_OK, DATASETLOCK_PROGRESS)
1999
2658
 
@@ -2047,7 +2706,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
2047
2706
  if rule.grouping != RuleGrouping.NONE:
2048
2707
  # Only send DATASETLOCK_OK callbacks for ALL/DATASET grouped rules
2049
2708
  if rule.notification == RuleNotification.YES:
2050
- dataset_locks = session.query(models.DatasetLock).filter_by(rule_id=rule.id).all()
2709
+ stmt = select(
2710
+ models.DatasetLock
2711
+ ).where(
2712
+ models.DatasetLock.rule_id == rule.id
2713
+ )
2714
+ dataset_locks = session.execute(stmt).scalars().all()
2051
2715
  for dataset_lock in dataset_locks:
2052
2716
  payload = {'scope': dataset_lock.scope.external,
2053
2717
  'name': dataset_lock.name,
@@ -2060,7 +2724,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
2060
2724
  add_message(event_type='DATASETLOCK_OK', payload=payload, session=session)
2061
2725
 
2062
2726
  elif rule.notification == RuleNotification.CLOSE:
2063
- dataset_locks = session.query(models.DatasetLock).filter_by(rule_id=rule.id).all()
2727
+ stmt = select(
2728
+ models.DatasetLock
2729
+ ).where(
2730
+ models.DatasetLock.rule_id == rule.id
2731
+ )
2732
+ dataset_locks = session.execute(stmt).scalars().all()
2064
2733
  for dataset_lock in dataset_locks:
2065
2734
  try:
2066
2735
  did = rucio.core.did.get_did(scope=dataset_lock.scope, name=dataset_lock.name, session=session)
@@ -2082,7 +2751,7 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
2082
2751
  pass
2083
2752
 
2084
2753
  elif rule.state == RuleState.REPLICATING and rule.notification == RuleNotification.PROGRESS and replicating_locks_before:
2085
- # For RuleNotification PROGRESS rules, also notifiy when REPLICATING thresholds are passed
2754
+ # For RuleNotification PROGRESS rules, also notify when REPLICATING thresholds are passed
2086
2755
  if __progress_class(replicating_locks_before, total_locks) != __progress_class(rule.locks_replicating_cnt, total_locks):
2087
2756
  try:
2088
2757
  did = rucio.core.did.get_did(scope=rule.scope, name=rule.name, session=session)
@@ -2101,7 +2770,12 @@ def generate_rule_notifications(rule, replicating_locks_before=None, *, session:
2101
2770
 
2102
2771
 
2103
2772
  @transactional_session
2104
- def generate_email_for_rule_ok_notification(rule, *, session: "Session", logger=logging.log):
2773
+ def generate_email_for_rule_ok_notification(
2774
+ rule: models.ReplicationRule,
2775
+ *,
2776
+ session: "Session",
2777
+ logger: LoggerFunction = logging.log
2778
+ ) -> None:
2105
2779
  """
2106
2780
  Generate (If necessary) an eMail for a rule with notification mode Y.
2107
2781
 
@@ -2122,7 +2796,7 @@ def generate_email_for_rule_ok_notification(rule, *, session: "Session", logger=
2122
2796
  try:
2123
2797
  with open(template_path, 'r') as templatefile:
2124
2798
  template = Template(templatefile.read())
2125
- except IOError as ex:
2799
+ except OSError as ex:
2126
2800
  logger(logging.ERROR, "Couldn't open file '%s'", template_path, exc_info=ex)
2127
2801
  return
2128
2802
 
@@ -2147,12 +2821,18 @@ def generate_email_for_rule_ok_notification(rule, *, session: "Session", logger=
2147
2821
  add_message(event_type='email',
2148
2822
  payload={'body': email_body,
2149
2823
  'to': [email],
2150
- 'subject': '[RUCIO] Replication rule %s has been succesfully transferred' % (str(rule.id))},
2824
+ 'subject': '[RUCIO] Replication rule %s has been successfully transferred' % (str(rule.id))},
2151
2825
  session=session)
2152
2826
 
2153
2827
 
2154
2828
  @transactional_session
2155
- def insert_rule_history(rule, recent=True, longterm=False, *, session: "Session"):
2829
+ def insert_rule_history(
2830
+ rule: models.ReplicationRule,
2831
+ recent: bool = True,
2832
+ longterm: bool = False,
2833
+ *,
2834
+ session: "Session"
2835
+ ) -> None:
2156
2836
  """
2157
2837
  Insert rule history to recent/longterm history.
2158
2838
 
@@ -2182,7 +2862,13 @@ def insert_rule_history(rule, recent=True, longterm=False, *, session: "Session"
2182
2862
 
2183
2863
 
2184
2864
  @transactional_session
2185
- def approve_rule(rule_id, approver=None, notify_approvers=True, *, session: "Session"):
2865
+ def approve_rule(
2866
+ rule_id: str,
2867
+ approver: str = '',
2868
+ notify_approvers: bool = True,
2869
+ *,
2870
+ session: "Session"
2871
+ ) -> None:
2186
2872
  """
2187
2873
  Approve a specific replication rule.
2188
2874
 
@@ -2194,7 +2880,12 @@ def approve_rule(rule_id, approver=None, notify_approvers=True, *, session: "Ses
2194
2880
  """
2195
2881
 
2196
2882
  try:
2197
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
2883
+ stmt = select(
2884
+ models.ReplicationRule
2885
+ ).where(
2886
+ models.ReplicationRule.id == rule_id
2887
+ )
2888
+ rule = session.execute(stmt).scalar_one()
2198
2889
  if rule.state == RuleState.WAITING_APPROVAL:
2199
2890
  rule.ignore_account_limit = True
2200
2891
  rule.state = RuleState.INJECT
@@ -2235,14 +2926,20 @@ def approve_rule(rule_id, approver=None, notify_approvers=True, *, session: "Ses
2235
2926
  'to': [recipient[0]],
2236
2927
  'subject': 'Re: [RUCIO] Request to approve replication rule %s' % (str(rule.id))},
2237
2928
  session=session)
2238
- except NoResultFound:
2239
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
2240
- except StatementError:
2241
- raise RucioException('Badly formatted rule id (%s)' % (rule_id))
2929
+ except NoResultFound as exc:
2930
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
2931
+ except StatementError as exc:
2932
+ raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
2242
2933
 
2243
2934
 
2244
2935
  @transactional_session
2245
- def deny_rule(rule_id, approver=None, reason=None, *, session: "Session"):
2936
+ def deny_rule(
2937
+ rule_id: str,
2938
+ approver: str = '',
2939
+ reason: Optional[str] = None,
2940
+ *,
2941
+ session: "Session"
2942
+ ) -> None:
2246
2943
  """
2247
2944
  Deny a specific replication rule.
2248
2945
 
@@ -2254,7 +2951,12 @@ def deny_rule(rule_id, approver=None, reason=None, *, session: "Session"):
2254
2951
  """
2255
2952
 
2256
2953
  try:
2257
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
2954
+ stmt = select(
2955
+ models.ReplicationRule
2956
+ ).where(
2957
+ models.ReplicationRule.id == rule_id
2958
+ )
2959
+ rule = session.execute(stmt).scalar_one()
2258
2960
  if rule.state == RuleState.WAITING_APPROVAL:
2259
2961
  with open('%s/rule_denied_user.tmpl' % config_get('common', 'mailtemplatedir'), 'r') as templatefile:
2260
2962
  template = Template(templatefile.read())
@@ -2294,26 +2996,35 @@ def deny_rule(rule_id, approver=None, reason=None, *, session: "Session"):
2294
2996
  'to': [recipient[0]],
2295
2997
  'subject': 'Re: [RUCIO] Request to approve replication rule %s' % (str(rule.id))},
2296
2998
  session=session)
2297
- except NoResultFound:
2298
- raise RuleNotFound('No rule with the id %s found' % rule_id)
2299
- except StatementError:
2300
- raise RucioException('Badly formatted rule id (%s)' % rule_id)
2999
+ except NoResultFound as exc:
3000
+ raise RuleNotFound('No rule with the id %s found' % rule_id) from exc
3001
+ except StatementError as exc:
3002
+ raise RucioException('Badly formatted rule id (%s)' % rule_id) from exc
2301
3003
 
2302
3004
 
2303
3005
  @transactional_session
2304
- def examine_rule(rule_id, *, session: "Session"):
3006
+ def examine_rule(
3007
+ rule_id: str,
3008
+ *,
3009
+ session: "Session"
3010
+ ) -> dict[str, Any]:
2305
3011
  """
2306
3012
  Examine a replication rule for transfer errors.
2307
3013
 
2308
3014
  :param rule_id: Replication rule id
2309
3015
  :param session: Session of the db.
2310
- :returns: Dictionary of informations
3016
+ :returns: Dictionary of information
2311
3017
  """
2312
3018
  result = {'rule_error': None,
2313
3019
  'transfers': []}
2314
3020
 
2315
3021
  try:
2316
- rule = session.query(models.ReplicationRule).filter_by(id=rule_id).one()
3022
+ stmt = select(
3023
+ models.ReplicationRule
3024
+ ).where(
3025
+ models.ReplicationRule.id == rule_id
3026
+ )
3027
+ rule = session.execute(stmt).scalar_one()
2317
3028
  if rule.state == RuleState.OK:
2318
3029
  result['rule_error'] = 'This replication rule is OK'
2319
3030
  elif rule.state == RuleState.REPLICATING:
@@ -2323,10 +3034,25 @@ def examine_rule(rule_id, *, session: "Session"):
2323
3034
  else:
2324
3035
  result['rule_error'] = rule.error
2325
3036
  # Get the stuck locks
2326
- stuck_locks = session.query(models.ReplicaLock).filter_by(rule_id=rule_id, state=LockState.STUCK).all()
3037
+ stmt = select(
3038
+ models.ReplicaLock
3039
+ ).where(
3040
+ and_(models.ReplicaLock.rule_id == rule_id,
3041
+ models.ReplicaLock.state == LockState.STUCK)
3042
+ )
3043
+ stuck_locks = session.execute(stmt).scalars().all()
2327
3044
  for lock in stuck_locks:
2328
3045
  # Get the count of requests in the request_history for each lock
2329
- transfers = session.query(models.RequestHistory).filter_by(scope=lock.scope, name=lock.name, dest_rse_id=lock.rse_id).order_by(models.RequestHistory.created_at.desc()).all() # pylint: disable=no-member
3046
+ stmt = select(
3047
+ models.RequestHistory
3048
+ ).where(
3049
+ and_(models.RequestHistory.scope == lock.scope,
3050
+ models.RequestHistory.name == lock.name,
3051
+ models.RequestHistory.dest_rse_id == lock.rse_id)
3052
+ ).order_by(
3053
+ desc(models.RequestHistory.created_at)
3054
+ )
3055
+ transfers = session.execute(stmt).scalars().all()
2330
3056
  transfer_cnt = len(transfers)
2331
3057
  # Get the error of the last request that has been tried and also the SOURCE used for the last request
2332
3058
  last_error, last_source, last_time, sources = None, None, None, []
@@ -2335,7 +3061,14 @@ def examine_rule(rule_id, *, session: "Session"):
2335
3061
  last_error = last_request.state
2336
3062
  last_time = last_request.created_at
2337
3063
  last_source = None if last_request.source_rse_id is None else get_rse_name(rse_id=last_request.source_rse_id, session=session)
2338
- available_replicas = session.query(models.RSEFileAssociation).filter_by(scope=lock.scope, name=lock.name, state=ReplicaState.AVAILABLE).all()
3064
+ stmt = select(
3065
+ models.RSEFileAssociation
3066
+ ).where(
3067
+ and_(models.RSEFileAssociation.scope == lock.scope,
3068
+ models.RSEFileAssociation.name == lock.name,
3069
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
3070
+ )
3071
+ available_replicas = session.execute(stmt).scalars().all()
2339
3072
 
2340
3073
  for replica in available_replicas:
2341
3074
  sources.append((get_rse_name(rse_id=replica.rse_id, session=session),
@@ -2351,14 +3084,18 @@ def examine_rule(rule_id, *, session: "Session"):
2351
3084
  'sources': sources,
2352
3085
  'last_time': last_time})
2353
3086
  return result
2354
- except NoResultFound:
2355
- raise RuleNotFound('No rule with the id %s found' % (rule_id))
2356
- except StatementError:
2357
- raise RucioException('Badly formatted rule id (%s)' % (rule_id))
3087
+ except NoResultFound as exc:
3088
+ raise RuleNotFound('No rule with the id %s found' % (rule_id)) from exc
3089
+ except StatementError as exc:
3090
+ raise RucioException('Badly formatted rule id (%s)' % (rule_id)) from exc
2358
3091
 
2359
3092
 
2360
3093
  @transactional_session
2361
- def get_evaluation_backlog(expiration_time=600, *, session: "Session"):
3094
+ def get_evaluation_backlog(
3095
+ expiration_time: int = 600,
3096
+ *,
3097
+ session: "Session"
3098
+ ) -> tuple[int, datetime]:
2362
3099
  """
2363
3100
  Counts the number of entries in the rule evaluation backlog.
2364
3101
  (Number of files to be evaluated)
@@ -2366,15 +3103,25 @@ def get_evaluation_backlog(expiration_time=600, *, session: "Session"):
2366
3103
  :returns: Tuple (Count, Datetime of oldest entry)
2367
3104
  """
2368
3105
 
2369
- result = REGION.get('rule_evaluation_backlog', expiration_time=expiration_time)
2370
- if result is NO_VALUE:
2371
- result = session.query(func.count(models.UpdatedDID.created_at), func.min(models.UpdatedDID.created_at)).one()
3106
+ cached_backlog: Union[NoValue, tuple[int, datetime]] = REGION.get('rule_evaluation_backlog', expiration_time=expiration_time)
3107
+ if isinstance(cached_backlog, NoValue):
3108
+ stmt = select(
3109
+ func.count(models.UpdatedDID.created_at),
3110
+ func.min(models.UpdatedDID.created_at)
3111
+ )
3112
+ result = session.execute(stmt).one()._tuple()
2372
3113
  REGION.set('rule_evaluation_backlog', result)
2373
- return result
3114
+ return result
3115
+ return cached_backlog
2374
3116
 
2375
3117
 
2376
3118
  @transactional_session
2377
- def release_parent_rule(child_rule_id, remove_parent_expiration=False, *, session: "Session"):
3119
+ def release_parent_rule(
3120
+ child_rule_id: str,
3121
+ remove_parent_expiration: bool = False,
3122
+ *,
3123
+ session: "Session"
3124
+ ) -> None:
2378
3125
  """
2379
3126
  Release a potential parent rule, because the child_rule is OK.
2380
3127
 
@@ -2385,8 +3132,14 @@ def release_parent_rule(child_rule_id, remove_parent_expiration=False, *, sessio
2385
3132
 
2386
3133
  session.flush()
2387
3134
 
2388
- parent_rules = session.query(models.ReplicationRule).filter_by(child_rule_id=child_rule_id).\
2389
- with_hint(models.ReplicationRule, "index(RULES RULES_CHILD_RULE_ID_IDX)", 'oracle').all()
3135
+ stmt = select(
3136
+ models.ReplicationRule
3137
+ ).with_hint(
3138
+ models.ReplicationRule, 'INDEX(RULES RULES_CHILD_RULE_ID_IDX)', 'oracle'
3139
+ ).where(
3140
+ models.ReplicationRule.child_rule_id == child_rule_id
3141
+ )
3142
+ parent_rules = session.execute(stmt).scalars().all()
2390
3143
  for rule in parent_rules:
2391
3144
  if remove_parent_expiration:
2392
3145
  rule.expires_at = None
@@ -2394,18 +3147,95 @@ def release_parent_rule(child_rule_id, remove_parent_expiration=False, *, sessio
2394
3147
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
2395
3148
 
2396
3149
 
3150
+ @stream_session
3151
+ def list_rules_for_rse_decommissioning(
3152
+ rse_id: str,
3153
+ *,
3154
+ session: "Session"
3155
+ ) -> Iterator[dict[str, Any]]:
3156
+ """Return a generator of rules at the RSE that is being decommissioned.
3157
+
3158
+ Decommissioning of an RSE involves deleting or moving away all rules that are
3159
+ locking the replicas that exist at the RSE. The rules can be enforcing
3160
+ dataset-level and/or file-level locks. Because rules are defined in terms of
3161
+ RSE expressions, we need to first identify the locks with the RSE id and make
3162
+ the list of rules that are enforcing such locks.
3163
+ This function has two yield statements corresponding to the two types
3164
+ (dataset-level and file-level) of locks. To avoid listing duplicates, the
3165
+ rules identified through the dataset-level locks are excluded from the
3166
+ second query using file-level locks.
3167
+
3168
+ :param rse_id: Id of the RSE being decommissioned.
3169
+ :param session: The database session in use.
3170
+ :returns: A generator that yields rule dictionaries.
3171
+ """
3172
+ # Get rules with dataset locks first.
3173
+ query_rules_from_dataset_locks = select(
3174
+ models.ReplicationRule
3175
+ ).distinct(
3176
+ ).join_from(
3177
+ models.DatasetLock,
3178
+ models.ReplicationRule,
3179
+ models.DatasetLock.rule_id == models.ReplicationRule.id
3180
+ ).where(
3181
+ models.DatasetLock.rse_id == rse_id
3182
+ )
3183
+
3184
+ for rule in session.execute(query_rules_from_dataset_locks).yield_per(5).scalars():
3185
+ yield rule.to_dict()
3186
+
3187
+ # Make a subquery from the previous query to be excluded from the next query
3188
+ dataset_rule_ids = query_rules_from_dataset_locks.with_only_columns(models.ReplicationRule.id)
3189
+
3190
+ # ReplicaLock ("locks") table is not indexed by RSE ID, so we instead go
3191
+ # through the RSEFileAssociation ("replicas") table.
3192
+ query_rules_from_replicas = select(
3193
+ models.ReplicationRule
3194
+ ).prefix_with(
3195
+ '/*+ USE_NL(locks) LEADING(replicas locks) */',
3196
+ dialect='oracle'
3197
+ ).distinct(
3198
+ ).join_from(
3199
+ models.RSEFileAssociation,
3200
+ models.ReplicaLock,
3201
+ and_(models.RSEFileAssociation.scope == models.ReplicaLock.scope,
3202
+ models.RSEFileAssociation.name == models.ReplicaLock.name,
3203
+ models.RSEFileAssociation.rse_id == models.ReplicaLock.rse_id)
3204
+ ).join(
3205
+ models.ReplicationRule,
3206
+ models.ReplicaLock.rule_id == models.ReplicationRule.id
3207
+ ).where(
3208
+ models.RSEFileAssociation.rse_id == rse_id,
3209
+ models.ReplicaLock.rule_id.not_in(dataset_rule_ids)
3210
+ )
3211
+
3212
+ for rule in session.execute(query_rules_from_replicas).yield_per(5).scalars():
3213
+ yield rule.to_dict()
3214
+
3215
+
2397
3216
  @transactional_session
2398
- def __find_missing_locks_and_create_them(datasetfiles, locks, replicas, source_replicas, rseselector, rule, source_rses, *, session: "Session", logger=logging.log):
3217
+ def __find_missing_locks_and_create_them(
3218
+ datasetfiles: Sequence[dict[str, Any]],
3219
+ locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
3220
+ replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
3221
+ source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
3222
+ rseselector: RSESelector,
3223
+ rule: models.ReplicationRule,
3224
+ source_rses: Sequence[str],
3225
+ *,
3226
+ session: "Session",
3227
+ logger: LoggerFunction = logging.log
3228
+ ) -> None:
2399
3229
  """
2400
3230
  Find missing locks for a rule and create them.
2401
3231
 
2402
- :param datasetfiles: Dict holding all datasets and files.
3232
+ :param datasetfiles: Sequence of dicts holding all datasets and files.
2403
3233
  :param locks: Dict holding locks.
2404
3234
  :param replicas: Dict holding replicas.
2405
3235
  :param source_replicas: Dict holding source replicas.
2406
3236
  :param rseselector: The RSESelector to be used.
2407
3237
  :param rule: The rule.
2408
- :param source_rses: RSE ids for eglible source RSEs.
3238
+ :param source_rses: RSE ids for eligible source RSEs.
2409
3239
  :param session: Session of the db.
2410
3240
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
2411
3241
  :raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
@@ -2442,17 +3272,20 @@ def __find_missing_locks_and_create_them(datasetfiles, locks, replicas, source_r
2442
3272
 
2443
3273
 
2444
3274
  @transactional_session
2445
- def __find_surplus_locks_and_remove_them(datasetfiles, locks, replicas, source_replicas, rseselector, rule, source_rses, *, session: "Session", logger=logging.log):
3275
+ def __find_surplus_locks_and_remove_them(
3276
+ datasetfiles: Sequence[dict[str, Any]],
3277
+ locks: dict[tuple[InternalScope, str], list[models.ReplicaLock]],
3278
+ rule: models.ReplicationRule,
3279
+ *,
3280
+ session: "Session",
3281
+ logger: LoggerFunction = logging.log
3282
+ ) -> None:
2446
3283
  """
2447
3284
  Find surplocks locks for a rule and delete them.
2448
3285
 
2449
3286
  :param datasetfiles: Dict holding all datasets and files.
2450
3287
  :param locks: Dict holding locks.
2451
- :param replicas: Dict holding replicas.
2452
- :param source_replicas: Dict holding all source replicas.
2453
- :param rseselector: The RSESelector to be used.
2454
3288
  :param rule: The rule.
2455
- :param source_rses: RSE ids for eglible source RSEs.
2456
3289
  :param session: Session of the db.
2457
3290
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
2458
3291
  :raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
@@ -2490,7 +3323,18 @@ def __find_surplus_locks_and_remove_them(datasetfiles, locks, replicas, source_r
2490
3323
 
2491
3324
 
2492
3325
  @transactional_session
2493
- def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_replicas, rseselector, rule, source_rses, *, session: "Session", logger=logging.log):
3326
+ def __find_stuck_locks_and_repair_them(
3327
+ datasetfiles: Sequence[dict[str, Any]],
3328
+ locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
3329
+ replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
3330
+ source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
3331
+ rseselector: RSESelector,
3332
+ rule: models.ReplicationRule,
3333
+ source_rses: Sequence[str],
3334
+ *,
3335
+ session: "Session",
3336
+ logger: LoggerFunction = logging.log
3337
+ ) -> None:
2494
3338
  """
2495
3339
  Find stuck locks for a rule and repair them.
2496
3340
 
@@ -2500,7 +3344,7 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
2500
3344
  :param source_replicas: Dict holding source replicas.
2501
3345
  :param rseselector: The RSESelector to be used.
2502
3346
  :param rule: The rule.
2503
- :param source_rses: RSE ids of eglible source RSEs.
3347
+ :param source_rses: RSE ids of eligible source RSEs.
2504
3348
  :param session: Session of the db.
2505
3349
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
2506
3350
  :raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs
@@ -2509,7 +3353,7 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
2509
3353
 
2510
3354
  logger(logging.DEBUG, "Finding and repairing stuck locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
2511
3355
 
2512
- replicas_to_create, locks_to_create, transfers_to_create,\
3356
+ replicas_to_create, locks_to_create, transfers_to_create, \
2513
3357
  locks_to_delete = repair_stuck_locks_and_apply_rule_grouping(datasetfiles=datasetfiles,
2514
3358
  locks=locks,
2515
3359
  replicas=replicas,
@@ -2549,7 +3393,12 @@ def __find_stuck_locks_and_repair_them(datasetfiles, locks, replicas, source_rep
2549
3393
 
2550
3394
 
2551
3395
  @transactional_session
2552
- def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
3396
+ def __evaluate_did_detach(
3397
+ eval_did: models.DataIdentifier,
3398
+ *,
3399
+ session: "Session",
3400
+ logger: LoggerFunction = logging.log
3401
+ ) -> None:
2553
3402
  """
2554
3403
  Evaluate a parent did which has children removed.
2555
3404
 
@@ -2566,9 +3415,25 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
2566
3415
  parent_dids = rucio.core.did.list_all_parent_dids(scope=eval_did.scope, name=eval_did.name, session=session)
2567
3416
 
2568
3417
  # Get all RR from parents and eval_did
2569
- rules = session.query(models.ReplicationRule).filter_by(scope=eval_did.scope, name=eval_did.name).with_for_update(nowait=True).all()
3418
+ stmt = select(
3419
+ models.ReplicationRule
3420
+ ).where(
3421
+ and_(models.ReplicationRule.scope == eval_did.scope,
3422
+ models.ReplicationRule.name == eval_did.name)
3423
+ ).with_for_update(
3424
+ nowait=True
3425
+ )
3426
+ rules = list(session.execute(stmt).scalars().all())
2570
3427
  for did in parent_dids:
2571
- rules.extend(session.query(models.ReplicationRule).filter_by(scope=did['scope'], name=did['name']).with_for_update(nowait=True).all())
3428
+ stmt = select(
3429
+ models.ReplicationRule
3430
+ ).where(
3431
+ and_(models.ReplicationRule.scope == did['scope'],
3432
+ models.ReplicationRule.name == did['name'])
3433
+ ).with_for_update(
3434
+ nowait=True
3435
+ )
3436
+ rules.extend(session.execute(stmt).scalars().all())
2572
3437
 
2573
3438
  # Iterate rules and delete locks
2574
3439
  transfers_to_delete = [] # [{'scope': , 'name':, 'rse_id':}]
@@ -2580,8 +3445,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
2580
3445
  files[(file['scope'], file['name'])] = True
2581
3446
  logger(logging.DEBUG, "Removing locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
2582
3447
  rule_locks_ok_cnt_before = rule.locks_ok_cnt
2583
- query = session.query(models.ReplicaLock).filter_by(rule_id=rule.id)
2584
- for lock in query:
3448
+ stmt = select(
3449
+ models.ReplicaLock
3450
+ ).where(
3451
+ models.ReplicaLock.rule_id == rule.id
3452
+ )
3453
+ for lock in session.execute(stmt).scalars().all():
2585
3454
  if (lock.scope, lock.name) not in files:
2586
3455
  if __delete_lock_and_update_replica(lock=lock, purge_replicas=force_epoch or rule.purge_replicas, nowait=True, session=session):
2587
3456
  transfers_to_delete.append({'scope': lock.scope, 'name': lock.name, 'rse_id': lock.rse_id})
@@ -2602,7 +3471,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
2602
3471
  for ds in rucio.core.did.list_child_datasets(scope=rule.scope, name=rule.name, session=session):
2603
3472
  child_datasets[(ds['scope'], ds['name'])] = True
2604
3473
  logger(logging.DEBUG, "Removing dataset_locks for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
2605
- query = session.query(models.DatasetLock).filter_by(rule_id=rule.id)
3474
+ stmt = select(
3475
+ models.DatasetLock
3476
+ ).where(
3477
+ models.DatasetLock.rule_id == rule.id
3478
+ )
3479
+ query = session.execute(stmt).scalars().all()
2606
3480
  for ds_lock in query:
2607
3481
  if (ds_lock.scope, ds_lock.name) not in child_datasets:
2608
3482
  ds_lock.delete(flush=False, session=session)
@@ -2615,7 +3489,14 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
2615
3489
  elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
2616
3490
  rule.state = RuleState.OK
2617
3491
  if rule.grouping != RuleGrouping.NONE:
2618
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.OK})
3492
+ stmt = update(
3493
+ models.DatasetLock
3494
+ ).where(
3495
+ models.DatasetLock.rule_id == rule.id
3496
+ ).values({
3497
+ models.DatasetLock.state: LockState.OK
3498
+ })
3499
+ session.execute(stmt)
2619
3500
  session.flush()
2620
3501
  if rule_locks_ok_cnt_before != rule.locks_ok_cnt:
2621
3502
  generate_rule_notifications(rule=rule, session=session)
@@ -2638,7 +3519,12 @@ def __evaluate_did_detach(eval_did, *, session: "Session", logger=logging.log):
2638
3519
 
2639
3520
 
2640
3521
  @transactional_session
2641
- def __oldest_file_under(scope, name, *, session: "Session"):
3522
+ def __oldest_file_under(
3523
+ scope: InternalScope,
3524
+ name: str,
3525
+ *,
3526
+ session: "Session"
3527
+ ) -> Optional[tuple[InternalScope, str]]:
2642
3528
  """
2643
3529
  Finds oldest file in oldest container/dataset in the container or the dataset, recursively.
2644
3530
  Oldest means attached to its parent first.
@@ -2647,10 +3533,15 @@ def __oldest_file_under(scope, name, *, session: "Session"):
2647
3533
  :param name: dataset or container name
2648
3534
  :returns: tuple (scope, name) or None
2649
3535
  """
2650
- children = session.query(models.DataIdentifierAssociation) \
2651
- .filter(models.DataIdentifierAssociation.scope == scope,
2652
- models.DataIdentifierAssociation.name == name) \
2653
- .order_by(models.DataIdentifierAssociation.created_at)
3536
+ stmt = select(
3537
+ models.DataIdentifierAssociation
3538
+ ).where(
3539
+ and_(models.DataIdentifierAssociation.scope == scope,
3540
+ models.DataIdentifierAssociation.name == name)
3541
+ ).order_by(
3542
+ models.DataIdentifierAssociation.created_at
3543
+ )
3544
+ children = session.execute(stmt).scalars().all()
2654
3545
  for child in children:
2655
3546
  if child.child_type == DIDType.FILE:
2656
3547
  return child.child_scope, child.child_name
@@ -2662,9 +3553,14 @@ def __oldest_file_under(scope, name, *, session: "Session"):
2662
3553
 
2663
3554
 
2664
3555
  @transactional_session
2665
- def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
3556
+ def __evaluate_did_attach(
3557
+ eval_did: models.DataIdentifier,
3558
+ *,
3559
+ session: "Session",
3560
+ logger: LoggerFunction = logging.log
3561
+ ) -> None:
2666
3562
  """
2667
- Evaluate a parent did which has new childs
3563
+ Evaluate a parent did which has new children
2668
3564
 
2669
3565
  :param eval_did: The did object in use.
2670
3566
  :param session: The database session in use.
@@ -2681,12 +3577,16 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2681
3577
 
2682
3578
  # Get immediate new child DID's
2683
3579
  with METRICS.timer('evaluate_did_attach.list_new_child_dids'):
2684
- new_child_dids = session.query(models.DataIdentifierAssociation)\
2685
- .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(contents contents_pk)", 'oracle')\
2686
- .filter(models.DataIdentifierAssociation.scope == eval_did.scope,
2687
- models.DataIdentifierAssociation.name == eval_did.name,
2688
- models.DataIdentifierAssociation.rule_evaluation == True).all() # noqa
2689
-
3580
+ stmt = select(
3581
+ models.DataIdentifierAssociation
3582
+ ).with_hint(
3583
+ models.DataIdentifierAssociation, 'INDEX_RS_ASC(CONTENTS CONTENTS_PK)', 'oracle'
3584
+ ).where(
3585
+ and_(models.DataIdentifierAssociation.scope == eval_did.scope,
3586
+ models.DataIdentifierAssociation.name == eval_did.name,
3587
+ models.DataIdentifierAssociation.rule_evaluation == true())
3588
+ )
3589
+ new_child_dids = session.execute(stmt).scalars().all()
2690
3590
  if new_child_dids:
2691
3591
  # Get all unsuspended RR from parents and eval_did
2692
3592
  with METRICS.timer('evaluate_did_attach.get_rules'):
@@ -2696,12 +3596,17 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2696
3596
  models.ReplicationRule.name == did['name']))
2697
3597
  rule_clauses.append(and_(models.ReplicationRule.scope == eval_did.scope,
2698
3598
  models.ReplicationRule.name == eval_did.name))
2699
- rules = session.query(models.ReplicationRule).filter(
2700
- or_(*rule_clauses),
2701
- models.ReplicationRule.state != RuleState.SUSPENDED,
2702
- models.ReplicationRule.state != RuleState.WAITING_APPROVAL,
2703
- models.ReplicationRule.state != RuleState.INJECT).with_for_update(nowait=True).all()
2704
-
3599
+ stmt = select(
3600
+ models.ReplicationRule
3601
+ ).where(
3602
+ and_(or_(*rule_clauses),
3603
+ models.ReplicationRule.state.not_in([RuleState.SUSPENDED,
3604
+ RuleState.WAITING_APPROVAL,
3605
+ RuleState.INJECT]))
3606
+ ).with_for_update(
3607
+ nowait=True
3608
+ )
3609
+ rules = session.execute(stmt).scalars().all()
2705
3610
  if rules:
2706
3611
  # Resolve the new_child_dids to its locks
2707
3612
  with METRICS.timer('evaluate_did_attach.resolve_did_to_locks_and_replicas'):
@@ -2754,7 +3659,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2754
3659
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
2755
3660
  # Try to update the DatasetLocks
2756
3661
  if rule.grouping != RuleGrouping.NONE:
2757
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
3662
+ stmt = update(
3663
+ models.DatasetLock
3664
+ ).where(
3665
+ models.DatasetLock.rule_id == rule.id
3666
+ ).values({
3667
+ models.DatasetLock.state: LockState.STUCK
3668
+ })
3669
+ session.execute(stmt)
2758
3670
  continue
2759
3671
 
2760
3672
  # 2. Create the RSE Selector
@@ -2773,7 +3685,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2773
3685
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
2774
3686
  # Try to update the DatasetLocks
2775
3687
  if rule.grouping != RuleGrouping.NONE:
2776
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
3688
+ stmt = update(
3689
+ models.DatasetLock
3690
+ ).where(
3691
+ models.DatasetLock.rule_id == rule.id
3692
+ ).values({
3693
+ models.DatasetLock.state: LockState.STUCK
3694
+ })
3695
+ session.execute(stmt)
2777
3696
  continue
2778
3697
 
2779
3698
  # 3. Apply the Replication rule to the Files
@@ -2816,7 +3735,14 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2816
3735
  insert_rule_history(rule=rule, recent=True, longterm=False, session=session)
2817
3736
  # Try to update the DatasetLocks
2818
3737
  if rule.grouping != RuleGrouping.NONE:
2819
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
3738
+ stmt = update(
3739
+ models.DatasetLock
3740
+ ).where(
3741
+ models.DatasetLock.rule_id == rule.id
3742
+ ).values({
3743
+ models.DatasetLock.state: LockState.STUCK
3744
+ })
3745
+ session.execute(stmt)
2820
3746
  continue
2821
3747
 
2822
3748
  # 4. Update the Rule State
@@ -2826,15 +3752,36 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2826
3752
  if locks_stuck_before != rule.locks_stuck_cnt:
2827
3753
  rule.state = RuleState.STUCK
2828
3754
  rule.error = 'MissingSourceReplica'
2829
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.STUCK})
3755
+ stmt = update(
3756
+ models.DatasetLock
3757
+ ).where(
3758
+ models.DatasetLock.rule_id == rule.id
3759
+ ).values({
3760
+ models.DatasetLock.state: LockState.STUCK
3761
+ })
3762
+ session.execute(stmt)
2830
3763
  elif rule.locks_replicating_cnt > 0:
2831
3764
  rule.state = RuleState.REPLICATING
2832
3765
  if rule.grouping != RuleGrouping.NONE:
2833
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.REPLICATING})
3766
+ stmt = update(
3767
+ models.DatasetLock
3768
+ ).where(
3769
+ models.DatasetLock.rule_id == rule.id
3770
+ ).values({
3771
+ models.DatasetLock.state: LockState.REPLICATING
3772
+ })
3773
+ session.execute(stmt)
2834
3774
  elif rule.locks_replicating_cnt == 0 and rule.locks_stuck_cnt == 0:
2835
3775
  rule.state = RuleState.OK
2836
3776
  if rule.grouping != RuleGrouping.NONE:
2837
- session.query(models.DatasetLock).filter_by(rule_id=rule.id).update({'state': LockState.OK})
3777
+ stmt = update(
3778
+ models.DatasetLock
3779
+ ).where(
3780
+ models.DatasetLock.rule_id == rule.id
3781
+ ).values({
3782
+ models.DatasetLock.state: LockState.OK
3783
+ })
3784
+ session.execute(stmt)
2838
3785
  session.flush()
2839
3786
  if rule_locks_ok_cnt_before < rule.locks_ok_cnt:
2840
3787
  generate_rule_notifications(rule=rule, session=session)
@@ -2852,9 +3799,20 @@ def __evaluate_did_attach(eval_did, *, session: "Session", logger=logging.log):
2852
3799
 
2853
3800
 
2854
3801
  @transactional_session
2855
- def __resolve_did_to_locks_and_replicas(did, nowait=False, restrict_rses=None, source_rses=None, only_stuck=False, *, session: "Session"):
3802
+ def __resolve_did_to_locks_and_replicas(
3803
+ did: models.DataIdentifier,
3804
+ nowait: bool = False,
3805
+ restrict_rses: Optional[Sequence[str]] = None,
3806
+ source_rses: Optional[Sequence[str]] = None,
3807
+ only_stuck: bool = False,
3808
+ *,
3809
+ session: "Session"
3810
+ ) -> tuple[list[dict[str, Any]],
3811
+ dict[tuple[str, str], models.ReplicaLock],
3812
+ dict[tuple[str, str], models.RSEFileAssociation],
3813
+ dict[tuple[str, str], str]]:
2856
3814
  """
2857
- Resolves a did to its constituent childs and reads the locks and replicas of all the constituent files.
3815
+ Resolves a did to its constituent children and reads the locks and replicas of all the constituent files.
2858
3816
 
2859
3817
  :param did: The db object of the did the rule is applied on.
2860
3818
  :param nowait: Nowait parameter for the FOR UPDATE statement.
@@ -2862,7 +3820,7 @@ def __resolve_did_to_locks_and_replicas(did, nowait=False, restrict_rses=None, s
2862
3820
  :param source_rses: Source rses for this rule. These replicas are not row-locked.
2863
3821
  :param only_stuck: Get results only for STUCK locks, if True.
2864
3822
  :param session: Session of the db.
2865
- :returns: (datasetfiles, locks, replicas)
3823
+ :returns: (datasetfiles, locks, replicas, source_replicas)
2866
3824
  """
2867
3825
 
2868
3826
  datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}]
@@ -2949,16 +3907,26 @@ def __resolve_did_to_locks_and_replicas(did, nowait=False, restrict_rses=None, s
2949
3907
 
2950
3908
 
2951
3909
  @transactional_session
2952
- def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], source_rses=None, *, session: "Session"):
3910
+ def __resolve_dids_to_locks_and_replicas(
3911
+ dids: Sequence[models.DataIdentifierAssociation],
3912
+ nowait: bool = False,
3913
+ restrict_rses: Optional[Sequence[str]] = None,
3914
+ source_rses: Optional[Sequence[str]] = None,
3915
+ *,
3916
+ session: "Session"
3917
+ ) -> tuple[list[dict[str, Any]],
3918
+ dict[tuple[str, str], models.ReplicaLock],
3919
+ dict[tuple[str, str], models.RSEFileAssociation],
3920
+ dict[tuple[str, str], str]]:
2953
3921
  """
2954
- Resolves a list of dids to its constituent childs and reads the locks and replicas of all the constituent files.
3922
+ Resolves a list of dids to its constituent children and reads the locks and replicas of all the constituent files.
2955
3923
 
2956
- :param dids: The list of DIDAssociation objects.
3924
+ :param dids: The list of DataIdentifierAssociation objects.
2957
3925
  :param nowait: Nowait parameter for the FOR UPDATE statement.
2958
3926
  :param restrict_rses: Possible rses of the rule, so only these replica/locks should be considered.
2959
3927
  :param source_rses: Source rses for this rule. These replicas are not row-locked.
2960
3928
  :param session: Session of the db.
2961
- :returns: (datasetfiles, locks, replicas)
3929
+ :returns: (datasetfiles, locks, replicas, source_replicas)
2962
3930
  """
2963
3931
 
2964
3932
  datasetfiles = [] # List of Datasets and their files in the Tree [{'scope':, 'name':, 'files': []}]
@@ -2966,6 +3934,7 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
2966
3934
  locks = {} # {(scope,name): [SQLAlchemy]}
2967
3935
  replicas = {} # {(scope, name): [SQLAlchemy]}
2968
3936
  source_replicas = {} # {(scope, name): [rse_id]
3937
+ restrict_rses = restrict_rses or []
2969
3938
 
2970
3939
  if dids[0].child_type == DIDType.FILE:
2971
3940
  # All the dids will be files!
@@ -3006,13 +3975,28 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
3006
3975
 
3007
3976
  for lock_clause_chunk in lock_clause_chunks:
3008
3977
  if locks_rse_clause:
3009
- tmp_locks = session.query(models.ReplicaLock).filter(or_(*lock_clause_chunk), or_(*locks_rse_clause))\
3010
- .with_hint(models.ReplicaLock, "index(LOCKS LOCKS_PK)", 'oracle')\
3011
- .with_for_update(nowait=nowait).all()
3978
+ stmt = select(
3979
+ models.ReplicaLock
3980
+ ).with_hint(
3981
+ models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
3982
+ ).where(
3983
+ and_(or_(*lock_clause_chunk),
3984
+ or_(*locks_rse_clause))
3985
+ ).with_for_update(
3986
+ nowait=nowait
3987
+ )
3988
+ tmp_locks = session.execute(stmt).scalars().all()
3012
3989
  else:
3013
- tmp_locks = session.query(models.ReplicaLock).filter(or_(*lock_clause_chunk))\
3014
- .with_hint(models.ReplicaLock, "index(LOCKS LOCKS_PK)", 'oracle')\
3015
- .with_for_update(nowait=nowait).all()
3990
+ stmt = select(
3991
+ models.ReplicaLock
3992
+ ).with_hint(
3993
+ models.ReplicaLock, 'INDEX(LOCKS LOCKS_PK)', 'oracle'
3994
+ ).where(
3995
+ or_(*lock_clause_chunk)
3996
+ ).with_for_update(
3997
+ nowait=nowait
3998
+ )
3999
+ tmp_locks = session.execute(stmt).scalars().all()
3016
4000
  for lock in tmp_locks:
3017
4001
  if (lock.scope, lock.name) not in locks:
3018
4002
  locks[(lock.scope, lock.name)] = [lock]
@@ -3021,13 +4005,30 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
3021
4005
 
3022
4006
  for replica_clause_chunk in replica_clause_chunks:
3023
4007
  if replicas_rse_clause:
3024
- tmp_replicas = session.query(models.RSEFileAssociation).filter(or_(*replica_clause_chunk), or_(*replicas_rse_clause), models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)\
3025
- .with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle')\
3026
- .with_for_update(nowait=nowait).all()
4008
+ stmt = select(
4009
+ models.RSEFileAssociation
4010
+ ).with_hint(
4011
+ models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
4012
+ ).where(
4013
+ and_(or_(*replica_clause_chunk),
4014
+ or_(*replicas_rse_clause),
4015
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
4016
+ ).with_for_update(
4017
+ nowait=nowait
4018
+ )
4019
+ tmp_replicas = session.execute(stmt).scalars().all()
3027
4020
  else:
3028
- tmp_replicas = session.query(models.RSEFileAssociation).filter(or_(*replica_clause_chunk), models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)\
3029
- .with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle')\
3030
- .with_for_update(nowait=nowait).all()
4021
+ stmt = select(
4022
+ models.RSEFileAssociation
4023
+ ).with_hint(
4024
+ models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
4025
+ ).where(
4026
+ and_(or_(*replica_clause_chunk),
4027
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
4028
+ ).with_for_update(
4029
+ nowait=nowait
4030
+ )
4031
+ tmp_replicas = session.execute(stmt).scalars().all()
3031
4032
  for replica in tmp_replicas:
3032
4033
  if (replica.scope, replica.name) not in replicas:
3033
4034
  replicas[(replica.scope, replica.name)] = [replica]
@@ -3036,9 +4037,18 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
3036
4037
 
3037
4038
  if source_rses:
3038
4039
  for replica_clause_chunk in replica_clause_chunks:
3039
- tmp_source_replicas = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id).\
3040
- filter(or_(*replica_clause_chunk), or_(*source_replicas_rse_clause), models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
3041
- .with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').all()
4040
+ stmt = select(
4041
+ models.RSEFileAssociation.scope,
4042
+ models.RSEFileAssociation.name,
4043
+ models.RSEFileAssociation.rse_id
4044
+ ).with_hint(
4045
+ models.RSEFileAssociation, 'INDEX(REPLICAS REPLICAS_PK)', 'oracle'
4046
+ ).where(
4047
+ and_(or_(*replica_clause_chunk),
4048
+ or_(*source_replicas_rse_clause),
4049
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
4050
+ )
4051
+ tmp_source_replicas = session.execute(stmt).all()
3042
4052
  for scope, name, rse_id in tmp_source_replicas:
3043
4053
  if (scope, name) not in source_replicas:
3044
4054
  source_replicas[(scope, name)] = [rse_id]
@@ -3047,7 +4057,13 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
3047
4057
  else:
3048
4058
  # The evaluate_dids will be containers and/or datasets
3049
4059
  for did in dids:
3050
- real_did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == did.child_scope, models.DataIdentifier.name == did.child_name).one()
4060
+ stmt = select(
4061
+ models.DataIdentifier
4062
+ ).where(
4063
+ and_(models.DataIdentifier.scope == did.child_scope,
4064
+ models.DataIdentifier.name == did.child_name)
4065
+ )
4066
+ real_did = session.execute(stmt).scalar_one()
3051
4067
  tmp_datasetfiles, tmp_locks, tmp_replicas, tmp_source_replicas = __resolve_did_to_locks_and_replicas(did=real_did,
3052
4068
  nowait=nowait,
3053
4069
  restrict_rses=restrict_rses,
@@ -3061,7 +4077,19 @@ def __resolve_dids_to_locks_and_replicas(dids, nowait=False, restrict_rses=[], s
3061
4077
 
3062
4078
 
3063
4079
  @transactional_session
3064
- def __create_locks_replicas_transfers(datasetfiles, locks, replicas, source_replicas, rseselector, rule, preferred_rse_ids=[], source_rses=[], *, session: "Session", logger=logging.log):
4080
+ def __create_locks_replicas_transfers(
4081
+ datasetfiles: Sequence[dict[str, Any]],
4082
+ locks: dict[tuple[InternalScope, str], Sequence[models.ReplicaLock]],
4083
+ replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
4084
+ source_replicas: dict[tuple[InternalScope, str], Sequence[models.CollectionReplica]],
4085
+ rseselector: RSESelector,
4086
+ rule: models.ReplicationRule,
4087
+ preferred_rse_ids: Optional[Sequence[str]] = None,
4088
+ source_rses: Optional[Sequence[str]] = None,
4089
+ *,
4090
+ session: "Session",
4091
+ logger: LoggerFunction = logging.log
4092
+ ) -> None:
3065
4093
  """
3066
4094
  Apply a created replication rule to a set of files
3067
4095
 
@@ -3072,13 +4100,15 @@ def __create_locks_replicas_transfers(datasetfiles, locks, replicas, source_repl
3072
4100
  :param rseselector: The RSESelector to be used.
3073
4101
  :param rule: The rule.
3074
4102
  :param preferred_rse_ids: Preferred RSE's to select.
3075
- :param source_rses: RSE ids of eglible source replicas.
4103
+ :param source_rses: RSE ids of eligible source replicas.
3076
4104
  :param session: Session of the db.
3077
4105
  :param logger: Optional decorated logger that can be passed from the calling daemons or servers.
3078
4106
  :raises: InsufficientAccountLimit, IntegrityError, InsufficientTargetRSEs, RSEOverQuota
3079
4107
  :attention: This method modifies the contents of the locks and replicas input parameters.
3080
4108
  """
3081
4109
 
4110
+ preferred_rse_ids = preferred_rse_ids or []
4111
+ source_rses = source_rses or []
3082
4112
  logger(logging.DEBUG, "Creating locks and replicas for rule %s [%d/%d/%d]", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt)
3083
4113
 
3084
4114
  replicas_to_create, locks_to_create, transfers_to_create = apply_rule_grouping(datasetfiles=datasetfiles,
@@ -3114,7 +4144,14 @@ def __create_locks_replicas_transfers(datasetfiles, locks, replicas, source_repl
3114
4144
 
3115
4145
 
3116
4146
  @transactional_session
3117
- def __delete_lock_and_update_replica(lock, purge_replicas=False, nowait=False, *, session: "Session", logger=logging.log):
4147
+ def __delete_lock_and_update_replica(
4148
+ lock: models.ReplicaLock,
4149
+ purge_replicas: bool = False,
4150
+ nowait: bool = False,
4151
+ *,
4152
+ session: "Session",
4153
+ logger: LoggerFunction = logging.log
4154
+ ) -> bool:
3118
4155
  """
3119
4156
  Delete a lock and update the associated replica.
3120
4157
 
@@ -3129,10 +4166,16 @@ def __delete_lock_and_update_replica(lock, purge_replicas=False, nowait=False, *
3129
4166
  logger(logging.DEBUG, "Deleting lock %s:%s for rule %s", lock.scope, lock.name, str(lock.rule_id))
3130
4167
  lock.delete(session=session, flush=False)
3131
4168
  try:
3132
- replica = session.query(models.RSEFileAssociation).filter(
3133
- models.RSEFileAssociation.scope == lock.scope,
3134
- models.RSEFileAssociation.name == lock.name,
3135
- models.RSEFileAssociation.rse_id == lock.rse_id).with_for_update(nowait=nowait).one()
4169
+ stmt = select(
4170
+ models.RSEFileAssociation
4171
+ ).where(
4172
+ and_(models.RSEFileAssociation.scope == lock.scope,
4173
+ models.RSEFileAssociation.name == lock.name,
4174
+ models.RSEFileAssociation.rse_id == lock.rse_id)
4175
+ ).with_for_update(
4176
+ nowait=nowait
4177
+ )
4178
+ replica = session.execute(stmt).scalar_one()
3136
4179
  replica.lock_cnt -= 1
3137
4180
  if replica.lock_cnt == 0:
3138
4181
  if purge_replicas:
@@ -3153,7 +4196,11 @@ def __delete_lock_and_update_replica(lock, purge_replicas=False, nowait=False, *
3153
4196
 
3154
4197
 
3155
4198
  @transactional_session
3156
- def __create_rule_approval_email(rule: "models.ReplicationRule", *, session: "Session"):
4199
+ def __create_rule_approval_email(
4200
+ rule: models.ReplicationRule,
4201
+ *,
4202
+ session: "Session"
4203
+ ) -> None:
3157
4204
  """
3158
4205
  Create the rule notification email.
3159
4206
 
@@ -3239,11 +4286,16 @@ def __create_rule_approval_email(rule: "models.ReplicationRule", *, session: "Se
3239
4286
 
3240
4287
 
3241
4288
  @transactional_session
3242
- def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Session"):
4289
+ def _create_recipients_list(
4290
+ rse_expression: str,
4291
+ filter_: Optional[str] = None,
4292
+ *,
4293
+ session: "Session"
4294
+ ) -> list[tuple[str, Union[str, InternalAccount]]]:
3243
4295
  """
3244
4296
  Create a list of recipients for a notification email based on rse_expression.
3245
4297
 
3246
- :param rse_exoression: The rse_expression.
4298
+ :param rse_expression: The rse_expression.
3247
4299
  :param session: The database session in use.
3248
4300
  """
3249
4301
 
@@ -3253,8 +4305,8 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
3253
4305
  # If there are accounts in the approvers-list of any of the RSEs only these should be used
3254
4306
  for rse in parse_expression(rse_expression, filter_=filter_, session=session):
3255
4307
  rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
3256
- if rse_attr.get('rule_approvers'):
3257
- for account in rse_attr.get('rule_approvers').split(','):
4308
+ if rse_attr.get(RseAttr.RULE_APPROVERS):
4309
+ for account in rse_attr.get(RseAttr.RULE_APPROVERS).split(','):
3258
4310
  account = InternalAccount(account)
3259
4311
  try:
3260
4312
  email = get_account(account=account, session=session).email
@@ -3267,12 +4319,12 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
3267
4319
  if not recipients:
3268
4320
  for rse in parse_expression(rse_expression, filter_=filter_, session=session):
3269
4321
  rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
3270
- if rse_attr.get('type', '') in ('LOCALGROUPDISK', 'LOCALGROUPTAPE'):
4322
+ if rse_attr.get(RseAttr.TYPE, '') in ('LOCALGROUPDISK', 'LOCALGROUPTAPE'):
3271
4323
 
3272
4324
  query = select(
3273
4325
  models.AccountAttrAssociation.account
3274
4326
  ).where(
3275
- models.AccountAttrAssociation.key == f'country-{rse_attr.get("country", "")}',
4327
+ models.AccountAttrAssociation.key == f'country-{rse_attr.get(RseAttr.COUNTRY, "")}',
3276
4328
  models.AccountAttrAssociation.value == 'admin'
3277
4329
  )
3278
4330
 
@@ -3288,12 +4340,12 @@ def _create_recipients_list(rse_expression: str, filter_=None, *, session: "Sess
3288
4340
  if not recipients:
3289
4341
  for rse in parse_expression(rse_expression, filter_=filter_, session=session):
3290
4342
  rse_attr = list_rse_attributes(rse_id=rse['id'], session=session)
3291
- if rse_attr.get('type', '') == 'GROUPDISK':
4343
+ if rse_attr.get(RseAttr.TYPE, '') == 'GROUPDISK':
3292
4344
 
3293
4345
  query = select(
3294
4346
  models.AccountAttrAssociation.account
3295
4347
  ).where(
3296
- models.AccountAttrAssociation.key == f'group-{rse_attr.get("physgroup", "")}',
4348
+ models.AccountAttrAssociation.key == f'group-{rse_attr.get(RseAttr.PHYSGROUP, "")}',
3297
4349
  models.AccountAttrAssociation.value == 'admin'
3298
4350
  )
3299
4351
 
@@ -3332,7 +4384,13 @@ def __progress_class(replicating_locks, total_locks):
3332
4384
 
3333
4385
  @policy_filter
3334
4386
  @transactional_session
3335
- def archive_localgroupdisk_datasets(scope, name, *, session: "Session", logger=logging.log):
4387
+ def archive_localgroupdisk_datasets(
4388
+ scope: InternalScope,
4389
+ name: str,
4390
+ *,
4391
+ session: "Session",
4392
+ logger: LoggerFunction = logging.log
4393
+ ) -> None:
3336
4394
  """
3337
4395
  ATLAS policy to archive a dataset which has a replica on LOCALGROUPDISK
3338
4396
 
@@ -3400,7 +4458,13 @@ def archive_localgroupdisk_datasets(scope, name, *, session: "Session", logger=l
3400
4458
 
3401
4459
  @policy_filter
3402
4460
  @read_session
3403
- def get_scratch_policy(account, rses, lifetime, *, session: "Session"):
4461
+ def get_scratch_policy(
4462
+ account: InternalAccount,
4463
+ rses: Sequence[dict[str, Any]],
4464
+ lifetime: Optional[int],
4465
+ *,
4466
+ session: "Session"
4467
+ ) -> Optional[int]:
3404
4468
  """
3405
4469
  ATLAS policy for rules on SCRATCHDISK
3406
4470
 
@@ -3414,6 +4478,6 @@ def get_scratch_policy(account, rses, lifetime, *, session: "Session"):
3414
4478
  # Check SCRATCHDISK Policy
3415
4479
  if not has_account_attribute(account=account, key='admin', session=session) and (lifetime is None or lifetime > 60 * 60 * 24 * scratchdisk_lifetime):
3416
4480
  # Check if one of the rses is a SCRATCHDISK:
3417
- if [rse for rse in rses if list_rse_attributes(rse_id=rse['id'], session=session).get('type') == 'SCRATCHDISK']:
4481
+ if [rse for rse in rses if list_rse_attributes(rse_id=rse['id'], session=session).get(RseAttr.TYPE) == 'SCRATCHDISK']:
3418
4482
  lifetime = 60 * 60 * 24 * scratchdisk_lifetime - 1
3419
4483
  return lifetime