rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
rucio/core/replica.py CHANGED
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +11,11 @@
12
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
14
+
15
15
  import copy
16
16
  import heapq
17
17
  import logging
18
+ import math
18
19
  import random
19
20
  from collections import defaultdict, namedtuple
20
21
  from curses.ascii import isprint
@@ -27,44 +28,43 @@ from struct import unpack
27
28
  from traceback import format_exc
28
29
  from typing import TYPE_CHECKING
29
30
 
30
- import math
31
31
  import requests
32
32
  from dogpile.cache.api import NO_VALUE
33
- from sqlalchemy import func, and_, or_, exists, not_, update, delete, insert, union
33
+ from sqlalchemy import and_, delete, exists, func, insert, not_, or_, union, update
34
34
  from sqlalchemy.exc import DatabaseError, IntegrityError
35
35
  from sqlalchemy.orm import aliased
36
36
  from sqlalchemy.orm.exc import FlushError, NoResultFound
37
- from sqlalchemy.sql import label
38
- from sqlalchemy.sql.expression import case, select, text, false, true, null, literal, literal_column
37
+ from sqlalchemy.sql.expression import case, false, literal, literal_column, null, select, text, true
39
38
 
40
39
  import rucio.core.did
41
40
  import rucio.core.lock
42
41
  from rucio.common import exception
43
42
  from rucio.common.cache import make_region_memcached
44
43
  from rucio.common.config import config_get, config_get_bool
45
- from rucio.common.constants import SuspiciousAvailability
44
+ from rucio.common.constants import RseAttr, SuspiciousAvailability
46
45
  from rucio.common.types import InternalScope
47
- from rucio.common.utils import chunks, clean_surls, str_to_date, add_url_query
46
+ from rucio.common.utils import add_url_query, chunks, clean_pfns, str_to_date
48
47
  from rucio.core.credential import get_signed_url
49
48
  from rucio.core.message import add_messages
50
49
  from rucio.core.monitor import MetricManager
51
- from rucio.core.rse import get_rse, get_rse_name, get_rse_attribute, get_rse_vo, list_rses
50
+ from rucio.core.rse import get_rse, get_rse_attribute, get_rse_name, get_rse_vo, list_rses
52
51
  from rucio.core.rse_counter import decrease, increase
53
52
  from rucio.core.rse_expression_parser import parse_expression
54
- from rucio.db.sqla import models, filter_thread_work
55
- from rucio.db.sqla.constants import (DIDType, ReplicaState, OBSOLETE, DIDAvailability,
56
- BadFilesStatus, RuleState, BadPFNStatus)
57
- from rucio.db.sqla.session import (read_session, stream_session, transactional_session,
58
- DEFAULT_SCHEMA_NAME, BASE)
53
+ from rucio.db.sqla import filter_thread_work, models
54
+ from rucio.db.sqla.constants import OBSOLETE, BadFilesStatus, BadPFNStatus, DIDAvailability, DIDType, ReplicaState, RuleState
55
+ from rucio.db.sqla.session import BASE, DEFAULT_SCHEMA_NAME, read_session, stream_session, transactional_session
59
56
  from rucio.db.sqla.util import temp_table_mngr
60
57
  from rucio.rse import rsemanager as rsemgr
61
58
 
62
59
  if TYPE_CHECKING:
63
- from collections.abc import Sequence
64
- from rucio.rse.protocols.protocol import RSEProtocol
60
+ from collections.abc import Iterable, Iterator, Sequence
65
61
  from typing import Any, Optional
62
+
66
63
  from sqlalchemy.orm import Session
67
64
 
65
+ from rucio.common.types import LoggerFunction
66
+ from rucio.rse.protocols.protocol import RSEProtocol
67
+
68
68
  REGION = make_region_memcached(expiration_time=60)
69
69
  METRICS = MetricManager(module=__name__)
70
70
 
@@ -88,33 +88,42 @@ def get_bad_replicas_summary(rse_expression=None, from_date=None, to_date=None,
88
88
  rse_clause = []
89
89
  if rse_expression:
90
90
  for rse in parse_expression(expression=rse_expression, filter_=filter_, session=session):
91
- rse_clause.append(models.BadReplicas.rse_id == rse['id'])
91
+ rse_clause.append(models.BadReplica.rse_id == rse['id'])
92
92
  elif filter_:
93
93
  # Ensure we limit results to current VO even if we don't specify an RSE expression
94
94
  for rse in list_rses(filters=filter_, session=session):
95
- rse_clause.append(models.BadReplicas.rse_id == rse['id'])
95
+ rse_clause.append(models.BadReplica.rse_id == rse['id'])
96
96
 
97
97
  if session.bind.dialect.name == 'oracle':
98
- to_days = func.trunc(models.BadReplicas.created_at, str('DD'))
98
+ to_days = func.trunc(models.BadReplica.created_at, 'DD')
99
99
  elif session.bind.dialect.name == 'mysql':
100
- to_days = func.date(models.BadReplicas.created_at)
100
+ to_days = func.date(models.BadReplica.created_at)
101
101
  elif session.bind.dialect.name == 'postgresql':
102
- to_days = func.date_trunc('day', models.BadReplicas.created_at)
102
+ to_days = func.date_trunc('day', models.BadReplica.created_at)
103
103
  else:
104
- to_days = func.strftime(models.BadReplicas.created_at, '%Y-%m-%d')
105
- query = session.query(func.count(), to_days, models.BadReplicas.rse_id, models.BadReplicas.state, models.BadReplicas.reason)
104
+ to_days = func.strftime(models.BadReplica.created_at, '%Y-%m-%d')
105
+
106
+ stmt = select(
107
+ func.count(),
108
+ to_days,
109
+ models.BadReplica.rse_id,
110
+ models.BadReplica.state,
111
+ models.BadReplica.reason
112
+ ).select_from(
113
+ models.BadReplica
114
+ )
106
115
  # To be added : HINTS
107
116
  if rse_clause != []:
108
- query = query.filter(or_(*rse_clause))
117
+ stmt = stmt.where(or_(*rse_clause))
109
118
  if from_date:
110
- query = query.filter(models.BadReplicas.created_at > from_date)
119
+ stmt = stmt.where(models.BadReplica.created_at > from_date)
111
120
  if to_date:
112
- query = query.filter(models.BadReplicas.created_at < to_date)
113
- summary = query.group_by(to_days, models.BadReplicas.rse_id, models.BadReplicas.reason, models.BadReplicas.state).all()
114
- for row in summary:
115
- if (row[2], row[1], row[4]) not in incidents:
116
- incidents[(row[2], row[1], row[4])] = {}
117
- incidents[(row[2], row[1], row[4])][str(row[3].name)] = row[0]
121
+ stmt = stmt.where(models.BadReplica.created_at < to_date)
122
+ stmt = stmt.group_by(to_days, models.BadReplica.rse_id, models.BadReplica.reason, models.BadReplica.state)
123
+ for count, to_days, rse_id, state, reason in session.execute(stmt):
124
+ if (rse_id, to_days, reason) not in incidents:
125
+ incidents[(rse_id, to_days, reason)] = {}
126
+ incidents[(rse_id, to_days, reason)][str(state.name)] = count
118
127
 
119
128
  for incident in incidents:
120
129
  res = incidents[incident]
@@ -160,27 +169,38 @@ def __exist_replicas(rse_id, replicas, *, session: "Session"):
160
169
  for clause in [path_clause, did_clause]:
161
170
  if clause:
162
171
  for chunk in chunks(clause, 10):
163
- query = session.query(models.RSEFileAssociation.path,
164
- models.RSEFileAssociation.scope,
165
- models.RSEFileAssociation.name,
166
- models.RSEFileAssociation.rse_id,
167
- models.RSEFileAssociation.bytes,
168
- func.max(case((models.BadReplicas.state == BadFilesStatus.SUSPICIOUS, 0),
169
- (models.BadReplicas.state == BadFilesStatus.BAD, 1),
170
- else_=0))).\
171
- with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_PATH_IDX", 'oracle').\
172
- outerjoin(models.BadReplicas,
173
- and_(models.RSEFileAssociation.scope == models.BadReplicas.scope,
174
- models.RSEFileAssociation.name == models.BadReplicas.name,
175
- models.RSEFileAssociation.rse_id == models.BadReplicas.rse_id)).\
176
- filter(models.RSEFileAssociation.rse_id == rse_id).filter(or_(*chunk)).\
177
- group_by(models.RSEFileAssociation.path,
178
- models.RSEFileAssociation.scope,
179
- models.RSEFileAssociation.name,
180
- models.RSEFileAssociation.rse_id,
181
- models.RSEFileAssociation.bytes)
182
-
183
- for path, scope, name, rse_id, size, state in query.all():
172
+ stmt = select(
173
+ models.RSEFileAssociation.path,
174
+ models.RSEFileAssociation.scope,
175
+ models.RSEFileAssociation.name,
176
+ models.RSEFileAssociation.rse_id,
177
+ models.RSEFileAssociation.bytes,
178
+ func.max(
179
+ case(
180
+ (models.BadReplica.state == BadFilesStatus.SUSPICIOUS, 0),
181
+ (models.BadReplica.state == BadFilesStatus.BAD, 1),
182
+ else_=0))
183
+ ).with_hint(
184
+ models.RSEFileAssociation,
185
+ 'INDEX(REPLICAS REPLICAS_PATH_IDX',
186
+ 'oracle'
187
+ ).outerjoin(
188
+ models.BadReplica,
189
+ and_(models.RSEFileAssociation.scope == models.BadReplica.scope,
190
+ models.RSEFileAssociation.name == models.BadReplica.name,
191
+ models.RSEFileAssociation.rse_id == models.BadReplica.rse_id)
192
+ ).where(
193
+ and_(models.RSEFileAssociation.rse_id == rse_id,
194
+ or_(*chunk))
195
+ ).group_by(
196
+ models.RSEFileAssociation.path,
197
+ models.RSEFileAssociation.scope,
198
+ models.RSEFileAssociation.name,
199
+ models.RSEFileAssociation.rse_id,
200
+ models.RSEFileAssociation.bytes
201
+ )
202
+
203
+ for path, scope, name, rse_id, size, state in session.execute(stmt).all():
184
204
  if (scope, name, path) in replicas:
185
205
  replicas.remove((scope, name, path))
186
206
  if (None, None, path) in replicas:
@@ -211,19 +231,26 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
211
231
  :param session: The database session in use.
212
232
  """
213
233
  result = []
214
- query = session.query(models.BadReplicas.scope, models.BadReplicas.name, models.BadReplicas.rse_id, models.BadReplicas.state, models.BadReplicas.created_at, models.BadReplicas.updated_at)
234
+ stmt = select(
235
+ models.BadReplica.scope,
236
+ models.BadReplica.name,
237
+ models.BadReplica.rse_id,
238
+ models.BadReplica.state,
239
+ models.BadReplica.created_at,
240
+ models.BadReplica.updated_at
241
+ )
215
242
  if state:
216
- query = query.filter(models.BadReplicas.state == state)
243
+ stmt = stmt.where(models.BadReplica.state == state)
217
244
  if rse_id:
218
- query = query.filter(models.BadReplicas.rse_id == rse_id)
245
+ stmt = stmt.where(models.BadReplica.rse_id == rse_id)
219
246
  if younger_than:
220
- query = query.filter(models.BadReplicas.created_at >= younger_than)
247
+ stmt = stmt.where(models.BadReplica.created_at >= younger_than)
221
248
  if older_than:
222
- query = query.filter(models.BadReplicas.created_at <= older_than)
249
+ stmt = stmt.where(models.BadReplica.created_at <= older_than)
223
250
  if limit:
224
- query = query.limit(limit)
251
+ stmt = stmt.limit(limit)
225
252
 
226
- for badfile in query.yield_per(1000):
253
+ for badfile in session.execute(stmt).yield_per(1000):
227
254
  if badfile.scope.vo == vo:
228
255
  if list_pfns:
229
256
  result.append({'scope': badfile.scope, 'name': badfile.name, 'type': DIDType.FILE})
@@ -245,7 +272,7 @@ def list_bad_replicas_status(state=BadFilesStatus.BAD, rse_id=None, younger_than
245
272
 
246
273
 
247
274
  @transactional_session
248
- def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesStatus.BAD, scheme='srm', force=False, *, session: "Session"):
275
+ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesStatus.BAD, scheme='srm', force=False, logger: "LoggerFunction" = logging.log, *, session: "Session"):
249
276
  """
250
277
  Declare a list of bad replicas.
251
278
 
@@ -267,25 +294,17 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
267
294
  rse_info = rsemgr.get_rse_info(rse_id=rse_id, session=session)
268
295
  proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
269
296
  if rse_info['deterministic']:
270
- # TBD : In case of deterministic RSE, call the extract_scope_from_path method
297
+ scope_proto = rsemgr.get_scope_protocol(vo=issuer.vo)
271
298
  parsed_pfn = proto.parse_pfns(pfns=pfns)
272
299
  for pfn in parsed_pfn:
273
- # WARNING : this part is ATLAS specific and must be changed
274
- path = parsed_pfn[pfn]['path']
275
- if path.startswith('/user') or path.startswith('/group'):
276
- scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
277
- name = parsed_pfn[pfn]['name']
278
- elif path.startswith('/'):
279
- scope = path.split('/')[1]
280
- name = parsed_pfn[pfn]['name']
281
- else:
282
- scope = path.split('/')[0]
283
- name = parsed_pfn[pfn]['name']
300
+ # Translate into a scope and name
301
+ name, scope = scope_proto(parsed_pfn[pfn])
284
302
 
285
303
  scope = InternalScope(scope, vo=issuer.vo)
286
304
  replicas.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': status})
287
305
  path = '%s%s' % (parsed_pfn[pfn]['path'], parsed_pfn[pfn]['name'])
288
306
  path_pfn_dict[path] = pfn
307
+ logger(logging.DEBUG, f"Declaring replica {scope}:{name} {status} at {rse_id} with path {path}")
289
308
 
290
309
  else:
291
310
  # For non-deterministic RSEs use the path + rse_id to extract the scope
@@ -295,10 +314,13 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
295
314
  replicas.append({'scope': None, 'name': None, 'rse_id': rse_id, 'path': path, 'state': status})
296
315
  path_pfn_dict[path] = pfn
297
316
 
317
+ logger(logging.DEBUG, f"Declaring replica with pfn: {pfn} {status} at {rse_id} with path {path}")
318
+
298
319
  else:
299
320
  # If pfns is a list of replicas, just use scope, name and rse_id
300
321
  for pfn in pfns:
301
322
  replicas.append({'scope': pfn['scope'], 'name': pfn['name'], 'rse_id': rse_id, 'state': status})
323
+ logger(logging.DEBUG, f"Declaring replica {pfn['scope']}:{pfn['name']} {status} at {rse_id} without path")
302
324
 
303
325
  replicas_list = []
304
326
  for replica in replicas:
@@ -308,7 +330,6 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
308
330
  bad_replicas_to_update = []
309
331
 
310
332
  for scope, name, path, __exists, already_declared, size in __exist_replicas(rse_id=rse_id, replicas=replicas_list, session=session):
311
-
312
333
  declared = False
313
334
 
314
335
  if __exists:
@@ -318,7 +339,7 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
318
339
  declared = True
319
340
 
320
341
  if status == BadFilesStatus.SUSPICIOUS or status == BadFilesStatus.BAD and not already_declared:
321
- new_bad_replica = models.BadReplicas(scope=scope, name=name, rse_id=rse_id, reason=reason, state=status, account=issuer, bytes=size)
342
+ new_bad_replica = models.BadReplica(scope=scope, name=name, rse_id=rse_id, reason=reason, state=status, account=issuer, bytes=size)
322
343
  new_bad_replica.save(session=session, flush=False)
323
344
  declared = True
324
345
 
@@ -333,7 +354,11 @@ def __declare_bad_file_replicas(pfns, rse_id, reason, issuer, status=BadFilesSta
333
354
  no_hidden_char = False
334
355
  break
335
356
  if no_hidden_char:
336
- unknown_replicas.append('%s %s' % (path_pfn_dict[path], 'Unknown replica'))
357
+ pfn = path_pfn_dict[path]
358
+ if f"{pfn} Unknown replica" not in unknown_replicas:
359
+ unknown_replicas.append('%s %s' % (pfn, 'Unknown replica'))
360
+ elif scope or name:
361
+ unknown_replicas.append(f"{(scope,name)} Unknown replica")
337
362
 
338
363
  if status == BadFilesStatus.BAD:
339
364
  # For BAD file, we modify the replica state, not for suspicious
@@ -379,11 +404,19 @@ def add_bad_dids(dids, rse_id, reason, issuer, state=BadFilesStatus.BAD, *, sess
379
404
  for scope, name, _, __exists, already_declared, size in __exist_replicas(rse_id=rse_id, replicas=replicas_list, session=session):
380
405
  if __exists and not already_declared:
381
406
  replicas_for_update.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'state': ReplicaState.BAD})
382
- new_bad_replica = models.BadReplicas(scope=scope, name=name, rse_id=rse_id, reason=reason, state=state,
383
- account=issuer, bytes=size)
407
+ new_bad_replica = models.BadReplica(scope=scope, name=name, rse_id=rse_id, reason=reason, state=state,
408
+ account=issuer, bytes=size)
384
409
  new_bad_replica.save(session=session, flush=False)
385
- session.query(models.Source).filter_by(scope=scope, name=name,
386
- rse_id=rse_id).delete(synchronize_session=False)
410
+ stmt = delete(
411
+ models.Source
412
+ ).where(
413
+ and_(models.Source.scope == scope,
414
+ models.Source.name == name,
415
+ models.Source.rse_id == rse_id)
416
+ ).execution_options(
417
+ synchronize_session=False
418
+ )
419
+ session.execute(stmt)
387
420
  else:
388
421
  if already_declared:
389
422
  unknown_replicas.append('%s:%s %s' % (did['scope'], name, 'Already declared'))
@@ -456,58 +489,65 @@ def get_pfn_to_rse(pfns, vo='def', *, session: "Session"):
456
489
  storage_elements = []
457
490
  se_condition = []
458
491
  dict_rse = {}
459
- surls = clean_surls(pfns)
460
- scheme = surls[0].split(':')[0] if surls else None
461
- for surl in surls:
462
- if surl.split(':')[0] != scheme:
492
+ cleaned_pfns = clean_pfns(pfns)
493
+ scheme = cleaned_pfns[0].split(':')[0] if cleaned_pfns else None
494
+ for pfn in cleaned_pfns:
495
+ if pfn.split(':')[0] != scheme:
463
496
  raise exception.InvalidType('The PFNs specified must have the same protocol')
464
497
 
465
- split_se = surl.split('/')[2].split(':')
498
+ split_se = pfn.split('/')[2].split(':')
466
499
  storage_element = split_se[0]
467
500
 
468
501
  if storage_element not in storage_elements:
469
502
  storage_elements.append(storage_element)
470
- se_condition.append(models.RSEProtocols.hostname == storage_element)
471
- query = session.query(models.RSEProtocols.rse_id,
472
- models.RSEProtocols.scheme,
473
- models.RSEProtocols.hostname,
474
- models.RSEProtocols.port,
475
- models.RSEProtocols.prefix).\
476
- join(models.RSE, models.RSE.id == models.RSEProtocols.rse_id).\
477
- filter(and_(or_(*se_condition), models.RSEProtocols.scheme == scheme)).\
478
- filter(models.RSE.deleted == false()).\
479
- filter(models.RSE.staging_area == false())
503
+ se_condition.append(models.RSEProtocol.hostname == storage_element)
504
+ stmt = select(
505
+ models.RSEProtocol.rse_id,
506
+ models.RSEProtocol.scheme,
507
+ models.RSEProtocol.hostname,
508
+ models.RSEProtocol.port,
509
+ models.RSEProtocol.prefix
510
+ ).join(
511
+ models.RSE,
512
+ models.RSEProtocol.rse_id == models.RSE.id
513
+ ).where(
514
+ and_(or_(*se_condition),
515
+ models.RSEProtocol.scheme == scheme,
516
+ models.RSE.deleted == false(),
517
+ models.RSE.staging_area == false())
518
+ )
519
+
480
520
  protocols = {}
481
521
 
482
- for rse_id, protocol, hostname, port, prefix in query.yield_per(10000):
522
+ for rse_id, protocol, hostname, port, prefix in session.execute(stmt).yield_per(10000):
483
523
  if rse_id not in protocols:
484
524
  protocols[rse_id] = []
485
525
  protocols[rse_id].append('%s://%s:%s%s' % (protocol, hostname, port, prefix))
486
526
  if '%s://%s%s' % (protocol, hostname, prefix) not in protocols[rse_id]:
487
527
  protocols[rse_id].append('%s://%s%s' % (protocol, hostname, prefix))
488
528
  hint = None
489
- for surl in surls:
529
+ for pfn in cleaned_pfns:
490
530
  if hint:
491
531
  for pattern in protocols[hint]:
492
- if surl.find(pattern) > -1:
493
- dict_rse[hint].append(surl)
532
+ if pfn.find(pattern) > -1:
533
+ dict_rse[hint].append(pfn)
494
534
  else:
495
535
  mult_rse_match = 0
496
536
  for rse_id in protocols:
497
537
  for pattern in protocols[rse_id]:
498
- if surl.find(pattern) > -1 and get_rse_vo(rse_id=rse_id, session=session) == vo:
538
+ if pfn.find(pattern) > -1 and get_rse_vo(rse_id=rse_id, session=session) == vo:
499
539
  mult_rse_match += 1
500
540
  if mult_rse_match > 1:
501
- print('ERROR, multiple matches : %s at %s' % (surl, rse_id))
502
- raise exception.RucioException('ERROR, multiple matches : %s at %s' % (surl, get_rse_name(rse_id=rse_id, session=session)))
541
+ print('ERROR, multiple matches : %s at %s' % (pfn, rse_id))
542
+ raise exception.RucioException('ERROR, multiple matches : %s at %s' % (pfn, get_rse_name(rse_id=rse_id, session=session)))
503
543
  hint = rse_id
504
544
  if hint not in dict_rse:
505
545
  dict_rse[hint] = []
506
- dict_rse[hint].append(surl)
546
+ dict_rse[hint].append(pfn)
507
547
  if mult_rse_match == 0:
508
548
  if 'unknown' not in unknown_replicas:
509
549
  unknown_replicas['unknown'] = []
510
- unknown_replicas['unknown'].append(surl)
550
+ unknown_replicas['unknown'].append(pfn)
511
551
  return scheme, dict_rse, unknown_replicas
512
552
 
513
553
 
@@ -520,18 +560,28 @@ def get_bad_replicas_backlog(*, session: "Session"):
520
560
 
521
561
  :returns: a list of dictionary {'rse_id': cnt_bad_replicas}.
522
562
  """
523
- query = session.query(func.count(models.RSEFileAssociation.rse_id), models.RSEFileAssociation.rse_id). \
524
- with_hint(models.RSEFileAssociation, 'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))', 'oracle'). \
525
- filter(models.RSEFileAssociation.state == ReplicaState.BAD)
526
-
527
- query = query.join(models.DataIdentifier,
528
- and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
529
- models.DataIdentifier.name == models.RSEFileAssociation.name)).\
530
- filter(models.DataIdentifier.availability != DIDAvailability.LOST).\
531
- group_by(models.RSEFileAssociation.rse_id)
563
+ stmt = select(
564
+ func.count(),
565
+ models.RSEFileAssociation.rse_id
566
+ ).select_from(
567
+ models.RSEFileAssociation
568
+ ).with_hint(
569
+ models.RSEFileAssociation,
570
+ 'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))',
571
+ 'oracle'
572
+ ).join(
573
+ models.DataIdentifier,
574
+ and_(models.RSEFileAssociation.scope == models.DataIdentifier.scope,
575
+ models.RSEFileAssociation.name == models.DataIdentifier.name)
576
+ ).where(
577
+ and_(models.DataIdentifier.availability != DIDAvailability.LOST,
578
+ models.RSEFileAssociation.state == ReplicaState.BAD)
579
+ ).group_by(
580
+ models.RSEFileAssociation.rse_id
581
+ )
532
582
 
533
583
  result = dict()
534
- for cnt, rse_id in query.all():
584
+ for cnt, rse_id in session.execute(stmt).all():
535
585
  result[rse_id] = cnt
536
586
  return result
537
587
 
@@ -549,27 +599,36 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, rses=None, *
549
599
  :returns: a list of dictionary {'scope' scope, 'name': name, 'rse_id': rse_id, 'rse': rse}.
550
600
  """
551
601
  schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
552
- query = session.query(models.RSEFileAssociation.scope,
553
- models.RSEFileAssociation.name,
554
- models.RSEFileAssociation.rse_id). \
555
- with_hint(models.RSEFileAssociation, 'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))', 'oracle'). \
556
- filter(models.RSEFileAssociation.state == ReplicaState.BAD)
557
-
558
- query = filter_thread_work(session=session, query=query, total_threads=total_threads, thread_id=thread, hash_variable='%sreplicas.name' % (schema_dot))
559
- query = query.join(models.DataIdentifier,
560
- and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
561
- models.DataIdentifier.name == models.RSEFileAssociation.name)).\
562
- filter(models.DataIdentifier.availability != DIDAvailability.LOST)
602
+
603
+ stmt = select(
604
+ models.RSEFileAssociation.scope,
605
+ models.RSEFileAssociation.name,
606
+ models.RSEFileAssociation.rse_id
607
+ ).with_hint(
608
+ models.RSEFileAssociation,
609
+ 'INDEX(DIDS DIDS_PK) USE_NL(DIDS) INDEX_RS_ASC(REPLICAS ("REPLICAS"."STATE"))',
610
+ 'oracle'
611
+ ).where(
612
+ models.RSEFileAssociation.state == ReplicaState.BAD
613
+ )
614
+
615
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_threads, thread_id=thread, hash_variable='%sreplicas.name' % (schema_dot))
616
+
617
+ stmt = stmt.join(
618
+ models.DataIdentifier,
619
+ and_(models.RSEFileAssociation.scope == models.DataIdentifier.scope,
620
+ models.RSEFileAssociation.name == models.DataIdentifier.name)
621
+ ).where(
622
+ models.DataIdentifier.availability != DIDAvailability.LOST
623
+ )
563
624
 
564
625
  if rses:
565
- rse_clause = list()
566
- for rse in rses:
567
- rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
568
- query = query.filter(or_(*rse_clause))
626
+ rse_clause = [models.RSEFileAssociation.rse_id == rse['id'] for rse in rses]
627
+ stmt = stmt.where(or_(*rse_clause))
569
628
 
570
- query = query.limit(limit)
629
+ stmt = stmt.limit(limit)
571
630
  rows = []
572
- for scope, name, rse_id in query.yield_per(1000):
631
+ for scope, name, rse_id in session.execute(stmt).yield_per(1000):
573
632
  rows.append({'scope': scope, 'name': name, 'rse_id': rse_id, 'rse': get_rse_name(rse_id=rse_id, session=session)})
574
633
  return rows
575
634
 
@@ -599,20 +658,12 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
599
658
  pfndict = {}
600
659
  proto = rsemgr.create_protocol(rse_info, 'read', scheme=scheme)
601
660
  if rse_info['deterministic']:
661
+ scope_proto = rsemgr.get_scope_protocol(vo=vo)
602
662
  parsed_pfn = proto.parse_pfns(pfns=pfns)
603
663
 
604
- # WARNING : this part is ATLAS specific and must be changed
605
664
  for pfn in parsed_pfn:
606
- path = parsed_pfn[pfn]['path']
607
- if path.startswith('/user') or path.startswith('/group'):
608
- scope = '%s.%s' % (path.split('/')[1], path.split('/')[2])
609
- name = parsed_pfn[pfn]['name']
610
- elif path.startswith('/'):
611
- scope = path.split('/')[1]
612
- name = parsed_pfn[pfn]['name']
613
- else:
614
- scope = path.split('/')[0]
615
- name = parsed_pfn[pfn]['name']
665
+ # Translate into a scope and name
666
+ name, scope = scope_proto(parsed_pfn[pfn])
616
667
  scope = InternalScope(scope, vo)
617
668
  yield {pfn: {'scope': scope, 'name': name}}
618
669
  else:
@@ -621,8 +672,16 @@ def get_did_from_pfns(pfns, rse_id=None, vo='def', *, session: "Session"):
621
672
  for pfn in parsed_pfn:
622
673
  path = '%s%s' % (parsed_pfn[pfn]['path'], parsed_pfn[pfn]['name'])
623
674
  pfndict[path] = pfn
624
- condition.append(and_(models.RSEFileAssociation.path == path, models.RSEFileAssociation.rse_id == rse_id))
625
- for scope, name, pfn in session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.path).filter(or_(*condition)):
675
+ condition.append(and_(models.RSEFileAssociation.path == path,
676
+ models.RSEFileAssociation.rse_id == rse_id))
677
+ stmt = select(
678
+ models.RSEFileAssociation.scope,
679
+ models.RSEFileAssociation.name,
680
+ models.RSEFileAssociation.path
681
+ ).where(
682
+ or_(*condition)
683
+ )
684
+ for scope, name, pfn in session.execute(stmt).all():
626
685
  yield {pfndict[pfn]: {'scope': scope, 'name': name}}
627
686
 
628
687
 
@@ -651,7 +710,7 @@ def _pick_n_random(nrandom, generator):
651
710
  element = next(iterator)
652
711
  i += 1
653
712
 
654
- index_to_substitute = random.randint(0, i)
713
+ index_to_substitute = random.randint(0, i) # noqa: S311
655
714
  if index_to_substitute < nrandom:
656
715
  selected[index_to_substitute] = element
657
716
  except StopIteration:
@@ -667,15 +726,21 @@ def _list_files_wo_replicas(files_wo_replica, *, session: "Session"):
667
726
  for file in sorted(files_wo_replica, key=lambda f: (f['scope'], f['name'])):
668
727
  file_wo_clause.append(and_(models.DataIdentifier.scope == file['scope'],
669
728
  models.DataIdentifier.name == file['name']))
670
- files_wo_replicas_query = session.query(models.DataIdentifier.scope,
671
- models.DataIdentifier.name,
672
- models.DataIdentifier.bytes,
673
- models.DataIdentifier.md5,
674
- models.DataIdentifier.adler32).\
675
- filter_by(did_type=DIDType.FILE).filter(or_(*file_wo_clause)).\
676
- with_hint(models.DataIdentifier, text="INDEX(DIDS DIDS_PK)", dialect_name='oracle')
677
-
678
- for scope, name, bytes_, md5, adler32 in files_wo_replicas_query:
729
+ stmt = select(
730
+ models.DataIdentifier.scope,
731
+ models.DataIdentifier.name,
732
+ models.DataIdentifier.bytes,
733
+ models.DataIdentifier.md5,
734
+ models.DataIdentifier.adler32
735
+ ).with_hint(
736
+ models.DataIdentifier,
737
+ 'INDEX(DIDS DIDS_PK)',
738
+ 'oracle'
739
+ ).where(
740
+ and_(models.DataIdentifier.did_type == DIDType.FILE,
741
+ or_(*file_wo_clause))
742
+ )
743
+ for scope, name, bytes_, md5, adler32 in session.execute(stmt):
679
744
  yield scope, name, bytes_, md5, adler32
680
745
 
681
746
 
@@ -795,7 +860,7 @@ def _build_list_replicas_pfn(
795
860
  """
796
861
  Generate the PFN for the given scope/name on the rse.
797
862
  If needed, sign the PFN url
798
- If relevant, add the server-side root proxy to te pfn url
863
+ If relevant, add the server-side root proxy to the pfn url
799
864
  """
800
865
  pfn: str = list(protocol.lfns2pfns(lfns={'scope': scope.external,
801
866
  'name': name,
@@ -803,7 +868,7 @@ def _build_list_replicas_pfn(
803
868
 
804
869
  # do we need to sign the URLs?
805
870
  if sign_urls and protocol.attributes['scheme'] == 'https':
806
- service = get_rse_attribute(rse_id, 'sign_url', session=session)
871
+ service = get_rse_attribute(rse_id, RseAttr.SIGN_URL, session=session)
807
872
  if service:
808
873
  pfn = get_signed_url(rse_id=rse_id, service=service, operation='read', url=pfn, lifetime=signature_lifetime)
809
874
 
@@ -814,7 +879,7 @@ def _build_list_replicas_pfn(
814
879
  if domain == 'wan' and protocol.attributes['scheme'] in ['root', 'http', 'https'] and client_location:
815
880
 
816
881
  if 'site' in client_location and client_location['site']:
817
- replica_site = get_rse_attribute(rse_id, 'site', session=session)
882
+ replica_site = get_rse_attribute(rse_id, RseAttr.SITE, session=session)
818
883
 
819
884
  # does it match with the client? if not, it's an outgoing connection
820
885
  # therefore the internal proxy must be prepended
@@ -843,7 +908,7 @@ def _build_list_replicas_pfn(
843
908
  # don't forget to mangle gfal-style davs URL into generic https URL
844
909
  pfn = f"root://{root_proxy_internal}//{pfn.replace('davs://', 'https://')}"
845
910
 
846
- simulate_multirange = get_rse_attribute(rse_id, 'simulate_multirange')
911
+ simulate_multirange = get_rse_attribute(rse_id, RseAttr.SIMULATE_MULTIRANGE)
847
912
 
848
913
  if simulate_multirange is not None:
849
914
  try:
@@ -851,9 +916,9 @@ def _build_list_replicas_pfn(
851
916
  simulate_multirange = int(simulate_multirange)
852
917
  except ValueError:
853
918
  simulate_multirange = 1
854
- logger(logging.WARNING, 'Value encountered when retrieving RSE attribute "simulate_multirange" not compatible with "int", used default value "1".')
919
+ logger(logging.WARNING, 'Value encountered when retrieving RSE attribute "%s" not compatible with "int", used default value "1".', RseAttr.SIMULATE_MULTIRANGE)
855
920
  if simulate_multirange <= 0:
856
- logger(logging.WARNING, f'Value {simulate_multirange} encountered when retrieving RSE attribute "simulate_multirange" is <= 0, used default value "1".')
921
+ logger(logging.WARNING, f'Value {simulate_multirange} encountered when retrieving RSE attribute "{RseAttr.SIMULATE_MULTIRANGE}" is <= 0, used default value "1".')
857
922
  simulate_multirange = 1
858
923
  pfn += f'&#multirange=false&nconnections={simulate_multirange}'
859
924
 
@@ -936,7 +1001,7 @@ def _list_replicas(replicas, show_pfns, schemes, files_wo_replica, client_locati
936
1001
  t_scope = scope
937
1002
  t_name = name
938
1003
 
939
- if 'determinism_type' in protocol.attributes: # PFN is cachable
1004
+ if 'determinism_type' in protocol.attributes: # PFN is cacheable
940
1005
  try:
941
1006
  path = pfns_cache['%s:%s:%s' % (protocol.attributes['determinism_type'], t_scope.internal, t_name)]
942
1007
  except KeyError: # No cache entry scope:name found for this protocol
@@ -1081,7 +1146,7 @@ def list_replicas(
1081
1146
  models.RSE.volatile,
1082
1147
  ).join(
1083
1148
  models.RSE,
1084
- and_(models.RSE.id == models.RSEFileAssociation.rse_id,
1149
+ and_(models.RSEFileAssociation.rse_id == models.RSE.id,
1085
1150
  models.RSE.deleted == false())
1086
1151
  )
1087
1152
 
@@ -1100,7 +1165,11 @@ def list_replicas(
1100
1165
  stmt = stmt.where(models.RSE.id.in_([rse['id'] for rse in rses]))
1101
1166
  else:
1102
1167
  rses_temp_table = temp_table_mngr(session).create_id_table()
1103
- session.execute(insert(rses_temp_table), [{'id': rse['id']} for rse in rses])
1168
+ values = [{'id': rse['id']} for rse in rses]
1169
+ insert_stmt = insert(
1170
+ rses_temp_table
1171
+ )
1172
+ session.execute(insert_stmt, values)
1104
1173
  stmt = stmt.join(rses_temp_table, models.RSE.id == rses_temp_table.id)
1105
1174
 
1106
1175
  if not all_states:
@@ -1122,16 +1191,16 @@ def list_replicas(
1122
1191
  created temporary table.
1123
1192
  """
1124
1193
  resolved_files_temp_table = temp_table_mngr(session).create_scope_name_table()
1194
+ selectable = rucio.core.did.list_child_dids_stmt(temp_table, did_type=DIDType.FILE)
1125
1195
 
1126
1196
  stmt = insert(
1127
1197
  resolved_files_temp_table
1128
1198
  ).from_select(
1129
1199
  ['scope', 'name'],
1130
- rucio.core.did.list_child_dids_stmt(temp_table, did_type=DIDType.FILE)
1200
+ selectable
1131
1201
  )
1132
- result = session.execute(stmt)
1133
1202
 
1134
- return result.rowcount, resolved_files_temp_table
1203
+ return session.execute(stmt).rowcount, resolved_files_temp_table
1135
1204
 
1136
1205
  def _list_replicas_for_collection_files_stmt(temp_table, replicas_subquery):
1137
1206
  """
@@ -1264,7 +1333,11 @@ def list_replicas(
1264
1333
  return
1265
1334
 
1266
1335
  input_dids_temp_table = temp_table_mngr(session).create_scope_name_table()
1267
- session.execute(insert(input_dids_temp_table), [{'scope': s, 'name': n} for s, n in dids])
1336
+ values = [{'scope': scope, 'name': name} for scope, name in dids]
1337
+ stmt = insert(
1338
+ input_dids_temp_table
1339
+ )
1340
+ session.execute(stmt, values)
1268
1341
 
1269
1342
  num_files, num_collections, num_constituents = _inspect_dids(input_dids_temp_table, session=session)
1270
1343
 
@@ -1325,10 +1398,16 @@ def list_replicas(
1325
1398
  # leave us with less than nrandom replicas.
1326
1399
  nrandom * 4
1327
1400
  )
1328
- # Re-use input temp table. We don't need its content anymore
1401
+ # Reuse input temp table. We don't need its content anymore
1329
1402
  random_dids_temp_table = input_dids_temp_table
1330
1403
  session.execute(delete(random_dids_temp_table))
1331
- session.execute(insert(random_dids_temp_table).from_select(['scope', 'name'], stmt))
1404
+ stmt = insert(
1405
+ random_dids_temp_table
1406
+ ).from_select(
1407
+ ['scope', 'name'],
1408
+ stmt
1409
+ )
1410
+ session.execute(stmt)
1332
1411
 
1333
1412
  # Fetch all replicas for randomly selected dids and apply filters on python side
1334
1413
  stmt = _list_replicas_for_collection_files_stmt(random_dids_temp_table, replicas_subquery)
@@ -1426,18 +1505,28 @@ def __bulk_add_file_dids(files, account, dataset_meta=None, *, session: "Session
1426
1505
  :param dids: the list of files.
1427
1506
  :param account: The account owner.
1428
1507
  :param session: The database session in use.
1429
- :returns: True is successful.
1508
+ :returns: list of replicas.
1430
1509
  """
1431
1510
  condition = []
1432
1511
  for f in files:
1433
- condition.append(and_(models.DataIdentifier.scope == f['scope'], models.DataIdentifier.name == f['name'], models.DataIdentifier.did_type == DIDType.FILE))
1434
-
1435
- q = session.query(models.DataIdentifier.scope,
1436
- models.DataIdentifier.name,
1437
- models.DataIdentifier.bytes,
1438
- models.DataIdentifier.adler32,
1439
- models.DataIdentifier.md5).with_hint(models.DataIdentifier, "INDEX(dids DIDS_PK)", 'oracle').filter(or_(*condition))
1440
- available_files = [dict([(column, getattr(row, column)) for column in row._fields]) for row in q]
1512
+ condition.append(and_(models.DataIdentifier.scope == f['scope'],
1513
+ models.DataIdentifier.name == f['name'],
1514
+ models.DataIdentifier.did_type == DIDType.FILE))
1515
+
1516
+ stmt = select(
1517
+ models.DataIdentifier.scope,
1518
+ models.DataIdentifier.name,
1519
+ models.DataIdentifier.bytes,
1520
+ models.DataIdentifier.md5,
1521
+ models.DataIdentifier.adler32,
1522
+ ).with_hint(
1523
+ models.DataIdentifier,
1524
+ 'INDEX(DIDS DIDS_PK)',
1525
+ 'oracle'
1526
+ ).where(
1527
+ or_(*condition)
1528
+ )
1529
+ available_files = [res._asdict() for res in session.execute(stmt).all()]
1441
1530
  new_files = list()
1442
1531
  for file in files:
1443
1532
  found = False
@@ -1484,14 +1573,25 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
1484
1573
  # Check for the replicas already available
1485
1574
  condition = []
1486
1575
  for f in files:
1487
- condition.append(and_(models.RSEFileAssociation.scope == f['scope'], models.RSEFileAssociation.name == f['name'], models.RSEFileAssociation.rse_id == rse_id))
1576
+ condition.append(and_(models.RSEFileAssociation.scope == f['scope'],
1577
+ models.RSEFileAssociation.name == f['name'],
1578
+ models.RSEFileAssociation.rse_id == rse_id))
1488
1579
 
1489
- query = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id).\
1490
- with_hint(models.RSEFileAssociation, text="INDEX(REPLICAS REPLICAS_PK)", dialect_name='oracle').\
1491
- filter(or_(*condition))
1492
- available_replicas = [dict([(column, getattr(row, column)) for column in row._fields]) for row in query]
1580
+ stmt = select(
1581
+ models.RSEFileAssociation.scope,
1582
+ models.RSEFileAssociation.name,
1583
+ models.RSEFileAssociation.rse_id,
1584
+ ).with_hint(
1585
+ models.RSEFileAssociation,
1586
+ 'INDEX(REPLICAS REPLICAS_PK)',
1587
+ 'oracle'
1588
+ ).where(
1589
+ or_(*condition)
1590
+ )
1493
1591
 
1494
- default_tombstone_delay = get_rse_attribute(rse_id, 'tombstone_delay', session=session)
1592
+ available_replicas = [res._asdict() for res in session.execute(stmt).all()]
1593
+
1594
+ default_tombstone_delay = get_rse_attribute(rse_id, RseAttr.TOMBSTONE_DELAY, session=session)
1495
1595
  default_tombstone = tombstone_from_delay(default_tombstone_delay)
1496
1596
 
1497
1597
  new_replicas = []
@@ -1512,7 +1612,10 @@ def __bulk_add_replicas(rse_id, files, account, *, session: "Session"):
1512
1612
  'lock_cnt': file.get('lock_cnt', 0),
1513
1613
  'tombstone': file.get('tombstone') or default_tombstone})
1514
1614
  try:
1515
- new_replicas and session.execute(insert(models.RSEFileAssociation), new_replicas)
1615
+ stmt = insert(
1616
+ models.RSEFileAssociation
1617
+ )
1618
+ new_replicas and session.execute(stmt, new_replicas)
1516
1619
  session.flush()
1517
1620
  return nbfiles, bytes_
1518
1621
  except IntegrityError as error:
@@ -1538,13 +1641,13 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1538
1641
  :param ignore_availability: Ignore the RSE blocklisting.
1539
1642
  :param session: The database session in use.
1540
1643
 
1541
- :returns: True is successful.
1644
+ :returns: list of replicas.
1542
1645
  """
1543
1646
 
1544
1647
  def _expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=None):
1545
1648
  p = rsemgr.create_protocol(rse_settings=rse_settings, operation='write', scheme=scheme, domain=domain, protocol_attr=protocol_attr)
1546
1649
  expected_pfns = p.lfns2pfns(lfns)
1547
- return clean_surls(expected_pfns.values())
1650
+ return clean_pfns(expected_pfns.values())
1548
1651
 
1549
1652
  replica_rse = get_rse(rse_id=rse_id, session=session)
1550
1653
 
@@ -1559,9 +1662,9 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1559
1662
  if not replica_rse['deterministic']:
1560
1663
  raise exception.UnsupportedOperation('PFN needed for this (non deterministic) RSE %s ' % (replica_rse['rse']))
1561
1664
 
1562
- replicas = __bulk_add_file_dids(files=files, account=account,
1563
- dataset_meta=dataset_meta,
1564
- session=session)
1665
+ __bulk_add_file_dids(files=files, account=account,
1666
+ dataset_meta=dataset_meta,
1667
+ session=session)
1565
1668
 
1566
1669
  pfns = {} # dict[str, list[str]], {scheme: [pfns], scheme: [pfns]}
1567
1670
  for file in files:
@@ -1582,7 +1685,7 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1582
1685
  else:
1583
1686
  # Check that the pfns match to the expected pfns
1584
1687
  lfns = [{'scope': i['scope'].external, 'name': i['name']} for i in files if i['pfn'].startswith(scheme)]
1585
- pfns[scheme] = clean_surls(pfns[scheme])
1688
+ pfns[scheme] = clean_pfns(pfns[scheme])
1586
1689
 
1587
1690
  for protocol_attr in rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme, domain='wan'):
1588
1691
  pfns[scheme] = list(set(pfns[scheme]) - set(_expected_pfns(lfns, rse_settings, scheme, operation='write', domain='wan', protocol_attr=protocol_attr)))
@@ -1597,11 +1700,25 @@ def add_replicas(rse_id, files, account, ignore_availability=True,
1597
1700
 
1598
1701
  nbfiles, bytes_ = __bulk_add_replicas(rse_id=rse_id, files=files, account=account, session=session)
1599
1702
  increase(rse_id=rse_id, files=nbfiles, bytes_=bytes_, session=session)
1600
- return replicas
1601
1703
 
1602
1704
 
1603
1705
  @transactional_session
1604
- def add_replica(rse_id, scope, name, bytes_, account, adler32=None, md5=None, dsn=None, pfn=None, meta=None, rules=[], tombstone=None, *, session: "Session"):
1706
+ def add_replica(
1707
+ rse_id: str,
1708
+ scope: InternalScope,
1709
+ name: str,
1710
+ bytes_: int,
1711
+ account: models.InternalAccount,
1712
+ adler32: "Optional[str]" = None,
1713
+ md5: "Optional[str]" = None,
1714
+ dsn: "Optional[str]" = None,
1715
+ pfn: "Optional[str]" = None,
1716
+ meta: "Optional[dict[str, Any]]" = None,
1717
+ rules: "Optional[list[dict[str, Any]]]" = None,
1718
+ tombstone: "Optional[datetime]" = None,
1719
+ *,
1720
+ session: "Session"
1721
+ ) -> "list[dict[str, Any]]":
1605
1722
  """
1606
1723
  Add File replica.
1607
1724
 
@@ -1618,10 +1735,10 @@ def add_replica(rse_id, scope, name, bytes_, account, adler32=None, md5=None, ds
1618
1735
  :param tombstone: If True, create replica with a tombstone.
1619
1736
  :param session: The database session in use.
1620
1737
 
1621
- :returns: True is successful.
1738
+ :returns: list of replicas.
1622
1739
  """
1623
- if meta is None:
1624
- meta = {}
1740
+ meta = meta or {}
1741
+ rules = rules or []
1625
1742
 
1626
1743
  file = {'scope': scope, 'name': name, 'bytes': bytes_, 'adler32': adler32, 'md5': md5, 'meta': meta, 'rules': rules, 'tombstone': tombstone}
1627
1744
  if pfn:
@@ -1653,7 +1770,11 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
1653
1770
  scope_name_temp_table2 = tt_mngr.create_scope_name_table()
1654
1771
  association_temp_table = tt_mngr.create_association_table()
1655
1772
 
1656
- session.execute(insert(scope_name_temp_table), [{'scope': file['scope'], 'name': file['name']} for file in files])
1773
+ values = [{'scope': file['scope'], 'name': file['name']} for file in files]
1774
+ stmt = insert(
1775
+ scope_name_temp_table
1776
+ )
1777
+ session.execute(stmt, values)
1657
1778
 
1658
1779
  # WARNING : This should not be necessary since that would mean the replica is used as a source.
1659
1780
  stmt = delete(
@@ -1685,9 +1806,10 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
1685
1806
  models.RSEFileAssociation,
1686
1807
  ).where(
1687
1808
  exists(select(1)
1688
- .where(and_(models.RSEFileAssociation.scope == scope_name_temp_table.scope,
1689
- models.RSEFileAssociation.name == scope_name_temp_table.name,
1690
- models.RSEFileAssociation.rse_id == rse_id)))
1809
+ .where(
1810
+ and_(models.RSEFileAssociation.scope == scope_name_temp_table.scope,
1811
+ models.RSEFileAssociation.name == scope_name_temp_table.name,
1812
+ models.RSEFileAssociation.rse_id == rse_id)))
1691
1813
  ).execution_options(
1692
1814
  synchronize_session=False
1693
1815
  )
@@ -1697,20 +1819,22 @@ def delete_replicas(rse_id, files, ignore_availability=True, *, session: "Sessio
1697
1819
 
1698
1820
  # Update bad replicas
1699
1821
  stmt = update(
1700
- models.BadReplicas,
1822
+ models.BadReplica,
1701
1823
  ).where(
1702
1824
  exists(select(1)
1703
- .where(and_(models.BadReplicas.scope == scope_name_temp_table.scope,
1704
- models.BadReplicas.name == scope_name_temp_table.name,
1705
- models.BadReplicas.rse_id == rse_id)))
1825
+ .where(
1826
+ and_(models.BadReplica.scope == scope_name_temp_table.scope,
1827
+ models.BadReplica.name == scope_name_temp_table.name,
1828
+ models.BadReplica.rse_id == rse_id)))
1706
1829
  ).where(
1707
- models.BadReplicas.state == BadFilesStatus.BAD
1708
- ).execution_options(
1830
+ models.BadReplica.state == BadFilesStatus.BAD
1831
+ ).values({
1832
+ models.BadReplica.state: BadFilesStatus.DELETED,
1833
+ models.BadReplica.updated_at: datetime.utcnow()
1834
+ }).execution_options(
1709
1835
  synchronize_session=False
1710
- ).values(
1711
- state=BadFilesStatus.DELETED,
1712
- updated_at=datetime.utcnow()
1713
1836
  )
1837
+
1714
1838
  res = session.execute(stmt)
1715
1839
 
1716
1840
  __cleanup_after_replica_deletion(scope_name_temp_table=scope_name_temp_table,
@@ -1762,8 +1886,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1762
1886
 
1763
1887
  if clt_to_update:
1764
1888
  # Get all collection_replicas at RSE, insert them into UpdatedCollectionReplica
1765
- session.query(scope_name_temp_table).delete()
1766
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in clt_to_update])
1889
+ stmt = delete(scope_name_temp_table)
1890
+ session.execute(stmt)
1891
+ values = [sn._asdict() for sn in clt_to_update]
1892
+ stmt = insert(scope_name_temp_table)
1893
+ session.execute(stmt, values)
1767
1894
  stmt = select(
1768
1895
  models.DataIdentifierAssociation.scope,
1769
1896
  models.DataIdentifierAssociation.name,
@@ -1790,8 +1917,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1790
1917
  while parents_to_analyze:
1791
1918
  did_associations_to_remove = set()
1792
1919
 
1793
- session.query(scope_name_temp_table).delete()
1794
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in parents_to_analyze])
1920
+ stmt = delete(scope_name_temp_table)
1921
+ session.execute(stmt)
1922
+ values = [sn._asdict() for sn in parents_to_analyze]
1923
+ stmt = insert(scope_name_temp_table)
1924
+ session.execute(stmt, values)
1795
1925
  parents_to_analyze.clear()
1796
1926
 
1797
1927
  stmt = select(
@@ -1862,7 +1992,7 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1862
1992
  did_condition.append(
1863
1993
  and_(models.DataIdentifier.scope == parent_scope,
1864
1994
  models.DataIdentifier.name == parent_name,
1865
- models.DataIdentifier.is_open == False, # NOQA
1995
+ models.DataIdentifier.is_open == false(),
1866
1996
  ~exists(1).where(
1867
1997
  and_(models.DataIdentifierAssociation.child_scope == parent_scope,
1868
1998
  models.DataIdentifierAssociation.child_name == parent_name)),
@@ -1871,8 +2001,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1871
2001
  models.DataIdentifierAssociation.name == parent_name))))
1872
2002
 
1873
2003
  if did_associations_to_remove:
1874
- session.query(association_temp_table).delete()
1875
- session.execute(insert(association_temp_table), [a._asdict() for a in did_associations_to_remove])
2004
+ stmt = delete(association_temp_table)
2005
+ session.execute(stmt)
2006
+ values = [a._asdict() for a in did_associations_to_remove]
2007
+ stmt = insert(association_temp_table)
2008
+ session.execute(stmt, values)
1876
2009
 
1877
2010
  # get the list of modified parent scope, name
1878
2011
  stmt = select(
@@ -1887,7 +2020,7 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1887
2020
  association_temp_table.name == models.DataIdentifier.name)
1888
2021
  ).where(
1889
2022
  or_(models.DataIdentifier.complete == true(),
1890
- models.DataIdentifier.complete is None),
2023
+ models.DataIdentifier.complete.is_(None)),
1891
2024
  )
1892
2025
  for parent_scope, parent_name, parent_did_type in session.execute(stmt):
1893
2026
  message = {'scope': parent_scope,
@@ -1917,9 +2050,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1917
2050
 
1918
2051
  # Get collection replicas of collections which became empty
1919
2052
  if clt_replicas_to_delete:
1920
- session.query(scope_name_temp_table).delete()
1921
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in clt_replicas_to_delete])
1922
- session.query(scope_name_temp_table2).delete()
2053
+ stmt = delete(scope_name_temp_table)
2054
+ session.execute(stmt)
2055
+ values = [sn._asdict() for sn in clt_replicas_to_delete]
2056
+ stmt = insert(scope_name_temp_table)
2057
+ session.execute(stmt, values)
2058
+ stmt = delete(scope_name_temp_table2)
2059
+ session.execute(stmt)
1923
2060
  stmt = select(
1924
2061
  models.CollectionReplica.scope,
1925
2062
  models.CollectionReplica.name,
@@ -1940,9 +2077,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1940
2077
  ).where(
1941
2078
  models.DataIdentifierAssociation.scope == null()
1942
2079
  )
1943
- session.execute(
1944
- insert(scope_name_temp_table2).from_select(['scope', 'name'], stmt)
2080
+ stmt = insert(
2081
+ scope_name_temp_table2
2082
+ ).from_select(
2083
+ ['scope', 'name'],
2084
+ stmt
1945
2085
  )
2086
+ session.execute(stmt)
1946
2087
  # Delete the retrieved collection replicas of empty collections
1947
2088
  stmt = delete(
1948
2089
  models.CollectionReplica,
@@ -1958,8 +2099,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1958
2099
  # Update incomplete state
1959
2100
  messages, dids_to_delete = [], set()
1960
2101
  if incomplete_dids:
1961
- session.query(scope_name_temp_table).delete()
1962
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in incomplete_dids])
2102
+ stmt = delete(scope_name_temp_table)
2103
+ session.execute(stmt)
2104
+ values = [sn._asdict() for sn in incomplete_dids]
2105
+ stmt = insert(scope_name_temp_table)
2106
+ session.execute(stmt, values)
1963
2107
  stmt = update(
1964
2108
  models.DataIdentifier
1965
2109
  ).where(
@@ -1968,22 +2112,29 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1968
2112
  models.DataIdentifier.name == scope_name_temp_table.name)))
1969
2113
  ).where(
1970
2114
  models.DataIdentifier.complete != false(),
1971
- ).execution_options(
2115
+ ).values({
2116
+ models.DataIdentifier.complete: False
2117
+ }).execution_options(
1972
2118
  synchronize_session=False
1973
- ).values(
1974
- complete=False
1975
2119
  )
2120
+
1976
2121
  session.execute(stmt)
1977
2122
 
1978
2123
  # delete empty dids
1979
2124
  if did_condition:
1980
2125
  for chunk in chunks(did_condition, 10):
1981
- query = session.query(models.DataIdentifier.scope,
1982
- models.DataIdentifier.name,
1983
- models.DataIdentifier.did_type). \
1984
- with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle'). \
1985
- filter(or_(*chunk))
1986
- for scope, name, did_type in query:
2126
+ stmt = select(
2127
+ models.DataIdentifier.scope,
2128
+ models.DataIdentifier.name,
2129
+ models.DataIdentifier.did_type,
2130
+ ).with_hint(
2131
+ models.DataIdentifier,
2132
+ 'INDEX(DIDS DIDS_PK)',
2133
+ 'oracle'
2134
+ ).where(
2135
+ or_(*chunk)
2136
+ )
2137
+ for scope, name, did_type in session.execute(stmt):
1987
2138
  if did_type == DIDType.DATASET:
1988
2139
  messages.append({'event_type': 'ERASE',
1989
2140
  'payload': dumps({'scope': scope.external,
@@ -1994,8 +2145,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
1994
2145
  # Remove Archive Constituents
1995
2146
  constituent_associations_to_delete = set()
1996
2147
  if affected_archives:
1997
- session.query(scope_name_temp_table).delete()
1998
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in affected_archives])
2148
+ stmt = delete(scope_name_temp_table)
2149
+ session.execute(stmt)
2150
+ values = [sn._asdict() for sn in affected_archives]
2151
+ stmt = insert(scope_name_temp_table)
2152
+ session.execute(stmt, values)
1999
2153
 
2000
2154
  stmt = select(
2001
2155
  models.ConstituentAssociation
@@ -2038,8 +2192,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2038
2192
  ).save(session=session, flush=False)
2039
2193
 
2040
2194
  if constituent_associations_to_delete:
2041
- session.query(association_temp_table).delete()
2042
- session.execute(insert(association_temp_table), [a._asdict() for a in constituent_associations_to_delete])
2195
+ stmt = delete(association_temp_table)
2196
+ session.execute(stmt)
2197
+ values = [a._asdict() for a in constituent_associations_to_delete]
2198
+ stmt = insert(association_temp_table)
2199
+ session.execute(stmt, values)
2043
2200
  stmt = delete(
2044
2201
  models.ConstituentAssociation
2045
2202
  ).where(
@@ -2061,8 +2218,11 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2061
2218
  rse_id=rse_id, files=[sn._asdict() for sn in chunk], session=session)
2062
2219
 
2063
2220
  if dids_to_delete:
2064
- session.query(scope_name_temp_table).delete()
2065
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in dids_to_delete])
2221
+ stmt = delete(scope_name_temp_table)
2222
+ session.execute(stmt)
2223
+ values = [sn._asdict() for sn in dids_to_delete]
2224
+ stmt = insert(scope_name_temp_table)
2225
+ session.execute(stmt, values)
2066
2226
 
2067
2227
  # Remove rules in Waiting for approval or Suspended
2068
2228
  stmt = delete(
@@ -2120,9 +2280,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2120
2280
  to_update = clt_to_set_not_archive.pop(0)
2121
2281
  if not to_update:
2122
2282
  continue
2123
- session.query(scope_name_temp_table).delete()
2124
- session.execute(insert(scope_name_temp_table), [sn._asdict() for sn in to_update])
2125
- session.query(scope_name_temp_table2).delete()
2283
+ stmt = delete(scope_name_temp_table)
2284
+ session.execute(stmt)
2285
+ values = [sn._asdict() for sn in to_update]
2286
+ stmt = insert(scope_name_temp_table)
2287
+ session.execute(stmt, values)
2288
+ stmt = delete(scope_name_temp_table2)
2289
+ session.execute(stmt)
2126
2290
 
2127
2291
  data_identifier_alias = aliased(models.DataIdentifier, name='did_alias')
2128
2292
  # Fetch rows to be updated
@@ -2149,7 +2313,13 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2149
2313
  ).where(
2150
2314
  data_identifier_alias.scope == null()
2151
2315
  )
2152
- session.execute(insert(scope_name_temp_table2).from_select(['scope', 'name'], stmt))
2316
+ stmt = insert(
2317
+ scope_name_temp_table2
2318
+ ).from_select(
2319
+ ['scope', 'name'],
2320
+ stmt
2321
+ )
2322
+ session.execute(stmt)
2153
2323
  # update the fetched rows
2154
2324
  stmt = update(
2155
2325
  models.DataIdentifier,
@@ -2157,10 +2327,10 @@ def __cleanup_after_replica_deletion(scope_name_temp_table, scope_name_temp_tabl
2157
2327
  exists(select(1)
2158
2328
  .where(and_(models.DataIdentifier.scope == scope_name_temp_table2.scope,
2159
2329
  models.DataIdentifier.name == scope_name_temp_table2.name)))
2160
- ).execution_options(
2330
+ ).values({
2331
+ models.DataIdentifier.is_archive: False
2332
+ }).execution_options(
2161
2333
  synchronize_session=False
2162
- ).values(
2163
- is_archive=False,
2164
2334
  )
2165
2335
  session.execute(stmt)
2166
2336
 
@@ -2178,8 +2348,14 @@ def get_replica(rse_id, scope, name, *, session: "Session"):
2178
2348
  :returns: A dictionary with the list of replica attributes.
2179
2349
  """
2180
2350
  try:
2181
- row = session.query(models.RSEFileAssociation).filter_by(rse_id=rse_id, scope=scope, name=name).one()
2182
- return row.to_dict()
2351
+ stmt = select(
2352
+ models.RSEFileAssociation
2353
+ ).where(
2354
+ and_(models.RSEFileAssociation.scope == scope,
2355
+ models.RSEFileAssociation.name == name,
2356
+ models.RSEFileAssociation.rse_id == rse_id)
2357
+ )
2358
+ return session.execute(stmt).scalar_one().to_dict()
2183
2359
  except NoResultFound:
2184
2360
  raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
2185
2361
 
@@ -2236,8 +2412,11 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2236
2412
  )
2237
2413
 
2238
2414
  for chunk in chunks(session.execute(stmt).yield_per(2 * limit), math.ceil(1.25 * limit)):
2239
- session.query(temp_table_cls).delete()
2240
- session.execute(insert(temp_table_cls), [{'scope': scope, 'name': name} for scope, name in chunk])
2415
+ stmt = delete(temp_table_cls)
2416
+ session.execute(stmt)
2417
+ values = [{'scope': scope, 'name': name} for scope, name in chunk]
2418
+ stmt = insert(temp_table_cls)
2419
+ session.execute(stmt, values)
2241
2420
 
2242
2421
  stmt = select(
2243
2422
  models.RSEFileAssociation.scope,
@@ -2254,7 +2433,9 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2254
2433
  models.RSEFileAssociation.name == temp_table_cls.name,
2255
2434
  models.RSEFileAssociation.rse_id == rse_id)
2256
2435
  ).with_hint(
2257
- replicas_alias, "index(%(name)s REPLICAS_PK)", 'oracle'
2436
+ replicas_alias,
2437
+ 'INDEX(%(name)s REPLICAS_PK)',
2438
+ 'oracle'
2258
2439
  ).outerjoin(
2259
2440
  replicas_alias,
2260
2441
  and_(models.RSEFileAssociation.scope == replicas_alias.scope,
@@ -2262,7 +2443,9 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2262
2443
  models.RSEFileAssociation.rse_id != replicas_alias.rse_id,
2263
2444
  replicas_alias.state == ReplicaState.AVAILABLE)
2264
2445
  ).with_hint(
2265
- models.Request, "INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX)", 'oracle'
2446
+ models.Request,
2447
+ 'INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX)',
2448
+ 'oracle'
2266
2449
  ).outerjoin(
2267
2450
  models.Request,
2268
2451
  and_(models.RSEFileAssociation.scope == models.Request.scope,
@@ -2304,8 +2487,11 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2304
2487
  break
2305
2488
 
2306
2489
  if rows:
2307
- session.query(temp_table_cls).delete()
2308
- session.execute(insert(temp_table_cls), [{'scope': r['scope'], 'name': r['name']} for r in rows])
2490
+ stmt = delete(temp_table_cls)
2491
+ session.execute(stmt)
2492
+ values = [{'scope': row['scope'], 'name': row['name']} for row in rows]
2493
+ stmt = insert(temp_table_cls)
2494
+ session.execute(stmt, values)
2309
2495
  stmt = update(
2310
2496
  models.RSEFileAssociation
2311
2497
  ).where(
@@ -2313,13 +2499,14 @@ def list_and_mark_unlocked_replicas(limit, bytes_=None, rse_id=None, delay_secon
2313
2499
  .where(and_(models.RSEFileAssociation.scope == temp_table_cls.scope,
2314
2500
  models.RSEFileAssociation.name == temp_table_cls.name,
2315
2501
  models.RSEFileAssociation.rse_id == rse_id)))
2316
- ).execution_options(
2502
+ ).values({
2503
+ models.RSEFileAssociation.updated_at: datetime.utcnow(),
2504
+ models.RSEFileAssociation.state: ReplicaState.BEING_DELETED,
2505
+ models.RSEFileAssociation.tombstone: OBSOLETE
2506
+ }).execution_options(
2317
2507
  synchronize_session=False
2318
- ).values(
2319
- updated_at=datetime.utcnow(),
2320
- state=ReplicaState.BEING_DELETED,
2321
- tombstone=OBSOLETE,
2322
2508
  )
2509
+
2323
2510
  session.execute(stmt)
2324
2511
 
2325
2512
  return rows
@@ -2336,11 +2523,17 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
2336
2523
  """
2337
2524
 
2338
2525
  for replica in replicas:
2339
- query = session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'])
2340
- try:
2341
- if nowait:
2342
- query.with_for_update(nowait=True).one()
2343
- except NoResultFound:
2526
+ stmt = select(
2527
+ models.RSEFileAssociation
2528
+ ).where(
2529
+ models.RSEFileAssociation.rse_id == replica['rse_id'],
2530
+ models.RSEFileAssociation.scope == replica['scope'],
2531
+ models.RSEFileAssociation.name == replica['name']
2532
+ ).with_for_update(
2533
+ nowait=nowait
2534
+ )
2535
+
2536
+ if session.execute(stmt).scalar_one_or_none() is None:
2344
2537
  # remember scope, name and rse
2345
2538
  raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (replica['scope'], replica['name'], get_rse_name(replica['rse_id'], session=session)))
2346
2539
 
@@ -2349,18 +2542,43 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
2349
2542
 
2350
2543
  values = {'state': replica['state']}
2351
2544
  if replica['state'] == ReplicaState.BEING_DELETED:
2352
- query = query.filter_by(lock_cnt=0)
2353
2545
  # Exclude replicas use as sources
2354
- stmt = exists(1).where(and_(models.RSEFileAssociation.scope == models.Source.scope,
2355
- models.RSEFileAssociation.name == models.Source.name,
2356
- models.RSEFileAssociation.rse_id == models.Source.rse_id))
2357
- query = query.filter(not_(stmt))
2546
+ stmt = stmt.where(
2547
+ and_(models.RSEFileAssociation.lock_cnt == 0,
2548
+ not_(exists(select(1)
2549
+ .where(and_(models.RSEFileAssociation.scope == models.Source.scope,
2550
+ models.RSEFileAssociation.name == models.Source.name,
2551
+ models.RSEFileAssociation.rse_id == models.Source.rse_id)))))
2552
+ )
2358
2553
  values['tombstone'] = OBSOLETE
2359
2554
  elif replica['state'] == ReplicaState.AVAILABLE:
2360
2555
  rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session)
2361
- query1 = session.query(models.BadReplicas).filter_by(state=BadFilesStatus.BAD, rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'])
2362
- if query1.count():
2363
- query1.update({'state': BadFilesStatus.RECOVERED, 'updated_at': datetime.utcnow()}, synchronize_session=False)
2556
+ stmt_bad_replicas = select(
2557
+ func.count()
2558
+ ).select_from(
2559
+ models.BadReplica
2560
+ ).where(
2561
+ and_(models.BadReplica.state == BadFilesStatus.BAD,
2562
+ models.BadReplica.rse_id == replica['rse_id'],
2563
+ models.BadReplica.scope == replica['scope'],
2564
+ models.BadReplica.name == replica['name'])
2565
+ )
2566
+
2567
+ if session.execute(stmt_bad_replicas).scalar():
2568
+ update_stmt = update(
2569
+ models.BadReplica
2570
+ ).where(
2571
+ and_(models.BadReplica.state == BadFilesStatus.BAD,
2572
+ models.BadReplica.rse_id == replica['rse_id'],
2573
+ models.BadReplica.scope == replica['scope'],
2574
+ models.BadReplica.name == replica['name'])
2575
+ ).values({
2576
+ models.BadReplica.state: BadFilesStatus.RECOVERED,
2577
+ models.BadReplica.updated_at: datetime.utcnow()
2578
+ }).execution_options(
2579
+ synchronize_session=False
2580
+ )
2581
+ session.execute(update_stmt)
2364
2582
  elif replica['state'] == ReplicaState.UNAVAILABLE:
2365
2583
  rucio.core.lock.failed_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'],
2366
2584
  error_message=replica.get('error_message', None),
@@ -2368,12 +2586,27 @@ def update_replicas_states(replicas, nowait=False, *, session: "Session"):
2368
2586
  broken_message=replica.get('broken_message', None),
2369
2587
  nowait=nowait, session=session)
2370
2588
  elif replica['state'] == ReplicaState.TEMPORARY_UNAVAILABLE:
2371
- query = query.filter(or_(models.RSEFileAssociation.state == ReplicaState.AVAILABLE, models.RSEFileAssociation.state == ReplicaState.TEMPORARY_UNAVAILABLE))
2589
+ stmt = stmt.where(
2590
+ models.RSEFileAssociation.state.in_([ReplicaState.AVAILABLE,
2591
+ ReplicaState.TEMPORARY_UNAVAILABLE])
2592
+ )
2372
2593
 
2373
2594
  if 'path' in replica and replica['path']:
2374
2595
  values['path'] = replica['path']
2375
2596
 
2376
- if not query.update(values, synchronize_session=False):
2597
+ update_stmt = update(
2598
+ models.RSEFileAssociation
2599
+ ).where(
2600
+ and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
2601
+ models.RSEFileAssociation.scope == replica['scope'],
2602
+ models.RSEFileAssociation.name == replica['name'])
2603
+ ).values(
2604
+ values
2605
+ ).execution_options(
2606
+ synchronize_session=False
2607
+ )
2608
+
2609
+ if not session.execute(update_stmt).rowcount:
2377
2610
  if 'rse' not in replica:
2378
2611
  replica['rse'] = get_rse_name(rse_id=replica['rse_id'], session=session)
2379
2612
  raise exception.UnsupportedOperation('State %(state)s for replica %(scope)s:%(name)s on %(rse)s cannot be updated' % replica)
@@ -2393,32 +2626,68 @@ def touch_replica(replica, *, session: "Session"):
2393
2626
  try:
2394
2627
  accessed_at, none_value = replica.get('accessed_at') or datetime.utcnow(), None
2395
2628
 
2396
- session.query(models.RSEFileAssociation).\
2397
- filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\
2398
- with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
2399
- with_for_update(nowait=True).one()
2400
-
2401
- stmt = update(models.RSEFileAssociation).\
2402
- filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\
2403
- prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle').\
2404
- execution_options(synchronize_session=False).\
2405
- values(accessed_at=accessed_at,
2406
- tombstone=case((and_(models.RSEFileAssociation.tombstone != none_value,
2407
- models.RSEFileAssociation.tombstone != OBSOLETE),
2408
- accessed_at),
2409
- else_=models.RSEFileAssociation.tombstone))
2629
+ stmt = select(
2630
+ models.RSEFileAssociation
2631
+ ).with_hint(
2632
+ models.RSEFileAssociation,
2633
+ 'INDEX(REPLICAS REPLICAS_PK)',
2634
+ 'oracle'
2635
+ ).where(
2636
+ and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
2637
+ models.RSEFileAssociation.scope == replica['scope'],
2638
+ models.RSEFileAssociation.name == replica['name'])
2639
+ ).with_for_update(
2640
+ nowait=True
2641
+ )
2642
+ session.execute(stmt).one()
2643
+
2644
+ stmt = update(
2645
+ models.RSEFileAssociation
2646
+ ).where(
2647
+ and_(models.RSEFileAssociation.rse_id == replica['rse_id'],
2648
+ models.RSEFileAssociation.scope == replica['scope'],
2649
+ models.RSEFileAssociation.name == replica['name'])
2650
+ ).prefix_with(
2651
+ '/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'
2652
+ ).values({
2653
+ models.RSEFileAssociation.accessed_at: accessed_at,
2654
+ models.RSEFileAssociation.tombstone: case(
2655
+ (models.RSEFileAssociation.tombstone.not_in([OBSOLETE, none_value]),
2656
+ accessed_at),
2657
+ else_=models.RSEFileAssociation.tombstone)
2658
+ }).execution_options(
2659
+ synchronize_session=False
2660
+ )
2410
2661
  session.execute(stmt)
2411
2662
 
2412
- session.query(models.DataIdentifier).\
2413
- filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\
2414
- with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
2415
- with_for_update(nowait=True).one()
2663
+ stmt = select(
2664
+ models.DataIdentifier
2665
+ ).with_hint(
2666
+ models.DataIdentifier,
2667
+ 'INDEX(DIDS DIDS_PK)',
2668
+ 'oracle'
2669
+ ).where(
2670
+ and_(models.DataIdentifier.scope == replica['scope'],
2671
+ models.DataIdentifier.name == replica['name'],
2672
+ models.DataIdentifier.did_type == DIDType.FILE)
2673
+ ).with_for_update(
2674
+ nowait=True
2675
+ )
2676
+ session.execute(stmt).one()
2416
2677
 
2417
- stmt = update(models.DataIdentifier).\
2418
- filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\
2419
- prefix_with("/*+ INDEX(DIDS DIDS_PK) */", dialect='oracle').\
2420
- execution_options(synchronize_session=False).\
2421
- values(accessed_at=accessed_at)
2678
+ stmt = update(
2679
+ models.DataIdentifier
2680
+ ).where(
2681
+ and_(models.DataIdentifier.scope == replica['scope'],
2682
+ models.DataIdentifier.name == replica['name'],
2683
+ models.DataIdentifier.did_type == DIDType.FILE)
2684
+ ).prefix_with(
2685
+ '/*+ INDEX(DIDS DIDS_PK) */', dialect='oracle'
2686
+ ).values({
2687
+ models.DataIdentifier.accessed_at: accessed_at
2688
+ }).execution_options(
2689
+ synchronize_session=False
2690
+ )
2422
2691
  session.execute(stmt)
2423
2692
 
2424
2693
  except DatabaseError:
@@ -2456,21 +2725,27 @@ def get_and_lock_file_replicas(scope, name, nowait=False, restrict_rses=None, *,
2456
2725
  :returns: List of SQLAlchemy Replica Objects
2457
2726
  """
2458
2727
 
2459
- query = session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name).filter(models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
2460
- if restrict_rses is not None:
2461
- if len(restrict_rses) < 10:
2462
- rse_clause = []
2463
- for rse_id in restrict_rses:
2464
- rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
2465
- if rse_clause:
2466
- query = query.filter(or_(*rse_clause))
2467
- return query.with_for_update(nowait=nowait).all()
2728
+ stmt = select(
2729
+ models.RSEFileAssociation
2730
+ ).where(
2731
+ and_(models.RSEFileAssociation.scope == scope,
2732
+ models.RSEFileAssociation.name == name,
2733
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
2734
+ ).with_for_update(
2735
+ nowait=nowait
2736
+ )
2737
+ if restrict_rses is not None and len(restrict_rses) < 10:
2738
+ rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
2739
+ if rse_clause:
2740
+ stmt = stmt.where(or_(*rse_clause))
2741
+
2742
+ return session.execute(stmt).scalars().all()
2468
2743
 
2469
2744
 
2470
2745
  @transactional_session
2471
2746
  def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
2472
2747
  """
2473
- Get soruce replicas for a specific scope:name.
2748
+ Get source replicas for a specific scope:name.
2474
2749
 
2475
2750
  :param scope: The scope of the did.
2476
2751
  :param name: The name of the did.
@@ -2479,15 +2754,21 @@ def get_source_replicas(scope, name, source_rses=None, *, session: "Session"):
2479
2754
  :returns: List of SQLAlchemy Replica Objects
2480
2755
  """
2481
2756
 
2482
- query = session.query(models.RSEFileAssociation.rse_id).filter_by(scope=scope, name=name).filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
2757
+ stmt = select(
2758
+ models.RSEFileAssociation.rse_id
2759
+ ).where(
2760
+ and_(models.RSEFileAssociation.scope == scope,
2761
+ models.RSEFileAssociation.name == name,
2762
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
2763
+ )
2483
2764
  if source_rses:
2484
2765
  if len(source_rses) < 10:
2485
2766
  rse_clause = []
2486
2767
  for rse_id in source_rses:
2487
2768
  rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
2488
2769
  if rse_clause:
2489
- query = query.filter(or_(*rse_clause))
2490
- return [a[0] for a in query.all()]
2770
+ stmt = stmt.where(or_(*rse_clause))
2771
+ return session.execute(stmt).scalars().all()
2491
2772
 
2492
2773
 
2493
2774
  @transactional_session
@@ -2507,24 +2788,39 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
2507
2788
  :returns: (files in dataset, replicas in dataset)
2508
2789
  """
2509
2790
  files, replicas = {}, {}
2510
- if session.bind.dialect.name == 'postgresql':
2511
- # Get content
2512
- content_query = session.query(models.DataIdentifierAssociation.child_scope,
2513
- models.DataIdentifierAssociation.child_name,
2514
- models.DataIdentifierAssociation.bytes,
2515
- models.DataIdentifierAssociation.md5,
2516
- models.DataIdentifierAssociation.adler32).\
2517
- with_hint(models.DataIdentifierAssociation,
2518
- "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
2519
- 'oracle').\
2520
- filter(models.DataIdentifierAssociation.scope == scope,
2521
- models.DataIdentifierAssociation.name == name)
2522
2791
 
2792
+ base_stmt = select(
2793
+ models.DataIdentifierAssociation.child_scope,
2794
+ models.DataIdentifierAssociation.child_name,
2795
+ models.DataIdentifierAssociation.bytes,
2796
+ models.DataIdentifierAssociation.md5,
2797
+ models.DataIdentifierAssociation.adler32,
2798
+ ).where(
2799
+ and_(models.DataIdentifierAssociation.scope == scope,
2800
+ models.DataIdentifierAssociation.name == name)
2801
+ )
2802
+
2803
+ stmt = base_stmt.add_columns(
2804
+ models.RSEFileAssociation
2805
+ ).where(
2806
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2807
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2808
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
2809
+ )
2810
+
2811
+ rse_clause = [true()]
2812
+ if restrict_rses is not None and len(restrict_rses) < 10:
2813
+ rse_clause = [models.RSEFileAssociation.rse_id == rse_id for rse_id in restrict_rses]
2814
+
2815
+ if session.bind.dialect.name == 'postgresql':
2523
2816
  if total_threads and total_threads > 1:
2524
- content_query = filter_thread_work(session=session, query=content_query, total_threads=total_threads,
2525
- thread_id=thread_id, hash_variable='child_name')
2817
+ base_stmt = filter_thread_work(session=session,
2818
+ query=base_stmt,
2819
+ total_threads=total_threads,
2820
+ thread_id=thread_id,
2821
+ hash_variable='child_name')
2526
2822
 
2527
- for child_scope, child_name, bytes_, md5, adler32 in content_query.yield_per(1000):
2823
+ for child_scope, child_name, bytes_, md5, adler32 in session.execute(base_stmt).yield_per(1000):
2528
2824
  files[(child_scope, child_name)] = {'scope': child_scope,
2529
2825
  'name': child_name,
2530
2826
  'bytes': bytes_,
@@ -2532,92 +2828,35 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
2532
2828
  'adler32': adler32}
2533
2829
  replicas[(child_scope, child_name)] = []
2534
2830
 
2535
- # Get replicas and lock them
2536
- query = session.query(models.DataIdentifierAssociation.child_scope,
2537
- models.DataIdentifierAssociation.child_name,
2538
- models.DataIdentifierAssociation.bytes,
2539
- models.DataIdentifierAssociation.md5,
2540
- models.DataIdentifierAssociation.adler32,
2541
- models.RSEFileAssociation)\
2542
- .with_hint(models.DataIdentifierAssociation,
2543
- "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
2544
- 'oracle')\
2545
- .filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2546
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2547
- models.RSEFileAssociation.state != ReplicaState.BEING_DELETED))\
2548
- .filter(models.DataIdentifierAssociation.scope == scope,
2549
- models.DataIdentifierAssociation.name == name)
2550
-
2551
- if restrict_rses is not None:
2552
- if len(restrict_rses) < 10:
2553
- rse_clause = []
2554
- for rse_id in restrict_rses:
2555
- rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
2556
- if rse_clause:
2557
- query = session.query(models.DataIdentifierAssociation.child_scope,
2558
- models.DataIdentifierAssociation.child_name,
2559
- models.DataIdentifierAssociation.bytes,
2560
- models.DataIdentifierAssociation.md5,
2561
- models.DataIdentifierAssociation.adler32,
2562
- models.RSEFileAssociation)\
2563
- .with_hint(models.DataIdentifierAssociation,
2564
- "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
2565
- 'oracle')\
2566
- .filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2567
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2568
- models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
2569
- or_(*rse_clause)))\
2570
- .filter(models.DataIdentifierAssociation.scope == scope,
2571
- models.DataIdentifierAssociation.name == name)
2572
-
2831
+ stmt = stmt.where(or_(*rse_clause))
2573
2832
  else:
2574
- query = session.query(models.DataIdentifierAssociation.child_scope,
2575
- models.DataIdentifierAssociation.child_name,
2576
- models.DataIdentifierAssociation.bytes,
2577
- models.DataIdentifierAssociation.md5,
2578
- models.DataIdentifierAssociation.adler32,
2579
- models.RSEFileAssociation)\
2580
- .with_hint(models.DataIdentifierAssociation,
2581
- "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
2582
- 'oracle') \
2583
- .with_hint(models.RSEFileAssociation, "INDEX(REPLICAS REPLICAS_PK)", 'oracle')\
2584
- .outerjoin(models.RSEFileAssociation,
2585
- and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2586
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2587
- models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)).\
2588
- filter(models.DataIdentifierAssociation.scope == scope,
2589
- models.DataIdentifierAssociation.name == name)
2590
-
2591
- if restrict_rses is not None:
2592
- if len(restrict_rses) < 10:
2593
- rse_clause = []
2594
- for rse_id in restrict_rses:
2595
- rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
2596
- if rse_clause:
2597
- query = session.query(models.DataIdentifierAssociation.child_scope,
2598
- models.DataIdentifierAssociation.child_name,
2599
- models.DataIdentifierAssociation.bytes,
2600
- models.DataIdentifierAssociation.md5,
2601
- models.DataIdentifierAssociation.adler32,
2602
- models.RSEFileAssociation)\
2603
- .with_hint(models.DataIdentifierAssociation,
2604
- "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)",
2605
- 'oracle')\
2606
- .outerjoin(models.RSEFileAssociation,
2607
- and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2608
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2609
- models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
2610
- or_(*rse_clause)))\
2611
- .filter(models.DataIdentifierAssociation.scope == scope,
2612
- models.DataIdentifierAssociation.name == name)
2833
+ stmt = base_stmt.add_columns(
2834
+ models.RSEFileAssociation
2835
+ ).with_hint(
2836
+ models.DataIdentifierAssociation,
2837
+ 'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
2838
+ 'oracle'
2839
+ ).outerjoin(
2840
+ models.RSEFileAssociation,
2841
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2842
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2843
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
2844
+ or_(*rse_clause))
2845
+ )
2613
2846
 
2614
2847
  if total_threads and total_threads > 1:
2615
- query = filter_thread_work(session=session, query=query, total_threads=total_threads,
2616
- thread_id=thread_id, hash_variable='child_name')
2617
-
2618
- query = query.with_for_update(nowait=nowait, of=models.RSEFileAssociation.lock_cnt)
2848
+ stmt = filter_thread_work(session=session,
2849
+ query=stmt,
2850
+ total_threads=total_threads,
2851
+ thread_id=thread_id,
2852
+ hash_variable='child_name')
2853
+
2854
+ stmt = stmt.with_for_update(
2855
+ nowait=nowait,
2856
+ of=models.RSEFileAssociation.lock_cnt
2857
+ )
2619
2858
 
2620
- for child_scope, child_name, bytes_, md5, adler32, replica in query.yield_per(1000):
2859
+ for child_scope, child_name, bytes_, md5, adler32, replica in session.execute(stmt).yield_per(1000):
2621
2860
  if (child_scope, child_name) not in files:
2622
2861
  files[(child_scope, child_name)] = {'scope': child_scope,
2623
2862
  'name': child_name,
@@ -2651,15 +2890,23 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
2651
2890
  :param session: The db session in use.
2652
2891
  :returns: (files in dataset, replicas in dataset)
2653
2892
  """
2654
- query = session.query(models.DataIdentifierAssociation.child_scope,
2655
- models.DataIdentifierAssociation.child_name,
2656
- models.RSEFileAssociation.rse_id)\
2657
- .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
2658
- .outerjoin(models.RSEFileAssociation,
2659
- and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2660
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2661
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE)).\
2662
- filter(models.DataIdentifierAssociation.scope == scope, models.DataIdentifierAssociation.name == name)
2893
+ stmt = select(
2894
+ models.DataIdentifierAssociation.child_scope,
2895
+ models.DataIdentifierAssociation.child_name,
2896
+ models.RSEFileAssociation.rse_id
2897
+ ).with_hint(
2898
+ models.DataIdentifierAssociation,
2899
+ 'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
2900
+ 'oracle'
2901
+ ).outerjoin(
2902
+ models.RSEFileAssociation,
2903
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2904
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2905
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
2906
+ ).where(
2907
+ and_(models.DataIdentifierAssociation.scope == scope,
2908
+ models.DataIdentifierAssociation.name == name)
2909
+ )
2663
2910
 
2664
2911
  if source_rses:
2665
2912
  if len(source_rses) < 10:
@@ -2667,25 +2914,34 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None,
2667
2914
  for rse_id in source_rses:
2668
2915
  rse_clause.append(models.RSEFileAssociation.rse_id == rse_id)
2669
2916
  if rse_clause:
2670
- query = session.query(models.DataIdentifierAssociation.child_scope,
2671
- models.DataIdentifierAssociation.child_name,
2672
- models.RSEFileAssociation.rse_id)\
2673
- .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
2674
- .outerjoin(models.RSEFileAssociation,
2675
- and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2676
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2677
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
2678
- or_(*rse_clause)))\
2679
- .filter(models.DataIdentifierAssociation.scope == scope,
2680
- models.DataIdentifierAssociation.name == name)
2681
-
2917
+ stmt = select(
2918
+ models.DataIdentifierAssociation.child_scope,
2919
+ models.DataIdentifierAssociation.child_name,
2920
+ models.RSEFileAssociation.rse_id
2921
+ ).with_hint(
2922
+ models.DataIdentifierAssociation,
2923
+ 'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
2924
+ 'oracle'
2925
+ ).outerjoin(
2926
+ models.RSEFileAssociation,
2927
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
2928
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
2929
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
2930
+ or_(*rse_clause))
2931
+ ).where(
2932
+ and_(models.DataIdentifierAssociation.scope == scope,
2933
+ models.DataIdentifierAssociation.name == name)
2934
+ )
2682
2935
  if total_threads and total_threads > 1:
2683
- query = filter_thread_work(session=session, query=query, total_threads=total_threads,
2684
- thread_id=thread_id, hash_variable='child_name')
2936
+ stmt = filter_thread_work(session=session,
2937
+ query=stmt,
2938
+ total_threads=total_threads,
2939
+ thread_id=thread_id,
2940
+ hash_variable='child_name')
2685
2941
 
2686
2942
  replicas = {}
2687
2943
 
2688
- for child_scope, child_name, rse_id in query:
2944
+ for child_scope, child_name, rse_id in session.execute(stmt):
2689
2945
 
2690
2946
  if (child_scope, child_name) in replicas:
2691
2947
  if rse_id:
@@ -2707,8 +2963,18 @@ def get_replica_atime(replica, *, session: "Session"):
2707
2963
 
2708
2964
  :returns: A datetime timestamp with the last access time.
2709
2965
  """
2710
- return session.query(models.RSEFileAssociation.accessed_at).filter_by(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id']).\
2711
- with_hint(models.RSEFileAssociation, text="INDEX(REPLICAS REPLICAS_PK)", dialect_name='oracle').one()[0]
2966
+ stmt = select(
2967
+ models.RSEFileAssociation.accessed_at
2968
+ ).with_hint(
2969
+ models.RSEFileAssociation,
2970
+ 'INDEX(REPLICAS REPLICAS_PK)',
2971
+ 'oracle'
2972
+ ).where(
2973
+ and_(models.RSEFileAssociation.scope == replica['scope'],
2974
+ models.RSEFileAssociation.name == replica['name'],
2975
+ models.RSEFileAssociation.rse_id == replica['rse_id'])
2976
+ )
2977
+ return session.execute(stmt).scalar_one()
2712
2978
 
2713
2979
 
2714
2980
  @transactional_session
@@ -2725,8 +2991,18 @@ def touch_collection_replicas(collection_replicas, *, session: "Session"):
2725
2991
  now = datetime.utcnow()
2726
2992
  for collection_replica in collection_replicas:
2727
2993
  try:
2728
- session.query(models.CollectionReplica).filter_by(scope=collection_replica['scope'], name=collection_replica['name'], rse_id=collection_replica['rse_id']).\
2729
- update({'accessed_at': collection_replica.get('accessed_at') or now}, synchronize_session=False)
2994
+ stmt = update(
2995
+ models.CollectionReplica
2996
+ ).where(
2997
+ and_(models.CollectionReplica.scope == collection_replica['scope'],
2998
+ models.CollectionReplica.name == collection_replica['name'],
2999
+ models.CollectionReplica.rse_id == collection_replica['rse_id'])
3000
+ ).values({
3001
+ models.CollectionReplica.accessed_at: collection_replica.get('accessed_at') or now
3002
+ }).execution_options(
3003
+ synchronize_session=False
3004
+ )
3005
+ session.execute(stmt)
2730
3006
  except DatabaseError:
2731
3007
  return False
2732
3008
 
@@ -2746,138 +3022,158 @@ def list_dataset_replicas(scope, name, deep=False, *, session: "Session"):
2746
3022
  """
2747
3023
 
2748
3024
  if not deep:
2749
- query = session.query(models.CollectionReplica.scope,
2750
- models.CollectionReplica.name,
2751
- models.RSE.rse,
2752
- models.CollectionReplica.rse_id,
2753
- models.CollectionReplica.bytes,
2754
- models.CollectionReplica.length,
2755
- models.CollectionReplica.available_bytes,
2756
- models.CollectionReplica.available_replicas_cnt.label("available_length"),
2757
- models.CollectionReplica.state,
2758
- models.CollectionReplica.created_at,
2759
- models.CollectionReplica.updated_at,
2760
- models.CollectionReplica.accessed_at)\
2761
- .filter_by(scope=scope, name=name, did_type=DIDType.DATASET)\
2762
- .filter(models.CollectionReplica.rse_id == models.RSE.id)\
2763
- .filter(models.RSE.deleted == false())
2764
-
2765
- for row in query:
3025
+ stmt = select(
3026
+ models.CollectionReplica.scope,
3027
+ models.CollectionReplica.name,
3028
+ models.RSE.rse,
3029
+ models.CollectionReplica.rse_id,
3030
+ models.CollectionReplica.bytes,
3031
+ models.CollectionReplica.length,
3032
+ models.CollectionReplica.available_bytes,
3033
+ models.CollectionReplica.available_replicas_cnt.label("available_length"),
3034
+ models.CollectionReplica.state,
3035
+ models.CollectionReplica.created_at,
3036
+ models.CollectionReplica.updated_at,
3037
+ models.CollectionReplica.accessed_at
3038
+ ).where(
3039
+ and_(models.CollectionReplica.scope == scope,
3040
+ models.CollectionReplica.name == name,
3041
+ models.CollectionReplica.did_type == DIDType.DATASET,
3042
+ models.CollectionReplica.rse_id == models.RSE.id,
3043
+ models.RSE.deleted == false())
3044
+ )
3045
+
3046
+ for row in session.execute(stmt).all():
2766
3047
  yield row._asdict()
2767
3048
 
2768
3049
  else:
3050
+ # Find maximum values
3051
+ stmt = select(
3052
+ func.sum(models.DataIdentifierAssociation.bytes).label("bytes"),
3053
+ func.count().label("length")
3054
+ ).select_from(
3055
+ models.DataIdentifierAssociation
3056
+ ).with_hint(
3057
+ models.DataIdentifierAssociation,
3058
+ 'INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
3059
+ 'oracle'
3060
+ ).where(
3061
+ and_(models.DataIdentifierAssociation.scope == scope,
3062
+ models.DataIdentifierAssociation.name == name)
3063
+ )
3064
+
3065
+ bytes_, length = session.execute(stmt).one()
3066
+ bytes_ = bytes_ or 0
3067
+
3068
+ # Find archives that contain files of the requested dataset
3069
+ sub_query_stmt = select(
3070
+ models.DataIdentifierAssociation.scope.label('dataset_scope'),
3071
+ models.DataIdentifierAssociation.name.label('dataset_name'),
3072
+ models.DataIdentifierAssociation.bytes.label('file_bytes'),
3073
+ models.ConstituentAssociation.child_scope.label('file_scope'),
3074
+ models.ConstituentAssociation.child_name.label('file_name'),
3075
+ models.RSEFileAssociation.scope.label('replica_scope'),
3076
+ models.RSEFileAssociation.name.label('replica_name'),
3077
+ models.RSE.rse,
3078
+ models.RSE.id.label('rse_id'),
3079
+ models.RSEFileAssociation.created_at,
3080
+ models.RSEFileAssociation.accessed_at,
3081
+ models.RSEFileAssociation.updated_at
3082
+ ).where(
3083
+ and_(models.DataIdentifierAssociation.scope == scope,
3084
+ models.DataIdentifierAssociation.name == name,
3085
+ models.ConstituentAssociation.child_scope == models.DataIdentifierAssociation.child_scope,
3086
+ models.ConstituentAssociation.child_name == models.DataIdentifierAssociation.child_name,
3087
+ models.ConstituentAssociation.scope == models.RSEFileAssociation.scope,
3088
+ models.ConstituentAssociation.name == models.RSEFileAssociation.name,
3089
+ models.RSEFileAssociation.rse_id == models.RSE.id,
3090
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
3091
+ models.RSE.deleted == false())
3092
+ ).subquery()
3093
+
3094
+ # Count the metrics
3095
+ group_query_stmt = select(
3096
+ sub_query_stmt.c.dataset_scope,
3097
+ sub_query_stmt.c.dataset_name,
3098
+ sub_query_stmt.c.file_scope,
3099
+ sub_query_stmt.c.file_name,
3100
+ sub_query_stmt.c.rse_id,
3101
+ sub_query_stmt.c.rse,
3102
+ func.sum(sub_query_stmt.c.file_bytes).label('file_bytes'),
3103
+ func.min(sub_query_stmt.c.created_at).label('created_at'),
3104
+ func.max(sub_query_stmt.c.updated_at).label('updated_at'),
3105
+ func.max(sub_query_stmt.c.accessed_at).label('accessed_at')
3106
+ ).group_by(
3107
+ sub_query_stmt.c.dataset_scope,
3108
+ sub_query_stmt.c.dataset_name,
3109
+ sub_query_stmt.c.file_scope,
3110
+ sub_query_stmt.c.file_name,
3111
+ sub_query_stmt.c.rse_id,
3112
+ sub_query_stmt.c.rse
3113
+ ).subquery()
3114
+
3115
+ # Bring it in the same column state as the non-archive query
3116
+ full_query_stmt = select(
3117
+ group_query_stmt.c.dataset_scope.label('scope'),
3118
+ group_query_stmt.c.dataset_name.label('name'),
3119
+ group_query_stmt.c.rse_id,
3120
+ group_query_stmt.c.rse,
3121
+ func.sum(group_query_stmt.c.file_bytes).label('available_bytes'),
3122
+ func.count().label('available_length'),
3123
+ func.min(group_query_stmt.c.created_at).label('created_at'),
3124
+ func.max(group_query_stmt.c.updated_at).label('updated_at'),
3125
+ func.max(group_query_stmt.c.accessed_at).label('accessed_at')
3126
+ ).group_by(
3127
+ group_query_stmt.c.dataset_scope,
3128
+ group_query_stmt.c.dataset_name,
3129
+ group_query_stmt.c.rse_id,
3130
+ group_query_stmt.c.rse
3131
+ )
3132
+
3133
+ # Find the non-archive dataset replicas
3134
+ sub_query_stmt = select(
3135
+ models.DataIdentifierAssociation.scope,
3136
+ models.DataIdentifierAssociation.name,
3137
+ models.RSEFileAssociation.rse_id,
3138
+ func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
3139
+ func.count().label("available_length"),
3140
+ func.min(models.RSEFileAssociation.created_at).label("created_at"),
3141
+ func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
3142
+ func.max(models.RSEFileAssociation.accessed_at).label("accessed_at")
3143
+ ).with_hint(
3144
+ models.DataIdentifierAssociation,
3145
+ 'INDEX_RS_ASC(CONTENTS CONTENTS_PK) INDEX_RS_ASC(REPLICAS REPLICAS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)',
3146
+ 'oracle'
3147
+ ).where(
3148
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
3149
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
3150
+ models.DataIdentifierAssociation.scope == scope,
3151
+ models.DataIdentifierAssociation.name == name,
3152
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE)
3153
+ ).group_by(
3154
+ models.DataIdentifierAssociation.scope,
3155
+ models.DataIdentifierAssociation.name,
3156
+ models.RSEFileAssociation.rse_id
3157
+ ).subquery()
2769
3158
 
2770
- # find maximum values
2771
- content_query = session\
2772
- .query(func.sum(models.DataIdentifierAssociation.bytes).label("bytes"),
2773
- func.count().label("length"))\
2774
- .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
2775
- .filter(models.DataIdentifierAssociation.scope == scope)\
2776
- .filter(models.DataIdentifierAssociation.name == name)
2777
-
2778
- bytes_, length = 0, 0
2779
- for row in content_query:
2780
- bytes_, length = row.bytes, row.length
2781
-
2782
- # find archives that contain files of the requested dataset
2783
- sub_query_archives = session\
2784
- .query(models.DataIdentifierAssociation.scope.label('dataset_scope'),
2785
- models.DataIdentifierAssociation.name.label('dataset_name'),
2786
- models.DataIdentifierAssociation.bytes.label('file_bytes'),
2787
- models.ConstituentAssociation.child_scope.label('file_scope'),
2788
- models.ConstituentAssociation.child_name.label('file_name'),
2789
- models.RSEFileAssociation.scope.label('replica_scope'),
2790
- models.RSEFileAssociation.name.label('replica_name'),
2791
- models.RSE.rse,
2792
- models.RSE.id.label('rse_id'),
2793
- models.RSEFileAssociation.created_at,
2794
- models.RSEFileAssociation.accessed_at,
2795
- models.RSEFileAssociation.updated_at)\
2796
- .filter(models.DataIdentifierAssociation.scope == scope)\
2797
- .filter(models.DataIdentifierAssociation.name == name)\
2798
- .filter(models.ConstituentAssociation.child_scope == models.DataIdentifierAssociation.child_scope)\
2799
- .filter(models.ConstituentAssociation.child_name == models.DataIdentifierAssociation.child_name)\
2800
- .filter(models.ConstituentAssociation.scope == models.RSEFileAssociation.scope)\
2801
- .filter(models.ConstituentAssociation.name == models.RSEFileAssociation.name)\
2802
- .filter(models.RSEFileAssociation.rse_id == models.RSE.id)\
2803
- .filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
2804
- .filter(models.RSE.deleted == false())\
2805
- .subquery()
2806
-
2807
- # count the metrics
2808
- group_query_archives = session\
2809
- .query(sub_query_archives.c.dataset_scope,
2810
- sub_query_archives.c.dataset_name,
2811
- sub_query_archives.c.file_scope,
2812
- sub_query_archives.c.file_name,
2813
- sub_query_archives.c.rse_id,
2814
- sub_query_archives.c.rse,
2815
- func.sum(sub_query_archives.c.file_bytes).label('file_bytes'),
2816
- func.min(sub_query_archives.c.created_at).label('created_at'),
2817
- func.max(sub_query_archives.c.updated_at).label('updated_at'),
2818
- func.max(sub_query_archives.c.accessed_at).label('accessed_at'))\
2819
- .group_by(sub_query_archives.c.dataset_scope,
2820
- sub_query_archives.c.dataset_name,
2821
- sub_query_archives.c.file_scope,
2822
- sub_query_archives.c.file_name,
2823
- sub_query_archives.c.rse_id,
2824
- sub_query_archives.c.rse)\
2825
- .subquery()
2826
-
2827
- # bring it in the same column state as the non-archive query
2828
- full_query_archives = session\
2829
- .query(group_query_archives.c.dataset_scope.label('scope'),
2830
- group_query_archives.c.dataset_name.label('name'),
2831
- group_query_archives.c.rse_id,
2832
- group_query_archives.c.rse,
2833
- func.sum(group_query_archives.c.file_bytes).label('available_bytes'),
2834
- func.count().label('available_length'),
2835
- func.min(group_query_archives.c.created_at).label('created_at'),
2836
- func.max(group_query_archives.c.updated_at).label('updated_at'),
2837
- func.max(group_query_archives.c.accessed_at).label('accessed_at'))\
2838
- .group_by(group_query_archives.c.dataset_scope,
2839
- group_query_archives.c.dataset_name,
2840
- group_query_archives.c.rse_id,
2841
- group_query_archives.c.rse)
2842
-
2843
- # find the non-archive dataset replicas
2844
- sub_query = session\
2845
- .query(models.DataIdentifierAssociation.scope,
2846
- models.DataIdentifierAssociation.name,
2847
- models.RSEFileAssociation.rse_id,
2848
- func.sum(models.RSEFileAssociation.bytes).label("available_bytes"),
2849
- func.count().label("available_length"),
2850
- func.min(models.RSEFileAssociation.created_at).label("created_at"),
2851
- func.max(models.RSEFileAssociation.updated_at).label("updated_at"),
2852
- func.max(models.RSEFileAssociation.accessed_at).label("accessed_at"))\
2853
- .with_hint(models.DataIdentifierAssociation, "INDEX_RS_ASC(CONTENTS CONTENTS_PK) INDEX_RS_ASC(REPLICAS REPLICAS_PK) NO_INDEX_FFS(CONTENTS CONTENTS_PK)", 'oracle')\
2854
- .filter(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope)\
2855
- .filter(models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name)\
2856
- .filter(models.DataIdentifierAssociation.scope == scope)\
2857
- .filter(models.DataIdentifierAssociation.name == name)\
2858
- .filter(models.RSEFileAssociation.state == ReplicaState.AVAILABLE)\
2859
- .group_by(models.DataIdentifierAssociation.scope,
2860
- models.DataIdentifierAssociation.name,
2861
- models.RSEFileAssociation.rse_id)\
2862
- .subquery()
2863
-
2864
- query = session\
2865
- .query(sub_query.c.scope,
2866
- sub_query.c.name,
2867
- sub_query.c.rse_id,
2868
- models.RSE.rse,
2869
- sub_query.c.available_bytes,
2870
- sub_query.c.available_length,
2871
- sub_query.c.created_at,
2872
- sub_query.c.updated_at,
2873
- sub_query.c.accessed_at)\
2874
- .filter(models.RSE.id == sub_query.c.rse_id)\
2875
- .filter(models.RSE.deleted == false())
2876
-
2877
- # join everything together
2878
- final_query = query.union_all(full_query_archives)
2879
-
2880
- for row in final_query.all():
3159
+ stmt = select(
3160
+ sub_query_stmt.c.scope,
3161
+ sub_query_stmt.c.name,
3162
+ sub_query_stmt.c.rse_id,
3163
+ models.RSE.rse,
3164
+ sub_query_stmt.c.available_bytes,
3165
+ sub_query_stmt.c.available_length,
3166
+ sub_query_stmt.c.created_at,
3167
+ sub_query_stmt.c.updated_at,
3168
+ sub_query_stmt.c.accessed_at
3169
+ ).where(
3170
+ and_(sub_query_stmt.c.rse_id == models.RSE.id,
3171
+ models.RSE.deleted == false())
3172
+ )
3173
+
3174
+ # Join everything together
3175
+ final_stmt = stmt.union_all(full_query_stmt)
3176
+ for row in session.execute(final_stmt).all():
2881
3177
  replica = row._asdict()
2882
3178
  replica['length'], replica['bytes'] = length, bytes_
2883
3179
  if replica['length'] == row.available_length:
@@ -2905,23 +3201,27 @@ def list_dataset_replicas_bulk(names_by_intscope, *, session: "Session"):
2905
3201
  try:
2906
3202
  # chunk size refers to the number of different scopes, see above
2907
3203
  for chunk in chunks(condition, 10):
2908
- query = session.query(models.CollectionReplica.scope,
2909
- models.CollectionReplica.name,
2910
- models.RSE.rse,
2911
- models.CollectionReplica.rse_id,
2912
- models.CollectionReplica.bytes,
2913
- models.CollectionReplica.length,
2914
- models.CollectionReplica.available_bytes,
2915
- models.CollectionReplica.available_replicas_cnt.label("available_length"),
2916
- models.CollectionReplica.state,
2917
- models.CollectionReplica.created_at,
2918
- models.CollectionReplica.updated_at,
2919
- models.CollectionReplica.accessed_at) \
2920
- .filter(models.CollectionReplica.did_type == DIDType.DATASET) \
2921
- .filter(models.CollectionReplica.rse_id == models.RSE.id) \
2922
- .filter(or_(*chunk)) \
2923
- .filter(models.RSE.deleted == false())
2924
- for row in query:
3204
+ stmt = select(
3205
+ models.CollectionReplica.scope,
3206
+ models.CollectionReplica.name,
3207
+ models.RSE.rse,
3208
+ models.CollectionReplica.rse_id,
3209
+ models.CollectionReplica.bytes,
3210
+ models.CollectionReplica.length,
3211
+ models.CollectionReplica.available_bytes,
3212
+ models.CollectionReplica.available_replicas_cnt.label("available_length"),
3213
+ models.CollectionReplica.state,
3214
+ models.CollectionReplica.created_at,
3215
+ models.CollectionReplica.updated_at,
3216
+ models.CollectionReplica.accessed_at
3217
+ ).where(
3218
+ and_(models.CollectionReplica.did_type == DIDType.DATASET,
3219
+ models.CollectionReplica.rse_id == models.RSE.id,
3220
+ models.RSE.deleted == false(),
3221
+ or_(*chunk))
3222
+ )
3223
+
3224
+ for row in session.execute(stmt).all():
2925
3225
  yield row._asdict()
2926
3226
  except NoResultFound:
2927
3227
  raise exception.DataIdentifierNotFound('No Data Identifiers found')
@@ -2995,50 +3295,74 @@ def list_datasets_per_rse(rse_id, filters=None, limit=None, *, session: "Session
2995
3295
 
2996
3296
  :returns: A list of dict dataset replicas
2997
3297
  """
2998
- query = session.query(models.CollectionReplica.scope,
2999
- models.CollectionReplica.name,
3000
- models.RSE.id.label('rse_id'),
3001
- models.RSE.rse,
3002
- models.CollectionReplica.bytes,
3003
- models.CollectionReplica.length,
3004
- models.CollectionReplica.available_bytes,
3005
- models.CollectionReplica.available_replicas_cnt.label("available_length"),
3006
- models.CollectionReplica.state,
3007
- models.CollectionReplica.created_at,
3008
- models.CollectionReplica.updated_at,
3009
- models.CollectionReplica.accessed_at)\
3010
- .filter_by(did_type=DIDType.DATASET)\
3011
- .filter(models.CollectionReplica.rse_id == models.RSE.id)\
3012
- .filter(models.RSE.id == rse_id)\
3013
- .filter(models.RSE.deleted == false())
3298
+ stmt = select(
3299
+ models.CollectionReplica.scope,
3300
+ models.CollectionReplica.name,
3301
+ models.RSE.id.label('rse_id'),
3302
+ models.RSE.rse,
3303
+ models.CollectionReplica.bytes,
3304
+ models.CollectionReplica.length,
3305
+ models.CollectionReplica.available_bytes,
3306
+ models.CollectionReplica.available_replicas_cnt.label("available_length"),
3307
+ models.CollectionReplica.state,
3308
+ models.CollectionReplica.created_at,
3309
+ models.CollectionReplica.updated_at,
3310
+ models.CollectionReplica.accessed_at
3311
+ ).where(
3312
+ and_(models.CollectionReplica.did_type == DIDType.DATASET,
3313
+ models.CollectionReplica.rse_id == models.RSE.id,
3314
+ models.RSE.deleted == false(),
3315
+ models.RSE.id == rse_id)
3316
+ )
3014
3317
 
3015
3318
  for (k, v) in filters and filters.items() or []:
3016
3319
  if k == 'name' or k == 'scope':
3017
3320
  v_str = v if k != 'scope' else v.internal
3018
3321
  if '*' in v_str or '%' in v_str:
3019
3322
  if session.bind.dialect.name == 'postgresql': # PostgreSQL escapes automatically
3020
- query = query.filter(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%')))
3323
+ stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%')))
3021
3324
  else:
3022
- query = query.filter(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%'), escape='\\'))
3325
+ stmt = stmt.where(getattr(models.CollectionReplica, k).like(v_str.replace('*', '%'), escape='\\'))
3023
3326
  else:
3024
- query = query.filter(getattr(models.CollectionReplica, k) == v)
3327
+ stmt = stmt.where(getattr(models.CollectionReplica, k) == v)
3025
3328
  # hints ?
3026
3329
  elif k == 'created_before':
3027
3330
  created_before = str_to_date(v)
3028
- query = query.filter(models.CollectionReplica.created_at <= created_before)
3331
+ stmt = stmt.where(models.CollectionReplica.created_at <= created_before)
3029
3332
  elif k == 'created_after':
3030
3333
  created_after = str_to_date(v)
3031
- query = query.filter(models.CollectionReplica.created_at >= created_after)
3334
+ stmt = stmt.where(models.CollectionReplica.created_at >= created_after)
3032
3335
  else:
3033
- query = query.filter(getattr(models.CollectionReplica, k) == v)
3336
+ stmt = stmt.where(getattr(models.CollectionReplica, k) == v)
3034
3337
 
3035
3338
  if limit:
3036
- query = query.limit(limit)
3339
+ stmt = stmt.limit(limit)
3037
3340
 
3038
- for row in query:
3341
+ for row in session.execute(stmt).all():
3039
3342
  yield row._asdict()
3040
3343
 
3041
3344
 
3345
+ @stream_session
3346
+ def list_replicas_per_rse(
3347
+ rse_id: str,
3348
+ limit: "Optional[int]" = None,
3349
+ *,
3350
+ session: "Session"
3351
+ ) -> "Iterator[dict[str, Any]]":
3352
+ """List all replicas at a given RSE."""
3353
+ list_stmt = select(
3354
+ models.RSEFileAssociation
3355
+ ).where(
3356
+ models.RSEFileAssociation.rse_id == rse_id
3357
+ )
3358
+
3359
+ if limit:
3360
+ list_stmt = list_stmt.limit(limit)
3361
+
3362
+ for replica in session.execute(list_stmt).yield_per(100).scalars():
3363
+ yield replica.to_dict()
3364
+
3365
+
3042
3366
  @transactional_session
3043
3367
  def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=None, *, session: "Session"):
3044
3368
  """
@@ -3050,14 +3374,31 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
3050
3374
  :returns: List of update requests for collection replicas.
3051
3375
  """
3052
3376
 
3377
+ stmt = delete(
3378
+ models.UpdatedCollectionReplica
3379
+ ).where(
3380
+ and_(models.UpdatedCollectionReplica.rse_id.is_(None),
3381
+ ~exists().where(
3382
+ and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,
3383
+ models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope)))
3384
+ ).execution_options(
3385
+ synchronize_session=False
3386
+ )
3387
+ session.execute(stmt)
3388
+
3053
3389
  # Delete update requests which do not have collection_replicas
3054
- session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.is_(None)
3055
- & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name, # NOQA: W503
3056
- models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope))).delete(synchronize_session=False)
3057
- session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.isnot(None)
3058
- & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name, # NOQA: W503
3059
- models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
3060
- models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id))).delete(synchronize_session=False)
3390
+ stmt = delete(
3391
+ models.UpdatedCollectionReplica
3392
+ ).where(
3393
+ and_(models.UpdatedCollectionReplica.rse_id.isnot(None),
3394
+ ~exists().where(
3395
+ and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,
3396
+ models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
3397
+ models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id)))
3398
+ ).execution_options(
3399
+ synchronize_session=False
3400
+ )
3401
+ session.execute(stmt)
3061
3402
 
3062
3403
  # Delete duplicates
3063
3404
  if session.bind.dialect.name == 'oracle':
@@ -3066,18 +3407,32 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
3066
3407
  schema = BASE.metadata.schema + '.'
3067
3408
  session.execute(text('DELETE FROM {schema}updated_col_rep A WHERE A.rowid > ANY (SELECT B.rowid FROM {schema}updated_col_rep B WHERE A.scope = B.scope AND A.name=B.name AND A.did_type=B.did_type AND (A.rse_id=B.rse_id OR (A.rse_id IS NULL and B.rse_id IS NULL)))'.format(schema=schema))) # NOQA: E501
3068
3409
  elif session.bind.dialect.name == 'mysql':
3069
- subquery1 = session.query(func.max(models.UpdatedCollectionReplica.id).label('max_id')).\
3070
- group_by(models.UpdatedCollectionReplica.scope,
3071
- models.UpdatedCollectionReplica.name,
3072
- models.UpdatedCollectionReplica.rse_id).subquery()
3073
- subquery2 = session.query(subquery1.c.max_id).subquery()
3074
- session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.notin_(subquery2)).delete(synchronize_session=False)
3410
+ subquery1 = select(
3411
+ func.max(models.UpdatedCollectionReplica.id).label('max_id')
3412
+ ).group_by(
3413
+ models.UpdatedCollectionReplica.scope,
3414
+ models.UpdatedCollectionReplica.name,
3415
+ models.UpdatedCollectionReplica.rse_id
3416
+ ).subquery()
3417
+
3418
+ subquery2 = select(
3419
+ subquery1.c.max_id
3420
+ )
3421
+
3422
+ stmt_del = delete(
3423
+ models.UpdatedCollectionReplica
3424
+ ).where(
3425
+ models.UpdatedCollectionReplica.id.not_in(subquery2)
3426
+ ).execution_options(
3427
+ synchronize_session=False
3428
+ )
3429
+ session.execute(stmt_del)
3075
3430
  else:
3076
- replica_update_requests = session.query(models.UpdatedCollectionReplica)
3431
+ stmt = select(models.UpdatedCollectionReplica)
3077
3432
  update_requests_with_rse_id = []
3078
3433
  update_requests_without_rse_id = []
3079
3434
  duplicate_request_ids = []
3080
- for update_request in replica_update_requests.all():
3435
+ for update_request in session.execute(stmt).scalars().all():
3081
3436
  if update_request.rse_id is not None:
3082
3437
  small_request = {'name': update_request.name, 'scope': update_request.scope, 'rse_id': update_request.rse_id}
3083
3438
  if small_request not in update_requests_with_rse_id:
@@ -3093,12 +3448,19 @@ def get_cleaned_updated_collection_replicas(total_workers, worker_number, limit=
3093
3448
  duplicate_request_ids.append(update_request.id)
3094
3449
  continue
3095
3450
  for chunk in chunks(duplicate_request_ids, 100):
3096
- session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.id.in_(chunk)).delete(synchronize_session=False)
3451
+ stmt = delete(
3452
+ models.UpdatedCollectionReplica
3453
+ ).where(
3454
+ models.UpdatedCollectionReplica.id.in_(chunk)
3455
+ ).execution_options(
3456
+ synchronize_session=False
3457
+ )
3458
+ session.execute(stmt)
3097
3459
 
3098
- query = session.query(models.UpdatedCollectionReplica)
3460
+ stmt = select(models.UpdatedCollectionReplica)
3099
3461
  if limit:
3100
- query = query.limit(limit)
3101
- return [update_request.to_dict() for update_request in query.all()]
3462
+ stmt = stmt.limit(limit)
3463
+ return [update_request.to_dict() for update_request in session.execute(stmt).scalars().all()]
3102
3464
 
3103
3465
 
3104
3466
  @transactional_session
@@ -3117,11 +3479,14 @@ def update_collection_replica(update_request, *, session: "Session"):
3117
3479
  available_replicas = 0
3118
3480
 
3119
3481
  try:
3120
- collection_replica = session.query(models.CollectionReplica)\
3121
- .filter_by(scope=update_request['scope'],
3122
- name=update_request['name'],
3123
- rse_id=update_request['rse_id'])\
3124
- .one()
3482
+ stmt = select(
3483
+ models.CollectionReplica
3484
+ ).where(
3485
+ and_(models.CollectionReplica.scope == update_request['scope'],
3486
+ models.CollectionReplica.name == update_request['name'],
3487
+ models.CollectionReplica.rse_id == update_request['rse_id'])
3488
+ )
3489
+ collection_replica = session.execute(stmt).scalar_one()
3125
3490
  ds_length = collection_replica.length
3126
3491
  old_available_replicas = collection_replica.available_replicas_cnt
3127
3492
  ds_bytes = collection_replica.bytes
@@ -3129,16 +3494,21 @@ def update_collection_replica(update_request, *, session: "Session"):
3129
3494
  pass
3130
3495
 
3131
3496
  try:
3132
- file_replica = session.query(models.RSEFileAssociation, models.DataIdentifierAssociation)\
3133
- .filter(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
3134
- models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
3135
- models.DataIdentifierAssociation.name == update_request['name'],
3136
- models.RSEFileAssociation.rse_id == update_request['rse_id'],
3137
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
3138
- update_request['scope'] == models.DataIdentifierAssociation.scope)\
3139
- .with_entities(label('ds_available_bytes', func.sum(models.RSEFileAssociation.bytes)),
3140
- label('available_replicas', func.count()))\
3141
- .one()
3497
+ stmt = select(
3498
+ func.sum(models.RSEFileAssociation.bytes).label('ds_available_bytes'),
3499
+ func.count().label('available_replicas')
3500
+ ).select_from(
3501
+ models.RSEFileAssociation
3502
+ ).where(
3503
+ and_(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
3504
+ models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
3505
+ models.RSEFileAssociation.rse_id == update_request['rse_id'],
3506
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
3507
+ models.DataIdentifierAssociation.name == update_request['name'],
3508
+ models.DataIdentifierAssociation.scope == update_request['scope'])
3509
+ )
3510
+ file_replica = session.execute(stmt).one()
3511
+
3142
3512
  available_replicas = file_replica.available_replicas
3143
3513
  ds_available_bytes = file_replica.ds_available_bytes
3144
3514
  except NoResultFound:
@@ -3150,36 +3520,53 @@ def update_collection_replica(update_request, *, session: "Session"):
3150
3520
  ds_replica_state = ReplicaState.UNAVAILABLE
3151
3521
 
3152
3522
  if old_available_replicas is not None and old_available_replicas > 0 and available_replicas == 0:
3153
- session.query(models.CollectionReplica).filter_by(scope=update_request['scope'],
3154
- name=update_request['name'],
3155
- rse_id=update_request['rse_id'])\
3156
- .delete()
3523
+ stmt = delete(
3524
+ models.CollectionReplica
3525
+ ).where(
3526
+ and_(models.CollectionReplica.scope == update_request['scope'],
3527
+ models.CollectionReplica.name == update_request['name'],
3528
+ models.CollectionReplica.rse_id == update_request['rse_id'])
3529
+ )
3530
+ session.execute(stmt)
3157
3531
  else:
3158
- updated_replica = session.query(models.CollectionReplica).filter_by(scope=update_request['scope'],
3159
- name=update_request['name'],
3160
- rse_id=update_request['rse_id'])\
3161
- .one()
3532
+ stmt = select(
3533
+ models.CollectionReplica
3534
+ ).where(
3535
+ and_(models.CollectionReplica.scope == update_request['scope'],
3536
+ models.CollectionReplica.name == update_request['name'],
3537
+ models.CollectionReplica.rse_id == update_request['rse_id'])
3538
+ )
3539
+ updated_replica = session.execute(stmt).scalar_one()
3540
+
3162
3541
  updated_replica.state = ds_replica_state
3163
3542
  updated_replica.available_replicas_cnt = available_replicas
3164
3543
  updated_replica.length = ds_length
3165
3544
  updated_replica.bytes = ds_bytes
3166
3545
  updated_replica.available_bytes = ds_available_bytes
3167
3546
  else:
3547
+ stmt = select(
3548
+ func.count().label('ds_length'),
3549
+ func.sum(models.DataIdentifierAssociation.bytes).label('ds_bytes')
3550
+ ).select_from(
3551
+ models.DataIdentifierAssociation
3552
+ ).where(
3553
+ and_(models.DataIdentifierAssociation.scope == update_request['scope'],
3554
+ models.DataIdentifierAssociation.name == update_request['name'])
3555
+ )
3556
+ association = session.execute(stmt).one()
3557
+
3168
3558
  # Check all dataset replicas
3169
- association = session.query(models.DataIdentifierAssociation)\
3170
- .filter_by(scope=update_request['scope'],
3171
- name=update_request['name'])\
3172
- .with_entities(label('ds_length', func.count()),
3173
- label('ds_bytes', func.sum(models.DataIdentifierAssociation.bytes)))\
3174
- .one()
3175
3559
  ds_length = association.ds_length
3176
3560
  ds_bytes = association.ds_bytes
3177
3561
  ds_replica_state = None
3178
3562
 
3179
- collection_replicas = session.query(models.CollectionReplica)\
3180
- .filter_by(scope=update_request['scope'], name=update_request['name'])\
3181
- .all()
3182
- for collection_replica in collection_replicas:
3563
+ stmt = select(
3564
+ models.CollectionReplica
3565
+ ).where(
3566
+ and_(models.CollectionReplica.scope == update_request['scope'],
3567
+ models.CollectionReplica.name == update_request['name'])
3568
+ )
3569
+ for collection_replica in session.execute(stmt).scalars().all():
3183
3570
  if ds_length:
3184
3571
  collection_replica.length = ds_length
3185
3572
  else:
@@ -3189,31 +3576,47 @@ def update_collection_replica(update_request, *, session: "Session"):
3189
3576
  else:
3190
3577
  collection_replica.bytes = 0
3191
3578
 
3192
- file_replicas = session.query(models.RSEFileAssociation, models.DataIdentifierAssociation)\
3193
- .filter(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
3194
- models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
3195
- models.DataIdentifierAssociation.name == update_request['name'],
3196
- models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
3197
- update_request['scope'] == models.DataIdentifierAssociation.scope)\
3198
- .with_entities(models.RSEFileAssociation.rse_id,
3199
- label('ds_available_bytes', func.sum(models.RSEFileAssociation.bytes)),
3200
- label('available_replicas', func.count()))\
3201
- .group_by(models.RSEFileAssociation.rse_id)\
3202
- .all()
3203
- for file_replica in file_replicas:
3579
+ stmt = select(
3580
+ func.sum(models.RSEFileAssociation.bytes).label('ds_available_bytes'),
3581
+ func.count().label('available_replicas'),
3582
+ models.RSEFileAssociation.rse_id
3583
+ ).select_from(
3584
+ models.RSEFileAssociation
3585
+ ).where(
3586
+ and_(models.RSEFileAssociation.scope == models.DataIdentifierAssociation.child_scope,
3587
+ models.RSEFileAssociation.name == models.DataIdentifierAssociation.child_name,
3588
+ models.RSEFileAssociation.state == ReplicaState.AVAILABLE,
3589
+ models.DataIdentifierAssociation.name == update_request['name'],
3590
+ models.DataIdentifierAssociation.scope == update_request['scope'])
3591
+ ).group_by(
3592
+ models.RSEFileAssociation.rse_id
3593
+ )
3594
+
3595
+ for file_replica in session.execute(stmt).all():
3204
3596
  if file_replica.available_replicas >= ds_length:
3205
3597
  ds_replica_state = ReplicaState.AVAILABLE
3206
3598
  else:
3207
3599
  ds_replica_state = ReplicaState.UNAVAILABLE
3208
3600
 
3209
- collection_replica = session.query(models.CollectionReplica)\
3210
- .filter_by(scope=update_request['scope'], name=update_request['name'], rse_id=file_replica.rse_id)\
3211
- .first()
3601
+ stmt = select(
3602
+ models.CollectionReplica
3603
+ ).where(
3604
+ and_(models.CollectionReplica.scope == update_request['scope'],
3605
+ models.CollectionReplica.name == update_request['name'],
3606
+ models.CollectionReplica.rse_id == file_replica.rse_id)
3607
+ )
3608
+ collection_replica = session.execute(stmt).scalars().first()
3212
3609
  if collection_replica:
3213
3610
  collection_replica.state = ds_replica_state
3214
3611
  collection_replica.available_replicas_cnt = file_replica.available_replicas
3215
3612
  collection_replica.available_bytes = file_replica.ds_available_bytes
3216
- session.query(models.UpdatedCollectionReplica).filter_by(id=update_request['id']).delete()
3613
+
3614
+ stmt = delete(
3615
+ models.UpdatedCollectionReplica
3616
+ ).where(
3617
+ models.UpdatedCollectionReplica.id == update_request['id']
3618
+ )
3619
+ session.execute(stmt)
3217
3620
 
3218
3621
 
3219
3622
  @read_session
@@ -3229,12 +3632,23 @@ def get_bad_pfns(limit=10000, thread=None, total_threads=None, *, session: "Sess
3229
3632
  returns: list of PFNs {'pfn': pfn, 'state': state, 'reason': reason, 'account': account, 'expires_at': expires_at}
3230
3633
  """
3231
3634
  result = []
3232
- query = session.query(models.BadPFNs.path, models.BadPFNs.state, models.BadPFNs.reason, models.BadPFNs.account, models.BadPFNs.expires_at)
3233
- query = filter_thread_work(session=session, query=query, total_threads=total_threads, thread_id=thread, hash_variable='path')
3234
- query.order_by(models.BadPFNs.created_at)
3235
- query = query.limit(limit)
3236
- for path, state, reason, account, expires_at in query.yield_per(1000):
3237
- result.append({'pfn': clean_surls([str(path)])[0], 'state': state, 'reason': reason, 'account': account, 'expires_at': expires_at})
3635
+
3636
+ stmt = select(
3637
+ models.BadPFN.path,
3638
+ models.BadPFN.state,
3639
+ models.BadPFN.reason,
3640
+ models.BadPFN.account,
3641
+ models.BadPFN.expires_at
3642
+ )
3643
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_threads, thread_id=thread, hash_variable='path')
3644
+ stmt = stmt.order_by(
3645
+ models.BadPFN.created_at
3646
+ ).limit(
3647
+ limit
3648
+ )
3649
+
3650
+ for path, state, reason, account, expires_at in session.execute(stmt).yield_per(1000):
3651
+ result.append({'pfn': clean_pfns([str(path)])[0], 'state': state, 'reason': reason, 'account': account, 'expires_at': expires_at})
3238
3652
  return result
3239
3653
 
3240
3654
 
@@ -3251,15 +3665,37 @@ def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAV
3251
3665
  :returns: True is successful.
3252
3666
  """
3253
3667
  for replica in replicas:
3668
+ scope_name_rse_state = and_(models.BadReplica.scope == replica['scope'],
3669
+ models.BadReplica.name == replica['name'],
3670
+ models.BadReplica.rse_id == replica['rse_id'],
3671
+ models.BadReplica.state == state)
3254
3672
  insert_new_row = True
3255
3673
  if state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
3256
- query = session.query(models.BadReplicas).filter_by(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], state=state)
3257
- if query.count():
3258
- query.update({'state': BadFilesStatus.TEMPORARY_UNAVAILABLE, 'updated_at': datetime.utcnow(), 'account': account, 'reason': reason, 'expires_at': expires_at}, synchronize_session=False)
3674
+ stmt = select(
3675
+ models.BadReplica
3676
+ ).where(
3677
+ scope_name_rse_state
3678
+ )
3679
+ if session.execute(stmt).scalar_one_or_none():
3680
+ stmt = update(
3681
+ models.BadReplica
3682
+ ).where(
3683
+ scope_name_rse_state
3684
+ ).values({
3685
+ models.BadReplica.state: BadFilesStatus.TEMPORARY_UNAVAILABLE,
3686
+ models.BadReplica.updated_at: datetime.utcnow(),
3687
+ models.BadReplica.account: account,
3688
+ models.BadReplica.reason: reason,
3689
+ models.BadReplica.expires_at: expires_at
3690
+ }).execution_options(
3691
+ synchronize_session=False
3692
+ )
3693
+ session.execute(stmt)
3694
+
3259
3695
  insert_new_row = False
3260
3696
  if insert_new_row:
3261
- new_bad_replica = models.BadReplicas(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], reason=reason,
3262
- state=state, account=account, bytes=None, expires_at=expires_at)
3697
+ new_bad_replica = models.BadReplica(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], reason=reason,
3698
+ state=state, account=account, bytes=None, expires_at=expires_at)
3263
3699
  new_bad_replica.save(session=session, flush=False)
3264
3700
  try:
3265
3701
  session.flush()
@@ -3286,11 +3722,17 @@ def bulk_delete_bad_pfns(pfns, *, session: "Session"):
3286
3722
  """
3287
3723
  pfn_clause = []
3288
3724
  for pfn in pfns:
3289
- pfn_clause.append(models.BadPFNs.path == pfn)
3725
+ pfn_clause.append(models.BadPFN.path == pfn)
3290
3726
 
3291
3727
  for chunk in chunks(pfn_clause, 100):
3292
- query = session.query(models.BadPFNs).filter(or_(*chunk))
3293
- query.delete(synchronize_session=False)
3728
+ stmt = delete(
3729
+ models.BadPFN
3730
+ ).where(
3731
+ or_(*chunk)
3732
+ ).execution_options(
3733
+ synchronize_session=False
3734
+ )
3735
+ session.execute(stmt)
3294
3736
 
3295
3737
  return True
3296
3738
 
@@ -3307,14 +3749,20 @@ def bulk_delete_bad_replicas(bad_replicas, *, session: "Session"):
3307
3749
  """
3308
3750
  replica_clause = []
3309
3751
  for replica in bad_replicas:
3310
- replica_clause.append(and_(models.BadReplicas.scope == replica['scope'],
3311
- models.BadReplicas.name == replica['name'],
3312
- models.BadReplicas.rse_id == replica['rse_id'],
3313
- models.BadReplicas.state == replica['state']))
3752
+ replica_clause.append(and_(models.BadReplica.scope == replica['scope'],
3753
+ models.BadReplica.name == replica['name'],
3754
+ models.BadReplica.rse_id == replica['rse_id'],
3755
+ models.BadReplica.state == replica['state']))
3314
3756
 
3315
3757
  for chunk in chunks(replica_clause, 100):
3316
- session.query(models.BadReplicas).filter(or_(*chunk)).\
3317
- delete(synchronize_session=False)
3758
+ stmt = delete(
3759
+ models.BadReplica
3760
+ ).where(
3761
+ or_(*chunk)
3762
+ ).execution_options(
3763
+ synchronize_session=False
3764
+ )
3765
+ session.execute(stmt)
3318
3766
  return True
3319
3767
 
3320
3768
 
@@ -3343,9 +3791,9 @@ def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, *, session:
3343
3791
  elif rep_state == BadPFNStatus.BAD and expires_at is not None:
3344
3792
  raise exception.InputValidationError("When adding a BAD pfn the expires_at value shouldn't be set.")
3345
3793
 
3346
- pfns = clean_surls(pfns)
3794
+ pfns = clean_pfns(pfns)
3347
3795
  for pfn in pfns:
3348
- new_pfn = models.BadPFNs(path=str(pfn), account=account, state=rep_state, reason=reason, expires_at=expires_at)
3796
+ new_pfn = models.BadPFN(path=str(pfn), account=account, state=rep_state, reason=reason, expires_at=expires_at)
3349
3797
  new_pfn = session.merge(new_pfn)
3350
3798
  new_pfn.save(session=session, flush=False)
3351
3799
 
@@ -3373,15 +3821,25 @@ def list_expired_temporary_unavailable_replicas(total_workers, worker_number, li
3373
3821
  :param session: The database session in use.
3374
3822
  """
3375
3823
 
3376
- query = session.query(models.BadReplicas.scope, models.BadReplicas.name, models.BadReplicas.rse_id).\
3377
- filter(models.BadReplicas.state == BadFilesStatus.TEMPORARY_UNAVAILABLE).\
3378
- filter(models.BadReplicas.expires_at < datetime.utcnow()).\
3379
- with_hint(models.ReplicationRule, "index(bad_replicas BAD_REPLICAS_EXPIRES_AT_IDX)", 'oracle').\
3380
- order_by(models.BadReplicas.expires_at)
3824
+ stmt = select(
3825
+ models.BadReplica.scope,
3826
+ models.BadReplica.name,
3827
+ models.BadReplica.rse_id,
3828
+ ).with_hint(
3829
+ models.ReplicationRule,
3830
+ 'INDEX(bad_replicas BAD_REPLICAS_EXPIRES_AT_IDX)',
3831
+ 'oracle'
3832
+ ).where(
3833
+ and_(models.BadReplica.state == BadFilesStatus.TEMPORARY_UNAVAILABLE,
3834
+ models.BadReplica.expires_at < datetime.utcnow())
3835
+ ).order_by(
3836
+ models.BadReplica.expires_at
3837
+ )
3838
+
3839
+ stmt = filter_thread_work(session=session, query=stmt, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
3840
+ stmt = stmt.limit(limit)
3381
3841
 
3382
- query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
3383
- query = query.limit(limit)
3384
- return query.all()
3842
+ return session.execute(stmt).all()
3385
3843
 
3386
3844
 
3387
3845
  @read_session
@@ -3395,9 +3853,15 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
3395
3853
  :returns: A dictionary with the list of states as keys and the rse_ids as value
3396
3854
  """
3397
3855
 
3398
- query = session.query(models.RSEFileAssociation.rse_id, models.RSEFileAssociation.state).filter_by(scope=scope, name=name)
3856
+ stmt = select(
3857
+ models.RSEFileAssociation.rse_id,
3858
+ models.RSEFileAssociation.state
3859
+ ).where(
3860
+ and_(models.RSEFileAssociation.scope == scope,
3861
+ models.RSEFileAssociation.name == name)
3862
+ )
3399
3863
  states = {}
3400
- for res in query.all():
3864
+ for res in session.execute(stmt).all():
3401
3865
  rse_id, state = res
3402
3866
  if state not in states:
3403
3867
  states[state] = []
@@ -3406,7 +3870,19 @@ def get_replicas_state(scope=None, name=None, *, session: "Session"):
3406
3870
 
3407
3871
 
3408
3872
  @read_session
3409
- def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logger=logging.log, younger_than=10, nattempts=0, nattempts_exact=False, *, session: "Session", exclude_states=['B', 'R', 'D'], is_suspicious=False):
3873
+ def get_suspicious_files(
3874
+ rse_expression: str,
3875
+ available_elsewhere: int,
3876
+ filter_: "Optional[dict[str, Any]]" = None,
3877
+ logger: "LoggerFunction" = logging.log,
3878
+ younger_than: "Optional[datetime]" = None,
3879
+ nattempts: int = 0,
3880
+ nattempts_exact: bool = False,
3881
+ *,
3882
+ session: "Session",
3883
+ exclude_states: "Optional[Iterable[str]]" = None,
3884
+ is_suspicious: bool = False
3885
+ ) -> "list[dict[str, Any]]":
3410
3886
  """
3411
3887
  Gets a list of replicas from bad_replicas table which are: declared more than <nattempts> times since <younger_than> date,
3412
3888
  present on the RSE specified by the <rse_expression> and do not have a state in <exclude_states> list.
@@ -3415,7 +3891,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
3415
3891
  Keyword Arguments:
3416
3892
  :param younger_than: Datetime object to select the replicas which were declared since younger_than date. Default value = 10 days ago.
3417
3893
  :param nattempts: The minimum number of replica appearances in the bad_replica DB table from younger_than date. Default value = 0.
3418
- :param nattempts_exact: If True, then only replicas with exactly 'nattempts' appearences in the bad_replica DB table are retrieved. Replicas with more appearences are ignored.
3894
+ :param nattempts_exact: If True, then only replicas with exactly 'nattempts' appearances in the bad_replica DB table are retrieved. Replicas with more appearances are ignored.
3419
3895
  :param rse_expression: The RSE expression where the replicas are located.
3420
3896
  :param filter_: Dictionary of attributes by which the RSE results should be filtered. e.g.: {'availability_write': True}
3421
3897
  :param exclude_states: List of states which eliminates replicas from search result if any of the states in the list
@@ -3432,6 +3908,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
3432
3908
  [{'scope': scope, 'name': name, 'rse': rse, 'rse_id': rse_id, cnt': cnt, 'created_at': created_at}, ...]
3433
3909
  """
3434
3910
 
3911
+ exclude_states = exclude_states or ['B', 'R', 'D']
3435
3912
  if available_elsewhere not in [SuspiciousAvailability["ALL"].value, SuspiciousAvailability["EXIST_COPIES"].value, SuspiciousAvailability["LAST_COPY"].value]:
3436
3913
  logger(logging.WARNING, """ERROR, available_elsewhere must be set to one of the following:
3437
3914
  SuspiciousAvailability["ALL"].value: (default) all suspicious replicas are returned
@@ -3454,7 +3931,7 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
3454
3931
  exclude_states_clause.append(BadFilesStatus(state))
3455
3932
 
3456
3933
  # making aliases for bad_replicas and replicas tables
3457
- bad_replicas_alias = aliased(models.BadReplicas, name='bad_replicas_alias')
3934
+ bad_replicas_alias = aliased(models.BadReplica, name='bad_replicas_alias')
3458
3935
  replicas_alias = aliased(models.RSEFileAssociation, name='replicas_alias')
3459
3936
 
3460
3937
  # assembling the selection rse_clause
@@ -3464,51 +3941,76 @@ def get_suspicious_files(rse_expression, available_elsewhere, filter_=None, logg
3464
3941
  for rse in parsedexp:
3465
3942
  rse_clause.append(models.RSEFileAssociation.rse_id == rse['id'])
3466
3943
 
3467
- # query base
3468
- query = session.query(func.count(), bad_replicas_alias.scope, bad_replicas_alias.name, models.RSEFileAssociation.rse_id, func.min(models.RSEFileAssociation.created_at))\
3469
- .filter(models.RSEFileAssociation.rse_id == bad_replicas_alias.rse_id,
3470
- models.RSEFileAssociation.scope == bad_replicas_alias.scope,
3471
- models.RSEFileAssociation.name == bad_replicas_alias.name,
3472
- bad_replicas_alias.created_at >= younger_than)
3944
+ stmt = select(
3945
+ func.count(),
3946
+ bad_replicas_alias.scope,
3947
+ bad_replicas_alias.name,
3948
+ models.RSEFileAssociation.rse_id,
3949
+ func.min(models.RSEFileAssociation.created_at)
3950
+ ).select_from(
3951
+ bad_replicas_alias
3952
+ ).where(
3953
+ models.RSEFileAssociation.rse_id == bad_replicas_alias.rse_id,
3954
+ models.RSEFileAssociation.scope == bad_replicas_alias.scope,
3955
+ models.RSEFileAssociation.name == bad_replicas_alias.name,
3956
+ bad_replicas_alias.created_at >= younger_than
3957
+ )
3473
3958
  if is_suspicious:
3474
- query.filter(bad_replicas_alias.state == BadFilesStatus.SUSPICIOUS)
3959
+ stmt = stmt.where(bad_replicas_alias.state == BadFilesStatus.SUSPICIOUS)
3475
3960
  if rse_clause:
3476
- query = query.filter(or_(*rse_clause))
3961
+ stmt = stmt.where(or_(*rse_clause))
3477
3962
 
3478
3963
  # Only return replicas that have at least one copy on another RSE
3479
3964
  if available_elsewhere == SuspiciousAvailability["EXIST_COPIES"].value:
3480
- available_replica = exists(select(1).where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
3481
- replicas_alias.scope == bad_replicas_alias.scope,
3482
- replicas_alias.name == bad_replicas_alias.name,
3483
- replicas_alias.rse_id != bad_replicas_alias.rse_id)))
3484
- query = query.filter(available_replica)
3965
+ available_replica = exists(select(1)
3966
+ .where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
3967
+ replicas_alias.scope == bad_replicas_alias.scope,
3968
+ replicas_alias.name == bad_replicas_alias.name,
3969
+ replicas_alias.rse_id != bad_replicas_alias.rse_id)))
3970
+ stmt = stmt.where(available_replica)
3485
3971
 
3486
3972
  # Only return replicas that are the last remaining copy
3487
3973
  if available_elsewhere == SuspiciousAvailability["LAST_COPY"].value:
3488
- last_replica = ~exists(select(1).where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
3489
- replicas_alias.scope == bad_replicas_alias.scope,
3490
- replicas_alias.name == bad_replicas_alias.name,
3491
- replicas_alias.rse_id != bad_replicas_alias.rse_id)))
3492
- query = query.filter(last_replica)
3974
+ last_replica = ~exists(select(1)
3975
+ .where(and_(replicas_alias.state == ReplicaState.AVAILABLE,
3976
+ replicas_alias.scope == bad_replicas_alias.scope,
3977
+ replicas_alias.name == bad_replicas_alias.name,
3978
+ replicas_alias.rse_id != bad_replicas_alias.rse_id)))
3979
+ stmt = stmt.where(last_replica)
3493
3980
 
3494
3981
  # it is required that the selected replicas
3495
3982
  # do not occur as BAD/DELETED/LOST/RECOVERED/...
3496
3983
  # in the bad_replicas table during the same time window.
3497
- other_states_present = exists(select(1).where(and_(models.BadReplicas.scope == bad_replicas_alias.scope,
3498
- models.BadReplicas.name == bad_replicas_alias.name,
3499
- models.BadReplicas.created_at >= younger_than,
3500
- models.BadReplicas.rse_id == bad_replicas_alias.rse_id,
3501
- models.BadReplicas.state.in_(exclude_states_clause))))
3502
- query = query.filter(not_(other_states_present))
3984
+ other_states_present = exists(select(1)
3985
+ .where(and_(models.BadReplica.scope == bad_replicas_alias.scope,
3986
+ models.BadReplica.name == bad_replicas_alias.name,
3987
+ models.BadReplica.created_at >= younger_than,
3988
+ models.BadReplica.rse_id == bad_replicas_alias.rse_id,
3989
+ models.BadReplica.state.in_(exclude_states_clause))))
3990
+ stmt = stmt.where(not_(other_states_present))
3503
3991
 
3504
3992
  # finally, the results are grouped by RSE, scope, name and required to have
3505
3993
  # at least 'nattempts' occurrences in the result of the query per replica.
3506
3994
  # If nattempts_exact, then only replicas are required to have exactly
3507
- # 'nattempts' occurences.
3995
+ # 'nattempts' occurrences.
3508
3996
  if nattempts_exact:
3509
- query_result = query.group_by(models.RSEFileAssociation.rse_id, bad_replicas_alias.scope, bad_replicas_alias.name).having(func.count() == nattempts).all()
3997
+ stmt = stmt.group_by(
3998
+ models.RSEFileAssociation.rse_id,
3999
+ bad_replicas_alias.scope,
4000
+ bad_replicas_alias.name
4001
+ ).having(
4002
+ func.count() == nattempts
4003
+ )
4004
+ query_result = session.execute(stmt).all()
3510
4005
  else:
3511
- query_result = query.group_by(models.RSEFileAssociation.rse_id, bad_replicas_alias.scope, bad_replicas_alias.name).having(func.count() > nattempts).all()
4006
+ stmt = stmt.group_by(
4007
+ models.RSEFileAssociation.rse_id,
4008
+ bad_replicas_alias.scope,
4009
+ bad_replicas_alias.name
4010
+ ).having(
4011
+ func.count() > nattempts
4012
+ )
4013
+ query_result = session.execute(stmt).all()
3512
4014
 
3513
4015
  # translating the rse_id to RSE name and assembling the return list of dictionaries
3514
4016
  result = []
@@ -3533,20 +4035,44 @@ def get_suspicious_reason(rse_id, scope, name, nattempts=0, logger=logging.log,
3533
4035
  :param session: The database session in use. Default value = None.
3534
4036
  """
3535
4037
  # Alias for bad replicas
3536
- bad_replicas_alias = aliased(models.BadReplicas, name='bad_replicas_alias')
4038
+ bad_replicas_alias = aliased(models.BadReplica, name='bad_replicas_alias')
3537
4039
 
3538
- # query base
3539
- query = session.query(bad_replicas_alias.scope, bad_replicas_alias.name, bad_replicas_alias.reason, bad_replicas_alias.rse_id)\
3540
- .filter(bad_replicas_alias.rse_id == rse_id,
3541
- bad_replicas_alias.scope == scope,
3542
- bad_replicas_alias.name == name)
3543
- count = query.count()
4040
+ stmt = select(
4041
+ bad_replicas_alias.scope,
4042
+ bad_replicas_alias.name,
4043
+ bad_replicas_alias.reason,
4044
+ bad_replicas_alias.rse_id
4045
+ ).where(
4046
+ and_(bad_replicas_alias.rse_id == rse_id,
4047
+ bad_replicas_alias.scope == scope,
4048
+ bad_replicas_alias.state == 'S',
4049
+ bad_replicas_alias.name == name,
4050
+ ~exists(select(1).where(
4051
+ and_(bad_replicas_alias.rse_id == rse_id,
4052
+ bad_replicas_alias.name == name,
4053
+ bad_replicas_alias.scope == scope,
4054
+ bad_replicas_alias.state != 'S'))))
4055
+ )
3544
4056
 
3545
- query_result = query.group_by(bad_replicas_alias.rse_id, bad_replicas_alias.scope, bad_replicas_alias.name, bad_replicas_alias.reason).having(func.count() > nattempts).all()
4057
+ count_query = select(
4058
+ func.count()
4059
+ ).select_from(
4060
+ stmt.subquery()
4061
+ )
4062
+ count = session.execute(count_query).scalar_one()
4063
+
4064
+ grouped_stmt = stmt.group_by(
4065
+ bad_replicas_alias.rse_id,
4066
+ bad_replicas_alias.scope,
4067
+ bad_replicas_alias.name,
4068
+ bad_replicas_alias.reason
4069
+ ).having(
4070
+ func.count() > nattempts
4071
+ )
3546
4072
 
3547
4073
  result = []
3548
4074
  rses = {}
3549
- for scope_, name_, reason, rse_id_ in query_result:
4075
+ for scope_, name_, reason, rse_id_ in session.execute(grouped_stmt).all():
3550
4076
  if rse_id_ not in rses:
3551
4077
  rse = get_rse_name(rse_id=rse_id_, session=session)
3552
4078
  rses[rse_id_] = rse
@@ -3570,27 +4096,31 @@ def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, *, session: "Session"
3570
4096
  :param session: database session in use.
3571
4097
  """
3572
4098
  stmt = update(models.RSEFileAssociation).where(
3573
- and_(
3574
- models.RSEFileAssociation.rse_id == rse_id,
3575
- models.RSEFileAssociation.name == name,
3576
- models.RSEFileAssociation.scope == scope,
3577
- ~exists().where(
3578
- and_(
3579
- models.ReplicaLock.rse_id == rse_id,
3580
- models.ReplicaLock.name == name,
3581
- models.ReplicaLock.scope == scope,
3582
- )
3583
- )
3584
- )
3585
- )\
3586
- .prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle')\
3587
- .execution_options(synchronize_session=False)\
3588
- .values(tombstone=tombstone)
3589
- result = session.execute(stmt)
4099
+ and_(models.RSEFileAssociation.rse_id == rse_id,
4100
+ models.RSEFileAssociation.name == name,
4101
+ models.RSEFileAssociation.scope == scope,
4102
+ ~exists().where(
4103
+ and_(models.ReplicaLock.rse_id == rse_id,
4104
+ models.ReplicaLock.name == name,
4105
+ models.ReplicaLock.scope == scope)))
4106
+ ).prefix_with(
4107
+ '/*+ INDEX(REPLICAS REPLICAS_PK) */', dialect='oracle'
4108
+ ).values({
4109
+ models.RSEFileAssociation.tombstone: tombstone
4110
+ }).execution_options(
4111
+ synchronize_session=False
4112
+ )
3590
4113
 
3591
- if result.rowcount == 0:
4114
+ if session.execute(stmt).rowcount == 0:
3592
4115
  try:
3593
- session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).one()
4116
+ stmt = select(
4117
+ models.RSEFileAssociation.tombstone
4118
+ ).where(
4119
+ and_(models.RSEFileAssociation.rse_id == rse_id,
4120
+ models.RSEFileAssociation.name == name,
4121
+ models.RSEFileAssociation.scope == scope)
4122
+ )
4123
+ session.execute(stmt).scalar_one()
3594
4124
  raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
3595
4125
  except NoResultFound:
3596
4126
  raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
@@ -3607,20 +4137,21 @@ def get_RSEcoverage_of_dataset(scope, name, *, session: "Session"):
3607
4137
  :return: Dictionary { rse_id : <total bytes present at rse_id> }
3608
4138
  """
3609
4139
 
3610
- query = session.query(models.RSEFileAssociation.rse_id, func.sum(models.DataIdentifierAssociation.bytes))
3611
-
3612
- query = query.filter(and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
3613
- models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
3614
- models.DataIdentifierAssociation.scope == scope,
3615
- models.DataIdentifierAssociation.name == name,
3616
- models.RSEFileAssociation.state != ReplicaState.BEING_DELETED,
3617
- ))
3618
-
3619
- query = query.group_by(models.RSEFileAssociation.rse_id)
4140
+ stmt = select(
4141
+ models.RSEFileAssociation.rse_id,
4142
+ func.sum(models.DataIdentifierAssociation.bytes)
4143
+ ).where(
4144
+ and_(models.DataIdentifierAssociation.child_scope == models.RSEFileAssociation.scope,
4145
+ models.DataIdentifierAssociation.child_name == models.RSEFileAssociation.name,
4146
+ models.DataIdentifierAssociation.scope == scope,
4147
+ models.DataIdentifierAssociation.name == name,
4148
+ models.RSEFileAssociation.state != ReplicaState.BEING_DELETED)
4149
+ ).group_by(
4150
+ models.RSEFileAssociation.rse_id
4151
+ )
3620
4152
 
3621
4153
  result = {}
3622
-
3623
- for rse_id, total in query:
4154
+ for rse_id, total in session.execute(stmt):
3624
4155
  if total:
3625
4156
  result[rse_id] = total
3626
4157