rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,17 +14,17 @@
15
14
 
16
15
  import json as json_lib
17
16
  import operator
18
- from typing import TYPE_CHECKING, cast, Any
17
+ from typing import TYPE_CHECKING, Any, cast
19
18
 
20
- from sqlalchemy.exc import DataError
21
- from sqlalchemy.orm.exc import NoResultFound
19
+ from sqlalchemy import and_, select
20
+ from sqlalchemy.exc import DataError, NoResultFound
22
21
 
23
22
  from rucio.common import exception
24
23
  from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin
25
24
  from rucio.core.did_meta_plugins.filter_engine import FilterEngine
26
25
  from rucio.db.sqla import models
27
26
  from rucio.db.sqla.constants import DIDType
28
- from rucio.db.sqla.session import read_session, transactional_session, stream_session
27
+ from rucio.db.sqla.session import read_session, stream_session, transactional_session
29
28
  from rucio.db.sqla.util import json_implemented
30
29
 
31
30
  if TYPE_CHECKING:
@@ -54,7 +53,13 @@ class JSONDidMeta(DidMetaPlugin):
54
53
  raise NotImplementedError
55
54
 
56
55
  try:
57
- row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one()
56
+ stmt = select(
57
+ models.DidMeta
58
+ ).where(
59
+ and_(models.DidMeta.scope == scope,
60
+ models.DidMeta.name == name)
61
+ )
62
+ row = session.execute(stmt).scalar_one()
58
63
  meta = getattr(row, 'meta')
59
64
  return json_lib.loads(meta) if session.bind.dialect.name in ['oracle', 'sqlite'] else meta
60
65
  except NoResultFound:
@@ -69,10 +74,22 @@ class JSONDidMeta(DidMetaPlugin):
69
74
  if not json_implemented(session=session):
70
75
  raise NotImplementedError
71
76
 
72
- if session.query(models.DataIdentifier).filter_by(scope=scope, name=name).one_or_none() is None:
77
+ stmt = select(
78
+ models.DataIdentifier
79
+ ).where(
80
+ and_(models.DataIdentifier.scope == scope,
81
+ models.DataIdentifier.name == name)
82
+ )
83
+ if session.execute(stmt).one_or_none() is None:
73
84
  raise exception.DataIdentifierNotFound("Data identifier '%s:%s' not found" % (scope, name))
74
85
 
75
- row_did_meta = session.query(models.DidMeta).filter_by(scope=scope, name=name).scalar()
86
+ stmt = select(
87
+ models.DidMeta
88
+ ).where(
89
+ and_(models.DidMeta.scope == scope,
90
+ models.DidMeta.name == name)
91
+ )
92
+ row_did_meta = session.execute(stmt).scalar_one_or_none()
76
93
  if row_did_meta is None:
77
94
  # Add metadata column to new table (if not already present)
78
95
  row_did_meta = models.DidMeta(scope=scope, name=name)
@@ -83,9 +100,9 @@ class JSONDidMeta(DidMetaPlugin):
83
100
  if row_did_meta.meta:
84
101
  if session.bind.dialect.name in ['oracle', 'sqlite']:
85
102
  # Oracle and sqlite returns a string instead of a dict
86
- existing_meta = json_lib.loads(cast(str, row_did_meta.meta))
103
+ existing_meta = json_lib.loads(cast("str", row_did_meta.meta))
87
104
  else:
88
- existing_meta = cast(dict[str, Any], row_did_meta.meta)
105
+ existing_meta = cast("dict[str, Any]", row_did_meta.meta)
89
106
 
90
107
  for key, value in metadata.items():
91
108
  existing_meta[key] = value
@@ -114,7 +131,13 @@ class JSONDidMeta(DidMetaPlugin):
114
131
  raise NotImplementedError
115
132
 
116
133
  try:
117
- row = session.query(models.DidMeta).filter_by(scope=scope, name=name).one()
134
+ stmt = select(
135
+ models.DidMeta
136
+ ).where(
137
+ and_(models.DidMeta.scope == scope,
138
+ models.DidMeta.name == name)
139
+ )
140
+ row = session.execute(stmt).scalar_one()
118
141
  existing_meta = getattr(row, 'meta')
119
142
  # Oracle returns a string instead of a dict
120
143
  if session.bind.dialect.name in ['oracle', 'sqlite'] and existing_meta is not None:
@@ -145,14 +168,14 @@ class JSONDidMeta(DidMetaPlugin):
145
168
  if not ignore_dids:
146
169
  ignore_dids = set()
147
170
 
148
- # backwards compatability for filters as single {}.
171
+ # backwards compatibility for filters as single {}.
149
172
  if isinstance(filters, dict):
150
173
  filters = [filters]
151
174
 
152
175
  # instantiate fe and create sqla query, note that coercion to a model keyword
153
176
  # is not appropriate here as the filter words are stored in a single json column.
154
177
  fe = FilterEngine(filters, model_class=models.DidMeta, strict_coerce=False)
155
- query = fe.create_sqla_query(
178
+ stmt = fe.create_sqla_query(
156
179
  additional_model_attributes=[
157
180
  models.DidMeta.scope,
158
181
  models.DidMeta.name
@@ -164,13 +187,15 @@ class JSONDidMeta(DidMetaPlugin):
164
187
  )
165
188
 
166
189
  if limit:
167
- query = query.limit(limit)
190
+ stmt = stmt.limit(
191
+ limit
192
+ )
168
193
  if recursive:
169
194
  from rucio.core.did import list_content
170
195
 
171
196
  # Get attached DIDs and save in list because query has to be finished before starting a new one in the recursion
172
197
  collections_content = []
173
- for did in query.yield_per(100):
198
+ for did in session.execute(stmt).yield_per(100):
174
199
  if (did.did_type == DIDType.CONTAINER or did.did_type == DIDType.DATASET):
175
200
  collections_content += [d for d in list_content(scope=did.scope, name=did.name)]
176
201
 
@@ -183,7 +208,7 @@ class JSONDidMeta(DidMetaPlugin):
183
208
  yield result
184
209
 
185
210
  try:
186
- for did in query.yield_per(5): # don't unpack this as it makes it dependent on query return order!
211
+ for did in session.execute(stmt).yield_per(5): # don't unpack this as it makes it dependent on query return order!
187
212
  if long:
188
213
  did_full = "{}:{}".format(did.scope, did.name)
189
214
  if did_full not in ignore_dids: # concatenating results of OR clauses may contain duplicate DIDs if query result sets not mutually exclusive.
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,14 +17,14 @@ from typing import TYPE_CHECKING
18
17
 
19
18
  import pymongo
20
19
 
21
- from rucio.common import config
22
- from rucio.common import exception
20
+ from rucio.common import config, exception
23
21
  from rucio.common.types import InternalScope
24
22
  from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin
25
23
  from rucio.core.did_meta_plugins.filter_engine import FilterEngine
26
24
 
27
25
  if TYPE_CHECKING:
28
26
  from typing import Optional
27
+
29
28
  from sqlalchemy.orm import Session
30
29
 
31
30
  IMMUTABLE_KEYS = [
@@ -37,7 +36,15 @@ IMMUTABLE_KEYS = [
37
36
 
38
37
 
39
38
  class MongoDidMeta(DidMetaPlugin):
40
- def __init__(self, host=None, port=None, db=None, collection=None):
39
+ def __init__(
40
+ self,
41
+ host: "Optional[str]" = None,
42
+ port: "Optional[int]" = None,
43
+ db: "Optional[str]" = None,
44
+ collection: "Optional[str]" = None,
45
+ user: "Optional[str]" = None,
46
+ password: "Optional[str]" = None,
47
+ ):
41
48
  super(MongoDidMeta, self).__init__()
42
49
  if host is None:
43
50
  host = config.config_get('metadata', 'mongo_service_host')
@@ -48,7 +55,17 @@ class MongoDidMeta(DidMetaPlugin):
48
55
  if collection is None:
49
56
  collection = config.config_get('metadata', 'mongo_collection')
50
57
 
51
- self.client = pymongo.MongoClient("mongodb://{}:{}/".format(host, port))
58
+ if user is None and config.config_has_section("metadata"):
59
+ user = config.config_get("metadata", "mongo_user", default=None)
60
+
61
+ if user is not None:
62
+ if password is None:
63
+ password = config.config_get("metadata", "mongo_password")
64
+ auth = "{user}:{password}@".format(user=user, password=password)
65
+ else:
66
+ auth = ""
67
+
68
+ self.client = pymongo.MongoClient("mongodb://{auth}{host}:{port}/".format(auth=auth, host=host, port=port))
52
69
  self.db = self.client[db]
53
70
  self.col = self.db[collection]
54
71
 
@@ -143,7 +160,7 @@ class MongoDidMeta(DidMetaPlugin):
143
160
  if not ignore_dids:
144
161
  ignore_dids = set()
145
162
 
146
- # backwards compatability for filters as single {}.
163
+ # backwards compatibility for filters as single {}.
147
164
  if isinstance(filters, dict):
148
165
  filters = [filters]
149
166
 
@@ -167,7 +184,7 @@ class MongoDidMeta(DidMetaPlugin):
167
184
  if limit:
168
185
  query_result = query_result.limit(limit)
169
186
  for did in query_result:
170
- did_full = did_full = "{}:{}".format(did['scope'], did['name'])
187
+ did_full = "{}:{}".format(did['scope'], did['name'])
171
188
  if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs
172
189
  ignore_dids.add(did_full)
173
190
  yield {
@@ -182,7 +199,7 @@ class MongoDidMeta(DidMetaPlugin):
182
199
  if limit:
183
200
  query_result = query_result.limit(limit)
184
201
  for did in query_result:
185
- did_full = did_full = "{}:{}".format(did['scope'], did['name'])
202
+ did_full = "{}:{}".format(did['scope'], did['name'])
186
203
  if did_full not in ignore_dids: # aggregating recursive queries may contain duplicate DIDs
187
204
  ignore_dids.add(did_full)
188
205
  yield did['name']
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,14 +19,14 @@ from typing import TYPE_CHECKING
20
19
  import psycopg2
21
20
  import psycopg2.extras
22
21
 
23
- from rucio.common import config
24
- from rucio.common import exception
22
+ from rucio.common import config, exception
25
23
  from rucio.common.types import InternalScope
26
24
  from rucio.core.did_meta_plugins.did_meta_plugin_interface import DidMetaPlugin
27
25
  from rucio.core.did_meta_plugins.filter_engine import FilterEngine
28
26
 
29
27
  if TYPE_CHECKING:
30
28
  from typing import Optional
29
+
31
30
  from sqlalchemy.orm import Session
32
31
 
33
32
 
@@ -252,7 +251,7 @@ class ExternalPostgresJSONDidMeta(DidMetaPlugin):
252
251
  if not ignore_dids:
253
252
  ignore_dids = set()
254
253
 
255
- # backwards compatability for filters as single {}.
254
+ # backwards compatibility for filters as single {}.
256
255
  if isinstance(filters, dict):
257
256
  filters = [filters]
258
257
 
rucio/core/dirac.py CHANGED
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,13 +15,14 @@
16
15
  import re
17
16
  from json import loads
18
17
  from json.decoder import JSONDecodeError
19
- from typing import TYPE_CHECKING
18
+ from typing import TYPE_CHECKING, Any, Optional
20
19
 
21
- from sqlalchemy.orm.exc import NoResultFound
20
+ from sqlalchemy import and_, select
21
+ from sqlalchemy.exc import NoResultFound
22
22
 
23
23
  from rucio.common.config import config_get
24
- from rucio.common.exception import InvalidType, UnsupportedOperation, ConfigNotFound, RucioException
25
- from rucio.common.types import InternalScope, InternalAccount
24
+ from rucio.common.exception import ConfigNotFound, InvalidType, RucioException, UnsupportedOperation
25
+ from rucio.common.types import InternalAccount, InternalScope
26
26
  from rucio.common.utils import extract_scope
27
27
  from rucio.core.did import add_did, attach_dids_to_dids
28
28
  from rucio.core.replica import add_replicas
@@ -30,14 +30,21 @@ from rucio.core.rule import add_rule, list_rules, update_rule
30
30
  from rucio.core.scope import list_scopes
31
31
  from rucio.db.sqla import models
32
32
  from rucio.db.sqla.constants import DIDType
33
- from rucio.db.sqla.session import transactional_session, read_session
33
+ from rucio.db.sqla.session import read_session, transactional_session
34
34
 
35
35
  if TYPE_CHECKING:
36
+ from collections.abc import Iterable
37
+
36
38
  from sqlalchemy.orm import Session
37
39
 
38
40
 
39
41
  @read_session
40
- def _exists(scope, name, *, session: "Session"):
42
+ def _exists(
43
+ scope: str,
44
+ name: str,
45
+ *,
46
+ session: "Session"
47
+ ) -> tuple[bool, Optional[DIDType]]:
41
48
  """
42
49
  Check if the did exists
43
50
 
@@ -46,15 +53,31 @@ def _exists(scope, name, *, session: "Session"):
46
53
  :session: The session used
47
54
  """
48
55
  try:
49
- res = session.query(models.DataIdentifier).filter_by(scope=scope, name=name).\
50
- with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').one()
51
- return True, res.did_type
56
+ stmt = select(
57
+ models.DataIdentifier.did_type
58
+ ).with_hint(
59
+ models.DataIdentifier,
60
+ "INDEX(DIDS DIDS_PK)",
61
+ 'oracle'
62
+ ).where(
63
+ and_(models.DataIdentifier.scope == scope,
64
+ models.DataIdentifier.name == name)
65
+ )
66
+ return True, session.execute(stmt).scalar_one()
52
67
  except NoResultFound:
53
68
  return False, None
54
69
 
55
70
 
56
71
  @transactional_session
57
- def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session"):
72
+ def add_files(
73
+ lfns: "Iterable[dict[str, Any]]",
74
+ account: str,
75
+ ignore_availability: bool,
76
+ parents_metadata: Optional[dict[str, Any]] = None,
77
+ vo: str = 'def',
78
+ *,
79
+ session: "Session"
80
+ ) -> None:
58
81
  """
59
82
  Bulk add files :
60
83
  - Create the file and replica.
@@ -64,19 +87,22 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
64
87
  :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
65
88
  :param issuer: The issuer account.
66
89
  :param ignore_availability: A boolean to ignore blocklisted sites.
90
+ :param parents_metadata: Metadata for selected hierarchy DIDs. (dictionary {'lpn': {key : value}})
67
91
  :param vo: The VO to act on
68
92
  :param session: The session used
69
93
  """
70
94
  rule_extension_list = []
71
95
  attachments = []
96
+ if not parents_metadata:
97
+ parents_metadata = {}
72
98
  # The list of scopes is necessary for the extract_scope
73
99
  filter_ = {'scope': InternalScope(scope='*', vo=vo)}
74
100
  scopes = list_scopes(filter_=filter_, session=session)
75
101
  scopes = [scope.external for scope in scopes]
76
102
  exist_lfn = []
77
103
  try:
78
- lifetime_dict: str = config_get(section='dirac', option='lifetime', default='{}', session=session)
79
- lifetime_dict = loads(lifetime_dict)
104
+ config_lifetime: str = config_get(section='dirac', option='lifetime', default='{}', session=session)
105
+ lifetime_dict: dict = loads(config_lifetime)
80
106
  except ConfigNotFound:
81
107
  lifetime_dict = {}
82
108
  except JSONDecodeError as err:
@@ -104,6 +130,7 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
104
130
  dsn_name = lpns[0]
105
131
  dsn_scope, _ = extract_scope(dsn_name, scopes)
106
132
  dsn_scope = InternalScope(dsn_scope, vo=vo)
133
+ dsn_meta = parents_metadata.get(dsn_name, {})
107
134
 
108
135
  # Compute lifetime
109
136
  lifetime = None
@@ -126,7 +153,7 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
126
153
  DIDType.DATASET,
127
154
  account=InternalAccount(account, vo=vo),
128
155
  statuses=None,
129
- meta=None,
156
+ meta=dsn_meta,
130
157
  rules=[{'copies': 1, 'rse_expression': 'ANY=true', 'weight': None, 'account': InternalAccount(account, vo=vo), 'lifetime': lifetime, 'grouping': 'NONE'}],
131
158
  lifetime=None,
132
159
  dids=None,
@@ -153,11 +180,12 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
153
180
  guid = lfn.get('guid', None)
154
181
  adler32 = lfn.get('adler32', None)
155
182
  pfn = lfn.get('pfn', None)
156
- files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32}
183
+ meta = lfn.get('meta', {})
184
+ files = {'scope': lfn_scope, 'name': filename, 'bytes': bytes_, 'adler32': adler32, 'meta': meta}
157
185
  if pfn:
158
186
  files['pfn'] = str(pfn)
159
187
  if guid:
160
- files['meta'] = {'guid': guid}
188
+ files['meta']['guid'] = guid
161
189
  add_replicas(rse_id=rse_id,
162
190
  files=[files],
163
191
  dataset_meta=None,
@@ -181,6 +209,7 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
181
209
  child_scope, _ = extract_scope(lpn, scopes)
182
210
  child_scope = InternalScope(child_scope, vo=vo)
183
211
  exists, did_type = _exists(child_scope, lpn)
212
+ child_meta = parents_metadata.get(lpn, {})
184
213
  if exists and did_type == DIDType.DATASET:
185
214
  raise UnsupportedOperation('Cannot create %s as container' % lpn)
186
215
  if (lpn not in exist_lfn) and not exists:
@@ -190,7 +219,7 @@ def add_files(lfns, account, ignore_availability, vo='def', *, session: "Session
190
219
  DIDType.CONTAINER,
191
220
  account=InternalAccount(account, vo=vo),
192
221
  statuses=None,
193
- meta=None,
222
+ meta=child_meta,
194
223
  rules=None,
195
224
  lifetime=None,
196
225
  dids=None,
rucio/core/distance.py CHANGED
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,21 +12,22 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any, Optional
17
16
 
17
+ from sqlalchemy import and_, delete, select, update
18
18
  from sqlalchemy.exc import DatabaseError, IntegrityError
19
19
  from sqlalchemy.orm import aliased
20
20
 
21
21
  from rucio.common import exception
22
- from rucio.db.sqla.models import Distance, RSE
23
- from rucio.db.sqla.session import transactional_session, read_session
22
+ from rucio.db.sqla.models import RSE, Distance
23
+ from rucio.db.sqla.session import read_session, transactional_session
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  from sqlalchemy.orm import Session
27
27
 
28
28
 
29
29
  @transactional_session
30
- def add_distance(src_rse_id, dest_rse_id, distance=None, *, session: "Session"):
30
+ def add_distance(src_rse_id: str, dest_rse_id: str, distance: Optional[int] = None, *, session: "Session") -> None:
31
31
  """
32
32
  Add a src-dest distance.
33
33
 
@@ -47,9 +47,9 @@ def add_distance(src_rse_id, dest_rse_id, distance=None, *, session: "Session"):
47
47
 
48
48
 
49
49
  @read_session
50
- def get_distances(src_rse_id=None, dest_rse_id=None, *, session: "Session") -> list[dict]:
50
+ def get_distances(src_rse_id: Optional[str] = None, dest_rse_id: Optional[str] = None, *, session: "Session") -> list[dict[str, Any]]:
51
51
  """
52
- Get distances between rses.
52
+ Get distances between RSEs.
53
53
 
54
54
  :param src_rse_id: The source RSE ID.
55
55
  :param dest_rse_id: The destination RSE ID.
@@ -59,81 +59,95 @@ def get_distances(src_rse_id=None, dest_rse_id=None, *, session: "Session") -> l
59
59
  """
60
60
 
61
61
  try:
62
- query = session.query(Distance)
62
+ stmt = select(
63
+ Distance
64
+ )
63
65
  if src_rse_id:
64
- query = query.filter(Distance.src_rse_id == src_rse_id)
66
+ stmt = stmt.where(
67
+ Distance.src_rse_id == src_rse_id
68
+ )
65
69
  if dest_rse_id:
66
- query = query.filter(Distance.dest_rse_id == dest_rse_id)
70
+ stmt = stmt.where(
71
+ Distance.dest_rse_id == dest_rse_id
72
+ )
67
73
 
68
74
  distances = []
69
- tmp = query.all()
70
- if tmp:
71
- for t in tmp:
72
- t2 = t.to_dict()
73
- t2['ranking'] = t2['distance'] # Compatibility with old clients
74
- distances.append(t2)
75
+ for t in session.execute(stmt).scalars().all():
76
+ t2 = t.to_dict()
77
+ t2['ranking'] = t2['distance'] # Compatibility with old clients
78
+ distances.append(t2)
75
79
  return distances
76
80
  except IntegrityError as error:
77
81
  raise exception.RucioException(error.args)
78
82
 
79
83
 
80
84
  @transactional_session
81
- def delete_distances(src_rse_id=None, dest_rse_id=None, *, session: "Session"):
85
+ def delete_distances(src_rse_id: Optional[str] = None, dest_rse_id: Optional[str] = None, *, session: "Session") -> None:
82
86
  """
83
- Delete distances with the given RSE ids.
87
+ Delete distances with the given RSE IDs.
84
88
 
85
89
  :param src_rse_id: The source RSE ID.
86
90
  :param dest_rse_id: The destination RSE ID.
87
91
  :param session: The database session to use.
88
92
  """
89
93
 
94
+ if not dest_rse_id or not src_rse_id:
95
+ return
90
96
  try:
91
- query = session.query(Distance)
92
-
93
- if src_rse_id:
94
- query = query.filter(Distance.src_rse_id == src_rse_id)
95
- if dest_rse_id:
96
- query = query.filter(Distance.dest_rse_id == dest_rse_id)
97
-
98
- query.delete()
97
+ stmt = delete(
98
+ Distance
99
+ ).where(
100
+ and_(Distance.src_rse_id == src_rse_id,
101
+ Distance.dest_rse_id == dest_rse_id)
102
+ )
103
+ session.execute(stmt)
99
104
  except IntegrityError as error:
100
105
  raise exception.RucioException(error.args)
101
106
 
102
107
 
103
108
  @transactional_session
104
- def update_distances(src_rse_id=None, dest_rse_id=None, distance=None, *, session: "Session"):
109
+ def update_distances(src_rse_id: Optional[str] = None, dest_rse_id: Optional[str] = None, distance: Optional[str] = None, *, session: "Session") -> None:
105
110
  """
106
- Update distances with the given RSE ids.
111
+ Update distances with the given RSE IDs.
107
112
 
108
113
  :param src_rse_id: The source RSE ID.
109
114
  :param dest_rse_id: The destination RSE ID.
110
115
  :param distance: The new distance to set
111
116
  :param session: The database session to use.
112
117
  """
118
+ if not dest_rse_id or not src_rse_id:
119
+ return
113
120
  try:
114
- query = session.query(Distance)
115
- if src_rse_id:
116
- query = query.filter(Distance.src_rse_id == src_rse_id)
117
- if dest_rse_id:
118
- query = query.filter(Distance.dest_rse_id == dest_rse_id)
119
- query.update({Distance.distance: distance})
121
+ stmt = update(
122
+ Distance
123
+ ).where(
124
+ and_(Distance.src_rse_id == src_rse_id,
125
+ Distance.dest_rse_id == dest_rse_id)
126
+ ).values({
127
+ Distance.distance: distance
128
+ })
129
+ session.execute(stmt)
120
130
  except IntegrityError as error:
121
131
  raise exception.RucioException(error.args)
122
132
 
123
133
 
124
134
  @read_session
125
- def list_distances(filter_={}, *, session: "Session"):
135
+ def list_distances(filter_: Optional[dict[str, Any]] = None, *, session: "Session") -> list[dict[str, Any]]:
126
136
  """
127
137
  Get distances between all the RSEs.
128
138
 
129
139
  :param filter_: dictionary to filter distances.
130
140
  :param session: The database session in use.
131
141
  """
132
- return [distance.to_dict() for distance in session.query(Distance).all()]
142
+ filter_ = filter_ or {}
143
+ stmt = select(
144
+ Distance
145
+ )
146
+ return [distance.to_dict() for distance in session.execute(stmt).scalars().all()]
133
147
 
134
148
 
135
149
  @read_session
136
- def export_distances(vo='def', *, session: "Session"):
150
+ def export_distances(vo: str = 'def', *, session: "Session") -> dict[str, Any]:
137
151
  """
138
152
  Export distances between all the RSEs using RSE ids.
139
153
  :param vo: The VO to export.
@@ -145,12 +159,21 @@ def export_distances(vo='def', *, session: "Session"):
145
159
  try:
146
160
  rse_src = aliased(RSE)
147
161
  rse_dest = aliased(RSE)
148
- query = session.query(Distance, rse_src.id, rse_dest.id)\
149
- .join(rse_src, rse_src.id == Distance.src_rse_id)\
150
- .join(rse_dest, rse_dest.id == Distance.dest_rse_id)\
151
- .filter(rse_src.vo == vo)\
152
- .filter(rse_dest.vo == vo)
153
- for result in query.all():
162
+ stmt = select(
163
+ Distance,
164
+ rse_src.id,
165
+ rse_dest.id
166
+ ).join(
167
+ rse_src,
168
+ rse_src.id == Distance.src_rse_id
169
+ ).join(
170
+ rse_dest,
171
+ rse_dest.id == Distance.dest_rse_id
172
+ ).where(
173
+ and_(rse_src.vo == vo,
174
+ rse_dest.vo == vo)
175
+ )
176
+ for result in session.execute(stmt).all():
154
177
  distance = result[0]
155
178
  src_id = result[1]
156
179
  dst_id = result[2]
rucio/core/exporter.py CHANGED
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,9 +12,10 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any
17
16
 
18
- from rucio.core import rse as rse_module, distance as distance_module
17
+ from rucio.core import distance as distance_module
18
+ from rucio.core import rse as rse_module
19
19
  from rucio.db.sqla.session import transactional_session
20
20
 
21
21
  if TYPE_CHECKING:
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
23
23
 
24
24
 
25
25
  @transactional_session
26
- def export_rses(vo='def', *, session: "Session"):
26
+ def export_rses(vo: str = 'def', *, session: "Session") -> dict[str, dict[str, Any]]:
27
27
  """
28
28
  Export RSE data.
29
29
 
@@ -39,7 +39,7 @@ def export_rses(vo='def', *, session: "Session"):
39
39
 
40
40
 
41
41
  @transactional_session
42
- def export_data(vo='def', distance=True, *, session: "Session"):
42
+ def export_data(vo: str = 'def', distance: bool = True, *, session: "Session") -> dict[str, Any]:
43
43
  """
44
44
  Export data.
45
45