rucio 32.8.6__py3-none-any.whl → 35.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of rucio might be problematic. Click here for more details.

Files changed (502) hide show
  1. rucio/__init__.py +0 -1
  2. rucio/alembicrevision.py +1 -2
  3. rucio/client/__init__.py +0 -1
  4. rucio/client/accountclient.py +45 -25
  5. rucio/client/accountlimitclient.py +37 -9
  6. rucio/client/baseclient.py +199 -154
  7. rucio/client/client.py +2 -3
  8. rucio/client/configclient.py +19 -6
  9. rucio/client/credentialclient.py +9 -4
  10. rucio/client/didclient.py +238 -63
  11. rucio/client/diracclient.py +13 -5
  12. rucio/client/downloadclient.py +162 -51
  13. rucio/client/exportclient.py +4 -4
  14. rucio/client/fileclient.py +3 -4
  15. rucio/client/importclient.py +4 -4
  16. rucio/client/lifetimeclient.py +21 -5
  17. rucio/client/lockclient.py +18 -8
  18. rucio/client/{metaclient.py → metaconventionsclient.py} +18 -15
  19. rucio/client/pingclient.py +0 -1
  20. rucio/client/replicaclient.py +15 -5
  21. rucio/client/requestclient.py +35 -19
  22. rucio/client/rseclient.py +133 -51
  23. rucio/client/ruleclient.py +29 -22
  24. rucio/client/scopeclient.py +8 -6
  25. rucio/client/subscriptionclient.py +47 -35
  26. rucio/client/touchclient.py +8 -4
  27. rucio/client/uploadclient.py +166 -82
  28. rucio/common/__init__.py +0 -1
  29. rucio/common/cache.py +4 -4
  30. rucio/common/config.py +52 -47
  31. rucio/common/constants.py +69 -2
  32. rucio/common/constraints.py +0 -1
  33. rucio/common/didtype.py +24 -22
  34. rucio/common/dumper/__init__.py +70 -41
  35. rucio/common/dumper/consistency.py +26 -22
  36. rucio/common/dumper/data_models.py +16 -23
  37. rucio/common/dumper/path_parsing.py +0 -1
  38. rucio/common/exception.py +281 -222
  39. rucio/common/extra.py +0 -1
  40. rucio/common/logging.py +54 -38
  41. rucio/common/pcache.py +122 -101
  42. rucio/common/plugins.py +153 -0
  43. rucio/common/policy.py +4 -4
  44. rucio/common/schema/__init__.py +17 -10
  45. rucio/common/schema/atlas.py +7 -5
  46. rucio/common/schema/belleii.py +7 -5
  47. rucio/common/schema/domatpc.py +7 -5
  48. rucio/common/schema/escape.py +7 -5
  49. rucio/common/schema/generic.py +8 -6
  50. rucio/common/schema/generic_multi_vo.py +7 -5
  51. rucio/common/schema/icecube.py +7 -5
  52. rucio/common/stomp_utils.py +0 -1
  53. rucio/common/stopwatch.py +0 -1
  54. rucio/common/test_rucio_server.py +2 -2
  55. rucio/common/types.py +262 -17
  56. rucio/common/utils.py +743 -451
  57. rucio/core/__init__.py +0 -1
  58. rucio/core/account.py +99 -29
  59. rucio/core/account_counter.py +89 -24
  60. rucio/core/account_limit.py +90 -24
  61. rucio/core/authentication.py +86 -29
  62. rucio/core/config.py +108 -38
  63. rucio/core/credential.py +14 -7
  64. rucio/core/did.py +680 -782
  65. rucio/core/did_meta_plugins/__init__.py +8 -6
  66. rucio/core/did_meta_plugins/did_column_meta.py +17 -12
  67. rucio/core/did_meta_plugins/did_meta_plugin_interface.py +60 -11
  68. rucio/core/did_meta_plugins/filter_engine.py +90 -50
  69. rucio/core/did_meta_plugins/json_meta.py +41 -16
  70. rucio/core/did_meta_plugins/mongo_meta.py +25 -8
  71. rucio/core/did_meta_plugins/postgres_meta.py +3 -4
  72. rucio/core/dirac.py +46 -17
  73. rucio/core/distance.py +66 -43
  74. rucio/core/exporter.py +5 -5
  75. rucio/core/heartbeat.py +181 -81
  76. rucio/core/identity.py +22 -12
  77. rucio/core/importer.py +23 -12
  78. rucio/core/lifetime_exception.py +32 -32
  79. rucio/core/lock.py +244 -142
  80. rucio/core/message.py +79 -38
  81. rucio/core/{meta.py → meta_conventions.py} +57 -44
  82. rucio/core/monitor.py +19 -13
  83. rucio/core/naming_convention.py +68 -27
  84. rucio/core/nongrid_trace.py +17 -5
  85. rucio/core/oidc.py +151 -29
  86. rucio/core/permission/__init__.py +18 -6
  87. rucio/core/permission/atlas.py +50 -35
  88. rucio/core/permission/belleii.py +6 -5
  89. rucio/core/permission/escape.py +8 -6
  90. rucio/core/permission/generic.py +82 -80
  91. rucio/core/permission/generic_multi_vo.py +9 -7
  92. rucio/core/quarantined_replica.py +91 -58
  93. rucio/core/replica.py +1303 -772
  94. rucio/core/replica_sorter.py +10 -12
  95. rucio/core/request.py +1133 -285
  96. rucio/core/rse.py +142 -102
  97. rucio/core/rse_counter.py +49 -18
  98. rucio/core/rse_expression_parser.py +6 -7
  99. rucio/core/rse_selector.py +41 -16
  100. rucio/core/rule.py +1538 -474
  101. rucio/core/rule_grouping.py +213 -68
  102. rucio/core/scope.py +50 -22
  103. rucio/core/subscription.py +92 -44
  104. rucio/core/topology.py +66 -24
  105. rucio/core/trace.py +42 -28
  106. rucio/core/transfer.py +543 -259
  107. rucio/core/vo.py +36 -18
  108. rucio/core/volatile_replica.py +59 -32
  109. rucio/daemons/__init__.py +0 -1
  110. rucio/daemons/abacus/__init__.py +0 -1
  111. rucio/daemons/abacus/account.py +29 -19
  112. rucio/daemons/abacus/collection_replica.py +21 -10
  113. rucio/daemons/abacus/rse.py +22 -12
  114. rucio/daemons/atropos/__init__.py +0 -1
  115. rucio/daemons/atropos/atropos.py +1 -2
  116. rucio/daemons/auditor/__init__.py +56 -28
  117. rucio/daemons/auditor/hdfs.py +17 -6
  118. rucio/daemons/auditor/srmdumps.py +116 -45
  119. rucio/daemons/automatix/__init__.py +0 -1
  120. rucio/daemons/automatix/automatix.py +30 -18
  121. rucio/daemons/badreplicas/__init__.py +0 -1
  122. rucio/daemons/badreplicas/minos.py +29 -18
  123. rucio/daemons/badreplicas/minos_temporary_expiration.py +5 -7
  124. rucio/daemons/badreplicas/necromancer.py +9 -13
  125. rucio/daemons/bb8/__init__.py +0 -1
  126. rucio/daemons/bb8/bb8.py +10 -13
  127. rucio/daemons/bb8/common.py +151 -154
  128. rucio/daemons/bb8/nuclei_background_rebalance.py +15 -9
  129. rucio/daemons/bb8/t2_background_rebalance.py +15 -8
  130. rucio/daemons/c3po/__init__.py +0 -1
  131. rucio/daemons/c3po/algorithms/__init__.py +0 -1
  132. rucio/daemons/c3po/algorithms/simple.py +8 -5
  133. rucio/daemons/c3po/algorithms/t2_free_space.py +10 -7
  134. rucio/daemons/c3po/algorithms/t2_free_space_only_pop.py +10 -7
  135. rucio/daemons/c3po/algorithms/t2_free_space_only_pop_with_network.py +30 -15
  136. rucio/daemons/c3po/c3po.py +81 -52
  137. rucio/daemons/c3po/collectors/__init__.py +0 -1
  138. rucio/daemons/c3po/collectors/agis.py +17 -17
  139. rucio/daemons/c3po/collectors/free_space.py +32 -13
  140. rucio/daemons/c3po/collectors/jedi_did.py +14 -5
  141. rucio/daemons/c3po/collectors/mock_did.py +11 -6
  142. rucio/daemons/c3po/collectors/network_metrics.py +12 -4
  143. rucio/daemons/c3po/collectors/workload.py +21 -19
  144. rucio/daemons/c3po/utils/__init__.py +0 -1
  145. rucio/daemons/c3po/utils/dataset_cache.py +15 -5
  146. rucio/daemons/c3po/utils/expiring_dataset_cache.py +16 -5
  147. rucio/daemons/c3po/utils/expiring_list.py +6 -7
  148. rucio/daemons/c3po/utils/popularity.py +5 -2
  149. rucio/daemons/c3po/utils/timeseries.py +25 -12
  150. rucio/daemons/cache/__init__.py +0 -1
  151. rucio/daemons/cache/consumer.py +21 -15
  152. rucio/daemons/common.py +42 -18
  153. rucio/daemons/conveyor/__init__.py +0 -1
  154. rucio/daemons/conveyor/common.py +69 -37
  155. rucio/daemons/conveyor/finisher.py +83 -46
  156. rucio/daemons/conveyor/poller.py +101 -69
  157. rucio/daemons/conveyor/preparer.py +35 -28
  158. rucio/daemons/conveyor/receiver.py +64 -21
  159. rucio/daemons/conveyor/stager.py +33 -28
  160. rucio/daemons/conveyor/submitter.py +71 -47
  161. rucio/daemons/conveyor/throttler.py +99 -35
  162. rucio/daemons/follower/__init__.py +0 -1
  163. rucio/daemons/follower/follower.py +12 -8
  164. rucio/daemons/hermes/__init__.py +0 -1
  165. rucio/daemons/hermes/hermes.py +57 -21
  166. rucio/daemons/judge/__init__.py +0 -1
  167. rucio/daemons/judge/cleaner.py +27 -17
  168. rucio/daemons/judge/evaluator.py +31 -18
  169. rucio/daemons/judge/injector.py +31 -23
  170. rucio/daemons/judge/repairer.py +28 -18
  171. rucio/daemons/oauthmanager/__init__.py +0 -1
  172. rucio/daemons/oauthmanager/oauthmanager.py +7 -8
  173. rucio/daemons/reaper/__init__.py +0 -1
  174. rucio/daemons/reaper/dark_reaper.py +15 -9
  175. rucio/daemons/reaper/reaper.py +109 -67
  176. rucio/daemons/replicarecoverer/__init__.py +0 -1
  177. rucio/daemons/replicarecoverer/suspicious_replica_recoverer.py +255 -116
  178. rucio/{api → daemons/rsedecommissioner}/__init__.py +0 -1
  179. rucio/daemons/rsedecommissioner/config.py +81 -0
  180. rucio/daemons/rsedecommissioner/profiles/__init__.py +24 -0
  181. rucio/daemons/rsedecommissioner/profiles/atlas.py +60 -0
  182. rucio/daemons/rsedecommissioner/profiles/generic.py +451 -0
  183. rucio/daemons/rsedecommissioner/profiles/types.py +92 -0
  184. rucio/daemons/rsedecommissioner/rse_decommissioner.py +280 -0
  185. rucio/daemons/storage/__init__.py +0 -1
  186. rucio/daemons/storage/consistency/__init__.py +0 -1
  187. rucio/daemons/storage/consistency/actions.py +152 -59
  188. rucio/daemons/tracer/__init__.py +0 -1
  189. rucio/daemons/tracer/kronos.py +47 -24
  190. rucio/daemons/transmogrifier/__init__.py +0 -1
  191. rucio/daemons/transmogrifier/transmogrifier.py +35 -26
  192. rucio/daemons/undertaker/__init__.py +0 -1
  193. rucio/daemons/undertaker/undertaker.py +10 -10
  194. rucio/db/__init__.py +0 -1
  195. rucio/db/sqla/__init__.py +16 -2
  196. rucio/db/sqla/constants.py +10 -1
  197. rucio/db/sqla/migrate_repo/__init__.py +0 -1
  198. rucio/db/sqla/migrate_repo/env.py +0 -1
  199. rucio/db/sqla/migrate_repo/versions/01eaf73ab656_add_new_rule_notification_state_progress.py +0 -1
  200. rucio/db/sqla/migrate_repo/versions/0437a40dbfd1_add_eol_at_in_rules.py +0 -3
  201. rucio/db/sqla/migrate_repo/versions/0f1adb7a599a_create_transfer_hops_table.py +1 -3
  202. rucio/db/sqla/migrate_repo/versions/102efcf145f4_added_stuck_at_column_to_rules.py +0 -3
  203. rucio/db/sqla/migrate_repo/versions/13d4f70c66a9_introduce_transfer_limits.py +1 -3
  204. rucio/db/sqla/migrate_repo/versions/140fef722e91_cleanup_distances_table.py +1 -3
  205. rucio/db/sqla/migrate_repo/versions/14ec5aeb64cf_add_request_external_host.py +0 -3
  206. rucio/db/sqla/migrate_repo/versions/156fb5b5a14_add_request_type_to_requests_idx.py +1 -4
  207. rucio/db/sqla/migrate_repo/versions/1677d4d803c8_split_rse_availability_into_multiple.py +0 -1
  208. rucio/db/sqla/migrate_repo/versions/16a0aca82e12_create_index_on_table_replicas_path.py +0 -2
  209. rucio/db/sqla/migrate_repo/versions/1803333ac20f_adding_provenance_and_phys_group.py +0 -1
  210. rucio/db/sqla/migrate_repo/versions/1a29d6a9504c_add_didtype_chck_to_requests.py +0 -1
  211. rucio/db/sqla/migrate_repo/versions/1a80adff031a_create_index_on_rules_hist_recent.py +0 -2
  212. rucio/db/sqla/migrate_repo/versions/1c45d9730ca6_increase_identity_length.py +0 -1
  213. rucio/db/sqla/migrate_repo/versions/1d1215494e95_add_quarantined_replicas_table.py +1 -3
  214. rucio/db/sqla/migrate_repo/versions/1d96f484df21_asynchronous_rules_and_rule_approval.py +0 -1
  215. rucio/db/sqla/migrate_repo/versions/1f46c5f240ac_add_bytes_column_to_bad_replicas.py +0 -3
  216. rucio/db/sqla/migrate_repo/versions/1fc15ab60d43_add_message_history_table.py +0 -1
  217. rucio/db/sqla/migrate_repo/versions/2190e703eb6e_move_rse_settings_to_rse_attributes.py +1 -2
  218. rucio/db/sqla/migrate_repo/versions/21d6b9dc9961_add_mismatch_scheme_state_to_requests.py +0 -1
  219. rucio/db/sqla/migrate_repo/versions/22cf51430c78_add_availability_column_to_table_rses.py +0 -3
  220. rucio/db/sqla/migrate_repo/versions/22d887e4ec0a_create_sources_table.py +1 -3
  221. rucio/db/sqla/migrate_repo/versions/25821a8a45a3_remove_unique_constraint_on_requests.py +1 -4
  222. rucio/db/sqla/migrate_repo/versions/25fc855625cf_added_unique_constraint_to_rules.py +0 -2
  223. rucio/db/sqla/migrate_repo/versions/269fee20dee9_add_repair_cnt_to_locks.py +0 -3
  224. rucio/db/sqla/migrate_repo/versions/271a46ea6244_add_ignore_availability_column_to_rules.py +0 -3
  225. rucio/db/sqla/migrate_repo/versions/277b5fbb41d3_switch_heartbeats_executable.py +1 -2
  226. rucio/db/sqla/migrate_repo/versions/27e3a68927fb_remove_replicas_tombstone_and_replicas_.py +0 -1
  227. rucio/db/sqla/migrate_repo/versions/2854cd9e168_added_rule_id_column.py +0 -1
  228. rucio/db/sqla/migrate_repo/versions/295289b5a800_processed_by_and__at_in_requests.py +0 -2
  229. rucio/db/sqla/migrate_repo/versions/2962ece31cf4_add_nbaccesses_column_in_the_did_table.py +0 -3
  230. rucio/db/sqla/migrate_repo/versions/2af3291ec4c_added_replicas_history_table.py +1 -3
  231. rucio/db/sqla/migrate_repo/versions/2b69addda658_add_columns_for_third_party_copy_read_.py +0 -2
  232. rucio/db/sqla/migrate_repo/versions/2b8e7bcb4783_add_config_table.py +1 -4
  233. rucio/db/sqla/migrate_repo/versions/2ba5229cb54c_add_submitted_at_to_requests_table.py +0 -3
  234. rucio/db/sqla/migrate_repo/versions/2cbee484dcf9_added_column_volume_to_rse_transfer_.py +1 -4
  235. rucio/db/sqla/migrate_repo/versions/2edee4a83846_add_source_to_requests_and_requests_.py +0 -1
  236. rucio/db/sqla/migrate_repo/versions/2eef46be23d4_change_tokens_pk.py +1 -3
  237. rucio/db/sqla/migrate_repo/versions/2f648fc909f3_index_in_rule_history_on_scope_name.py +0 -2
  238. rucio/db/sqla/migrate_repo/versions/3082b8cef557_add_naming_convention_table_and_closed_.py +1 -3
  239. rucio/db/sqla/migrate_repo/versions/30fa38b6434e_add_index_on_service_column_in_the_message_table.py +1 -3
  240. rucio/db/sqla/migrate_repo/versions/3152492b110b_added_staging_area_column.py +1 -2
  241. rucio/db/sqla/migrate_repo/versions/32c7d2783f7e_create_bad_replicas_table.py +1 -3
  242. rucio/db/sqla/migrate_repo/versions/3345511706b8_replicas_table_pk_definition_is_in_.py +1 -3
  243. rucio/db/sqla/migrate_repo/versions/35ef10d1e11b_change_index_on_table_requests.py +0 -2
  244. rucio/db/sqla/migrate_repo/versions/379a19b5332d_create_rse_limits_table.py +1 -3
  245. rucio/db/sqla/migrate_repo/versions/384b96aa0f60_created_rule_history_tables.py +2 -3
  246. rucio/db/sqla/migrate_repo/versions/3ac1660a1a72_extend_distance_table.py +0 -3
  247. rucio/db/sqla/migrate_repo/versions/3ad36e2268b0_create_collection_replicas_updates_table.py +1 -4
  248. rucio/db/sqla/migrate_repo/versions/3c9df354071b_extend_waiting_request_state.py +0 -1
  249. rucio/db/sqla/migrate_repo/versions/3d9813fab443_add_a_new_state_lost_in_badfilesstatus.py +0 -1
  250. rucio/db/sqla/migrate_repo/versions/40ad39ce3160_add_transferred_at_to_requests_table.py +0 -3
  251. rucio/db/sqla/migrate_repo/versions/4207be2fd914_add_notification_column_to_rules.py +0 -1
  252. rucio/db/sqla/migrate_repo/versions/42db2617c364_create_index_on_requests_external_id.py +0 -2
  253. rucio/db/sqla/migrate_repo/versions/436827b13f82_added_column_activity_to_table_requests.py +0 -3
  254. rucio/db/sqla/migrate_repo/versions/44278720f774_update_requests_typ_sta_upd_idx_index.py +0 -2
  255. rucio/db/sqla/migrate_repo/versions/45378a1e76a8_create_collection_replica_table.py +2 -4
  256. rucio/db/sqla/migrate_repo/versions/469d262be19_removing_created_at_index.py +0 -2
  257. rucio/db/sqla/migrate_repo/versions/4783c1f49cb4_create_distance_table.py +1 -3
  258. rucio/db/sqla/migrate_repo/versions/49a21b4d4357_create_index_on_table_tokens.py +1 -4
  259. rucio/db/sqla/migrate_repo/versions/4a2cbedda8b9_add_source_replica_expression_column_to_.py +0 -3
  260. rucio/db/sqla/migrate_repo/versions/4a7182d9578b_added_bytes_length_accessed_at_columns.py +0 -3
  261. rucio/db/sqla/migrate_repo/versions/4bab9edd01fc_create_index_on_requests_rule_id.py +0 -2
  262. rucio/db/sqla/migrate_repo/versions/4c3a4acfe006_new_attr_account_table.py +1 -3
  263. rucio/db/sqla/migrate_repo/versions/4cf0a2e127d4_adding_transient_metadata.py +0 -3
  264. rucio/db/sqla/migrate_repo/versions/4df2c5ddabc0_remove_temporary_dids.py +55 -0
  265. rucio/db/sqla/migrate_repo/versions/50280c53117c_add_qos_class_to_rse.py +0 -2
  266. rucio/db/sqla/migrate_repo/versions/52153819589c_add_rse_id_to_replicas_table.py +0 -2
  267. rucio/db/sqla/migrate_repo/versions/52fd9f4916fa_added_activity_to_rules.py +0 -3
  268. rucio/db/sqla/migrate_repo/versions/53b479c3cb0f_fix_did_meta_table_missing_updated_at_.py +0 -3
  269. rucio/db/sqla/migrate_repo/versions/5673b4b6e843_add_wfms_metadata_to_rule_tables.py +0 -3
  270. rucio/db/sqla/migrate_repo/versions/575767d9f89_added_source_history_table.py +1 -2
  271. rucio/db/sqla/migrate_repo/versions/58bff7008037_add_started_at_to_requests.py +0 -3
  272. rucio/db/sqla/migrate_repo/versions/58c8b78301ab_rename_callback_to_message.py +1 -3
  273. rucio/db/sqla/migrate_repo/versions/5f139f77382a_added_child_rule_id_column.py +1 -3
  274. rucio/db/sqla/migrate_repo/versions/688ef1840840_adding_did_meta_table.py +1 -2
  275. rucio/db/sqla/migrate_repo/versions/6e572a9bfbf3_add_new_split_container_column_to_rules.py +0 -3
  276. rucio/db/sqla/migrate_repo/versions/70587619328_add_comment_column_for_subscriptions.py +0 -3
  277. rucio/db/sqla/migrate_repo/versions/739064d31565_remove_history_table_pks.py +1 -2
  278. rucio/db/sqla/migrate_repo/versions/7541902bf173_add_didsfollowed_and_followevents_table.py +2 -4
  279. rucio/db/sqla/migrate_repo/versions/7ec22226cdbf_new_replica_state_for_temporary_.py +0 -1
  280. rucio/db/sqla/migrate_repo/versions/810a41685bc1_added_columns_rse_transfer_limits.py +1 -4
  281. rucio/db/sqla/migrate_repo/versions/83f991c63a93_correct_rse_expression_length.py +0 -2
  282. rucio/db/sqla/migrate_repo/versions/8523998e2e76_increase_size_of_extended_attributes_.py +0 -3
  283. rucio/db/sqla/migrate_repo/versions/8ea9122275b1_adding_missing_function_based_indices.py +1 -2
  284. rucio/db/sqla/migrate_repo/versions/90f47792bb76_add_clob_payload_to_messages.py +0 -3
  285. rucio/db/sqla/migrate_repo/versions/914b8f02df38_new_table_for_lifetime_model_exceptions.py +1 -3
  286. rucio/db/sqla/migrate_repo/versions/94a5961ddbf2_add_estimator_columns.py +0 -3
  287. rucio/db/sqla/migrate_repo/versions/9a1b149a2044_add_saml_identity_type.py +0 -1
  288. rucio/db/sqla/migrate_repo/versions/9a45bc4ea66d_add_vp_table.py +1 -2
  289. rucio/db/sqla/migrate_repo/versions/9eb936a81eb1_true_is_true.py +0 -2
  290. rucio/db/sqla/migrate_repo/versions/a08fa8de1545_transfer_stats_table.py +55 -0
  291. rucio/db/sqla/migrate_repo/versions/a118956323f8_added_vo_table_and_vo_col_to_rse.py +1 -3
  292. rucio/db/sqla/migrate_repo/versions/a193a275255c_add_status_column_in_messages.py +0 -2
  293. rucio/db/sqla/migrate_repo/versions/a5f6f6e928a7_1_7_0.py +1 -4
  294. rucio/db/sqla/migrate_repo/versions/a616581ee47_added_columns_to_table_requests.py +0 -1
  295. rucio/db/sqla/migrate_repo/versions/a6eb23955c28_state_idx_non_functional.py +0 -1
  296. rucio/db/sqla/migrate_repo/versions/a74275a1ad30_added_global_quota_table.py +1 -3
  297. rucio/db/sqla/migrate_repo/versions/a93e4e47bda_heartbeats.py +1 -4
  298. rucio/db/sqla/migrate_repo/versions/ae2a56fcc89_added_comment_column_to_rules.py +0 -1
  299. rucio/db/sqla/migrate_repo/versions/b0070f3695c8_add_deletedidmeta_table.py +57 -0
  300. rucio/db/sqla/migrate_repo/versions/b4293a99f344_added_column_identity_to_table_tokens.py +0 -3
  301. rucio/db/sqla/migrate_repo/versions/b5493606bbf5_fix_primary_key_for_subscription_history.py +41 -0
  302. rucio/db/sqla/migrate_repo/versions/b7d287de34fd_removal_of_replicastate_source.py +1 -2
  303. rucio/db/sqla/migrate_repo/versions/b818052fa670_add_index_to_quarantined_replicas.py +1 -3
  304. rucio/db/sqla/migrate_repo/versions/b8caac94d7f0_add_comments_column_for_subscriptions_.py +0 -3
  305. rucio/db/sqla/migrate_repo/versions/b96a1c7e1cc4_new_bad_pfns_table_and_bad_replicas_.py +1 -5
  306. rucio/db/sqla/migrate_repo/versions/bb695f45c04_extend_request_state.py +1 -3
  307. rucio/db/sqla/migrate_repo/versions/bc68e9946deb_add_staging_timestamps_to_request.py +0 -3
  308. rucio/db/sqla/migrate_repo/versions/bf3baa1c1474_correct_pk_and_idx_for_history_tables.py +1 -3
  309. rucio/db/sqla/migrate_repo/versions/c0937668555f_add_qos_policy_map_table.py +1 -2
  310. rucio/db/sqla/migrate_repo/versions/c129ccdb2d5_add_lumiblocknr_to_dids.py +0 -3
  311. rucio/db/sqla/migrate_repo/versions/ccdbcd48206e_add_did_type_column_index_on_did_meta_.py +1 -4
  312. rucio/db/sqla/migrate_repo/versions/cebad904c4dd_new_payload_column_for_heartbeats.py +1 -2
  313. rucio/db/sqla/migrate_repo/versions/d1189a09c6e0_oauth2_0_and_jwt_feature_support_adding_.py +1 -4
  314. rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py +1 -3
  315. rucio/db/sqla/migrate_repo/versions/d6dceb1de2d_added_purge_column_to_rules.py +1 -4
  316. rucio/db/sqla/migrate_repo/versions/d6e2c3b2cf26_remove_third_party_copy_column_from_rse.py +0 -2
  317. rucio/db/sqla/migrate_repo/versions/d91002c5841_new_account_limits_table.py +1 -3
  318. rucio/db/sqla/migrate_repo/versions/e138c364ebd0_extending_columns_for_filter_and_.py +0 -3
  319. rucio/db/sqla/migrate_repo/versions/e59300c8b179_support_for_archive.py +1 -3
  320. rucio/db/sqla/migrate_repo/versions/f1b14a8c2ac1_postgres_use_check_constraints.py +0 -1
  321. rucio/db/sqla/migrate_repo/versions/f41ffe206f37_oracle_global_temporary_tables.py +1 -2
  322. rucio/db/sqla/migrate_repo/versions/f85a2962b021_adding_transfertool_column_to_requests_.py +1 -3
  323. rucio/db/sqla/migrate_repo/versions/fa7a7d78b602_increase_refresh_token_size.py +0 -2
  324. rucio/db/sqla/migrate_repo/versions/fb28a95fe288_add_replicas_rse_id_tombstone_idx.py +0 -1
  325. rucio/db/sqla/migrate_repo/versions/fe1a65b176c9_set_third_party_copy_read_and_write_.py +1 -2
  326. rucio/db/sqla/migrate_repo/versions/fe8ea2fa9788_added_third_party_copy_column_to_rse_.py +0 -3
  327. rucio/db/sqla/models.py +122 -216
  328. rucio/db/sqla/sautils.py +12 -5
  329. rucio/db/sqla/session.py +71 -43
  330. rucio/db/sqla/types.py +3 -4
  331. rucio/db/sqla/util.py +91 -69
  332. rucio/gateway/__init__.py +13 -0
  333. rucio/{api → gateway}/account.py +119 -46
  334. rucio/{api → gateway}/account_limit.py +12 -13
  335. rucio/{api → gateway}/authentication.py +106 -33
  336. rucio/{api → gateway}/config.py +12 -13
  337. rucio/{api → gateway}/credential.py +15 -4
  338. rucio/{api → gateway}/did.py +384 -140
  339. rucio/{api → gateway}/dirac.py +16 -6
  340. rucio/{api → gateway}/exporter.py +3 -4
  341. rucio/{api → gateway}/heartbeat.py +17 -5
  342. rucio/{api → gateway}/identity.py +63 -19
  343. rucio/{api → gateway}/importer.py +3 -4
  344. rucio/{api → gateway}/lifetime_exception.py +35 -10
  345. rucio/{api → gateway}/lock.py +34 -12
  346. rucio/{api/meta.py → gateway/meta_conventions.py} +18 -16
  347. rucio/{api → gateway}/permission.py +4 -5
  348. rucio/{api → gateway}/quarantined_replica.py +13 -4
  349. rucio/{api → gateway}/replica.py +12 -11
  350. rucio/{api → gateway}/request.py +129 -28
  351. rucio/{api → gateway}/rse.py +11 -12
  352. rucio/{api → gateway}/rule.py +117 -35
  353. rucio/{api → gateway}/scope.py +24 -14
  354. rucio/{api → gateway}/subscription.py +65 -43
  355. rucio/{api → gateway}/vo.py +17 -7
  356. rucio/rse/__init__.py +3 -4
  357. rucio/rse/protocols/__init__.py +0 -1
  358. rucio/rse/protocols/bittorrent.py +184 -0
  359. rucio/rse/protocols/cache.py +1 -2
  360. rucio/rse/protocols/dummy.py +1 -2
  361. rucio/rse/protocols/gfal.py +12 -10
  362. rucio/rse/protocols/globus.py +7 -7
  363. rucio/rse/protocols/gsiftp.py +2 -3
  364. rucio/rse/protocols/http_cache.py +1 -2
  365. rucio/rse/protocols/mock.py +1 -2
  366. rucio/rse/protocols/ngarc.py +1 -2
  367. rucio/rse/protocols/posix.py +12 -13
  368. rucio/rse/protocols/protocol.py +116 -52
  369. rucio/rse/protocols/rclone.py +6 -7
  370. rucio/rse/protocols/rfio.py +4 -5
  371. rucio/rse/protocols/srm.py +9 -10
  372. rucio/rse/protocols/ssh.py +8 -9
  373. rucio/rse/protocols/storm.py +2 -3
  374. rucio/rse/protocols/webdav.py +17 -14
  375. rucio/rse/protocols/xrootd.py +23 -17
  376. rucio/rse/rsemanager.py +19 -7
  377. rucio/tests/__init__.py +0 -1
  378. rucio/tests/common.py +43 -17
  379. rucio/tests/common_server.py +3 -3
  380. rucio/transfertool/__init__.py +0 -1
  381. rucio/transfertool/bittorrent.py +199 -0
  382. rucio/transfertool/bittorrent_driver.py +52 -0
  383. rucio/transfertool/bittorrent_driver_qbittorrent.py +133 -0
  384. rucio/transfertool/fts3.py +250 -138
  385. rucio/transfertool/fts3_plugins.py +152 -0
  386. rucio/transfertool/globus.py +9 -8
  387. rucio/transfertool/globus_library.py +1 -2
  388. rucio/transfertool/mock.py +21 -12
  389. rucio/transfertool/transfertool.py +33 -24
  390. rucio/vcsversion.py +4 -4
  391. rucio/version.py +5 -13
  392. rucio/web/__init__.py +0 -1
  393. rucio/web/rest/__init__.py +0 -1
  394. rucio/web/rest/flaskapi/__init__.py +0 -1
  395. rucio/web/rest/flaskapi/authenticated_bp.py +0 -1
  396. rucio/web/rest/flaskapi/v1/__init__.py +0 -1
  397. rucio/web/rest/flaskapi/v1/accountlimits.py +15 -13
  398. rucio/web/rest/flaskapi/v1/accounts.py +49 -48
  399. rucio/web/rest/flaskapi/v1/archives.py +12 -10
  400. rucio/web/rest/flaskapi/v1/auth.py +146 -144
  401. rucio/web/rest/flaskapi/v1/common.py +82 -41
  402. rucio/web/rest/flaskapi/v1/config.py +5 -6
  403. rucio/web/rest/flaskapi/v1/credentials.py +7 -8
  404. rucio/web/rest/flaskapi/v1/dids.py +158 -28
  405. rucio/web/rest/flaskapi/v1/dirac.py +8 -8
  406. rucio/web/rest/flaskapi/v1/export.py +3 -5
  407. rucio/web/rest/flaskapi/v1/heartbeats.py +3 -5
  408. rucio/web/rest/flaskapi/v1/identities.py +3 -5
  409. rucio/web/rest/flaskapi/v1/import.py +3 -4
  410. rucio/web/rest/flaskapi/v1/lifetime_exceptions.py +6 -9
  411. rucio/web/rest/flaskapi/v1/locks.py +2 -4
  412. rucio/web/rest/flaskapi/v1/main.py +10 -2
  413. rucio/web/rest/flaskapi/v1/{meta.py → meta_conventions.py} +26 -11
  414. rucio/web/rest/flaskapi/v1/metrics.py +1 -2
  415. rucio/web/rest/flaskapi/v1/nongrid_traces.py +4 -4
  416. rucio/web/rest/flaskapi/v1/ping.py +6 -7
  417. rucio/web/rest/flaskapi/v1/redirect.py +8 -9
  418. rucio/web/rest/flaskapi/v1/replicas.py +43 -19
  419. rucio/web/rest/flaskapi/v1/requests.py +178 -21
  420. rucio/web/rest/flaskapi/v1/rses.py +61 -26
  421. rucio/web/rest/flaskapi/v1/rules.py +48 -18
  422. rucio/web/rest/flaskapi/v1/scopes.py +3 -5
  423. rucio/web/rest/flaskapi/v1/subscriptions.py +22 -18
  424. rucio/web/rest/flaskapi/v1/traces.py +4 -4
  425. rucio/web/rest/flaskapi/v1/types.py +20 -0
  426. rucio/web/rest/flaskapi/v1/vos.py +3 -5
  427. rucio/web/rest/main.py +0 -1
  428. rucio/web/rest/metrics.py +0 -1
  429. rucio/web/rest/ping.py +27 -0
  430. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/ldap.cfg.template +1 -1
  431. rucio-35.8.0.data/data/rucio/requirements.server.txt +268 -0
  432. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/bootstrap.py +3 -3
  433. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/merge_rucio_configs.py +2 -5
  434. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/tools/reset_database.py +3 -3
  435. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio +87 -85
  436. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-account +0 -1
  437. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-collection-replica +0 -1
  438. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-abacus-rse +0 -1
  439. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-admin +45 -32
  440. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-atropos +0 -1
  441. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-auditor +13 -7
  442. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-automatix +1 -2
  443. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-bb8 +0 -1
  444. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-c3po +0 -1
  445. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-client +2 -3
  446. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-cache-consumer +0 -1
  447. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-finisher +1 -2
  448. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-poller +0 -1
  449. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-preparer +0 -1
  450. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-receiver +0 -1
  451. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-stager +0 -1
  452. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-submitter +2 -3
  453. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-conveyor-throttler +0 -1
  454. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dark-reaper +0 -1
  455. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-dumper +11 -10
  456. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-follower +0 -1
  457. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-hermes +0 -1
  458. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-cleaner +0 -1
  459. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-evaluator +2 -3
  460. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-injector +0 -1
  461. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-judge-repairer +0 -1
  462. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-kronos +1 -3
  463. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos +0 -1
  464. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-minos-temporary-expiration +0 -1
  465. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-necromancer +1 -2
  466. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-oauth-manager +2 -3
  467. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-reaper +0 -1
  468. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-replica-recoverer +6 -7
  469. rucio-35.8.0.data/scripts/rucio-rse-decommissioner +66 -0
  470. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-storage-consistency-actions +0 -1
  471. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-transmogrifier +0 -1
  472. {rucio-32.8.6.data → rucio-35.8.0.data}/scripts/rucio-undertaker +1 -2
  473. rucio-35.8.0.dist-info/METADATA +72 -0
  474. rucio-35.8.0.dist-info/RECORD +493 -0
  475. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/WHEEL +1 -1
  476. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/AUTHORS.rst +3 -0
  477. rucio/api/temporary_did.py +0 -49
  478. rucio/common/schema/cms.py +0 -478
  479. rucio/common/schema/lsst.py +0 -423
  480. rucio/core/permission/cms.py +0 -1166
  481. rucio/core/temporary_did.py +0 -188
  482. rucio/daemons/reaper/light_reaper.py +0 -255
  483. rucio/web/rest/flaskapi/v1/tmp_dids.py +0 -115
  484. rucio-32.8.6.data/data/rucio/requirements.txt +0 -55
  485. rucio-32.8.6.data/scripts/rucio-light-reaper +0 -53
  486. rucio-32.8.6.dist-info/METADATA +0 -83
  487. rucio-32.8.6.dist-info/RECORD +0 -481
  488. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic.ini.template +0 -0
  489. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/alembic_offline.ini.template +0 -0
  490. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/globus-config.yml.template +0 -0
  491. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approval_request.tmpl +0 -0
  492. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_admin.tmpl +0 -0
  493. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_approved_user.tmpl +0 -0
  494. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_admin.tmpl +0 -0
  495. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_denied_user.tmpl +0 -0
  496. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/mail_templates/rule_ok_notification.tmpl +0 -0
  497. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rse-accounts.cfg.template +0 -0
  498. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.atlas.client.template +0 -0
  499. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio.cfg.template +0 -0
  500. {rucio-32.8.6.data → rucio-35.8.0.data}/data/rucio/etc/rucio_multi_vo.cfg.template +0 -0
  501. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/licenses/LICENSE +0 -0
  502. {rucio-32.8.6.dist-info → rucio-35.8.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,22 +12,32 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any, Optional
17
16
 
18
- from rucio.api.permission import has_permission
19
- from rucio.api.scope import list_scopes
20
17
  from rucio.common.exception import AccessDenied
21
18
  from rucio.common.utils import extract_scope
22
19
  from rucio.core import dirac
23
20
  from rucio.core.rse import get_rse_id
24
21
  from rucio.db.sqla.session import transactional_session
22
+ from rucio.gateway.permission import has_permission
23
+ from rucio.gateway.scope import list_scopes
25
24
 
26
25
  if TYPE_CHECKING:
26
+ from collections.abc import Iterable
27
+
27
28
  from sqlalchemy.orm import Session
28
29
 
29
30
 
30
31
  @transactional_session
31
- def add_files(lfns, issuer, ignore_availability, vo='def', *, session: "Session"):
32
+ def add_files(
33
+ lfns: "Iterable[dict[str, Any]]",
34
+ issuer: str,
35
+ ignore_availability: bool,
36
+ parents_metadata: Optional[dict[str, Any]] = None,
37
+ vo: str = 'def',
38
+ *,
39
+ session: "Session"
40
+ ) -> None:
32
41
  """
33
42
  Bulk add files :
34
43
  - Create the file and replica.
@@ -38,6 +47,7 @@ def add_files(lfns, issuer, ignore_availability, vo='def', *, session: "Session"
38
47
  :param lfns: List of lfn (dictionary {'lfn': <lfn>, 'rse': <rse>, 'bytes': <bytes>, 'adler32': <adler32>, 'guid': <guid>, 'pfn': <pfn>}
39
48
  :param issuer: The issuer account.
40
49
  :param ignore_availability: A boolean to ignore blocked sites.
50
+ :param parents_metadata: Metadata for selected hierarchy DIDs. (dictionary {'lpn': {key : value}}). Default=None
41
51
  :param vo: The VO to act on.
42
52
  :param session: The database session in use.
43
53
 
@@ -68,4 +78,4 @@ def add_files(lfns, issuer, ignore_availability, vo='def', *, session: "Session"
68
78
  if not has_permission(issuer=issuer, action='add_dids', kwargs=kwargs, vo=vo, session=session):
69
79
  raise AccessDenied('Account %s can not bulk add data identifier for VO %s' % (issuer, vo))
70
80
 
71
- dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, vo=vo, session=session)
81
+ dirac.add_files(lfns=lfns, account=issuer, ignore_availability=ignore_availability, parents_metadata=parents_metadata, vo=vo, session=session)
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,20 +12,20 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any
17
16
 
18
- from rucio.api import permission
19
17
  from rucio.common import exception
20
18
  from rucio.core import exporter
21
19
  from rucio.core.rse import get_rse_name
22
20
  from rucio.db.sqla.session import read_session
21
+ from rucio.gateway import permission
23
22
 
24
23
  if TYPE_CHECKING:
25
24
  from sqlalchemy.orm import Session
26
25
 
27
26
 
28
27
  @read_session
29
- def export_data(issuer, distance=True, vo='def', *, session: "Session"):
28
+ def export_data(issuer: str, distance: bool = True, vo: str = 'def', *, session: "Session") -> dict[str, Any]:
30
29
  """
31
30
  Export data from Rucio.
32
31
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,19 +12,21 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Optional
17
16
 
18
- from rucio.api import permission
19
17
  from rucio.common import exception
20
18
  from rucio.core import heartbeat
21
19
  from rucio.db.sqla.session import read_session, transactional_session
20
+ from rucio.gateway import permission
22
21
 
23
22
  if TYPE_CHECKING:
23
+ from threading import Thread
24
+
24
25
  from sqlalchemy.orm import Session
25
26
 
26
27
 
27
28
  @read_session
28
- def list_heartbeats(issuer=None, vo='def', *, session: "Session"):
29
+ def list_heartbeats(issuer: Optional[str] = None, vo: str = 'def', *, session: "Session") -> list["heartbeat.HeartbeatDict"]:
29
30
  """
30
31
  Return a list of tuples of all heartbeats.
31
32
 
@@ -42,7 +43,18 @@ def list_heartbeats(issuer=None, vo='def', *, session: "Session"):
42
43
 
43
44
 
44
45
  @transactional_session
45
- def create_heartbeat(executable, hostname, pid, older_than, payload, thread=None, issuer=None, vo='def', *, session: "Session"):
46
+ def create_heartbeat(
47
+ executable: str,
48
+ hostname: str,
49
+ pid: int,
50
+ older_than: int,
51
+ payload: Optional[str],
52
+ thread: Optional["Thread"] = None,
53
+ issuer: Optional[str] = None,
54
+ vo: str = 'def',
55
+ *,
56
+ session: "Session"
57
+ ) -> None:
46
58
  """
47
59
  Creates a heartbeat.
48
60
  :param issuer: The issuer account.
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,22 +16,31 @@
17
16
  Interface for identity abstraction layer
18
17
  """
19
18
 
20
- from typing import TYPE_CHECKING
19
+ from typing import TYPE_CHECKING, Optional
21
20
 
22
- from rucio.api import permission
23
21
  from rucio.common import exception
24
22
  from rucio.common.types import InternalAccount
25
23
  from rucio.core import identity
26
24
  from rucio.db.sqla.constants import IdentityType
27
25
  from rucio.db.sqla.session import read_session, transactional_session
26
+ from rucio.gateway import permission
28
27
 
29
28
  if TYPE_CHECKING:
30
- from typing import Optional
29
+ from collections.abc import Sequence
30
+
31
+ from sqlalchemy import Row
31
32
  from sqlalchemy.orm import Session
32
33
 
33
34
 
34
35
  @transactional_session
35
- def add_identity(identity_key, id_type, email, password=None, *, session: "Session"):
36
+ def add_identity(
37
+ identity_key: str,
38
+ id_type: str,
39
+ email: str,
40
+ password: Optional[str] = None,
41
+ *,
42
+ session: "Session"
43
+ ) -> None:
36
44
  """
37
45
  Creates a user identity.
38
46
 
@@ -46,7 +54,14 @@ def add_identity(identity_key, id_type, email, password=None, *, session: "Sessi
46
54
 
47
55
 
48
56
  @transactional_session
49
- def del_identity(identity_key, id_type, issuer, vo='def', *, session: "Session"):
57
+ def del_identity(
58
+ identity_key: str,
59
+ id_type: str,
60
+ issuer: str,
61
+ vo: str = 'def',
62
+ *,
63
+ session: "Session"
64
+ ) -> None:
50
65
  """
51
66
  Deletes a user identity.
52
67
  :param identity_key: The identity key name. For example x509 DN, or a username.
@@ -55,16 +70,27 @@ def del_identity(identity_key, id_type, issuer, vo='def', *, session: "Session")
55
70
  :param vo: the VO of the issuer.
56
71
  :param session: The database session in use.
57
72
  """
58
- id_type = IdentityType[id_type.upper()]
59
- kwargs = {'accounts': identity.list_accounts_for_identity(identity_key, id_type, session=session)}
73
+ converted_id_type = IdentityType[id_type.upper()]
74
+ kwargs = {'accounts': identity.list_accounts_for_identity(identity_key, converted_id_type, session=session)}
60
75
  if not permission.has_permission(issuer=issuer, vo=vo, action='del_identity', kwargs=kwargs, session=session):
61
76
  raise exception.AccessDenied('Account %s can not delete identity' % (issuer))
62
77
 
63
- return identity.del_identity(identity_key, id_type, session=session)
78
+ return identity.del_identity(identity_key, converted_id_type, session=session)
64
79
 
65
80
 
66
81
  @transactional_session
67
- def add_account_identity(identity_key, id_type, account, email, issuer, default=False, password=None, vo='def', *, session: "Session"):
82
+ def add_account_identity(
83
+ identity_key: str,
84
+ id_type: str,
85
+ account: str,
86
+ email: str,
87
+ issuer: str,
88
+ default: bool = False,
89
+ password: Optional[str] = None,
90
+ vo: str = 'def',
91
+ *,
92
+ session: "Session"
93
+ ) -> None:
68
94
  """
69
95
  Adds a membership association between identity and account.
70
96
 
@@ -82,14 +108,14 @@ def add_account_identity(identity_key, id_type, account, email, issuer, default=
82
108
  if not permission.has_permission(issuer=issuer, vo=vo, action='add_account_identity', kwargs=kwargs, session=session):
83
109
  raise exception.AccessDenied('Account %s can not add account identity' % (issuer))
84
110
 
85
- account = InternalAccount(account, vo=vo)
111
+ internal_account = InternalAccount(account, vo=vo)
86
112
 
87
113
  return identity.add_account_identity(identity=identity_key, type_=IdentityType[id_type.upper()], default=default,
88
- email=email, account=account, password=password, session=session)
114
+ email=email, account=internal_account, password=password, session=session)
89
115
 
90
116
 
91
117
  @read_session
92
- def verify_identity(identity_key: str, id_type: str, password: "Optional[str]" = None, *, session: "Session") -> bool:
118
+ def verify_identity(identity_key: str, id_type: str, password: Optional[str] = None, *, session: "Session") -> bool:
93
119
  """
94
120
  Verifies a user identity.
95
121
  :param identity_key: The identity key name. For example x509 DN, or a username.
@@ -101,7 +127,15 @@ def verify_identity(identity_key: str, id_type: str, password: "Optional[str]" =
101
127
 
102
128
 
103
129
  @transactional_session
104
- def del_account_identity(identity_key, id_type, account, issuer, vo='def', *, session: "Session"):
130
+ def del_account_identity(
131
+ identity_key: str,
132
+ id_type: str,
133
+ account: str,
134
+ issuer: str,
135
+ vo: str = 'def',
136
+ *,
137
+ session: "Session"
138
+ ) -> None:
105
139
  """
106
140
  Removes a membership association between identity and account.
107
141
 
@@ -116,13 +150,13 @@ def del_account_identity(identity_key, id_type, account, issuer, vo='def', *, se
116
150
  if not permission.has_permission(issuer=issuer, vo=vo, action='del_account_identity', kwargs=kwargs, session=session):
117
151
  raise exception.AccessDenied('Account %s can not delete account identity' % (issuer))
118
152
 
119
- account = InternalAccount(account, vo=vo)
153
+ internal_account = InternalAccount(account, vo=vo)
120
154
 
121
- return identity.del_account_identity(identity_key, IdentityType[id_type.upper()], account, session=session)
155
+ return identity.del_account_identity(identity_key, IdentityType[id_type.upper()], internal_account, session=session)
122
156
 
123
157
 
124
158
  @read_session
125
- def list_identities(*, session: "Session", **kwargs):
159
+ def list_identities(*, session: "Session", **kwargs) -> "Sequence[Row[tuple[str, IdentityType]]]":
126
160
  """
127
161
  Returns a list of all enabled identities.
128
162
 
@@ -133,7 +167,12 @@ def list_identities(*, session: "Session", **kwargs):
133
167
 
134
168
 
135
169
  @read_session
136
- def get_default_account(identity_key, id_type, *, session: "Session"):
170
+ def get_default_account(
171
+ identity_key: str,
172
+ id_type: str,
173
+ *,
174
+ session: "Session"
175
+ ) -> str:
137
176
  """
138
177
  Returns the default account for this identity.
139
178
 
@@ -146,7 +185,12 @@ def get_default_account(identity_key, id_type, *, session: "Session"):
146
185
 
147
186
 
148
187
  @read_session
149
- def list_accounts_for_identity(identity_key, id_type, *, session: "Session"):
188
+ def list_accounts_for_identity(
189
+ identity_key: str,
190
+ id_type: str,
191
+ *,
192
+ session: "Session"
193
+ ) -> list[str]:
150
194
  """
151
195
  Returns a list of all accounts for an identity.
152
196
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,21 +12,21 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any
17
16
 
18
- from rucio.api import permission
19
17
  from rucio.common import exception
20
18
  from rucio.common.schema import validate_schema
21
19
  from rucio.common.types import InternalAccount
22
20
  from rucio.core import importer
23
21
  from rucio.db.sqla.session import transactional_session
22
+ from rucio.gateway import permission
24
23
 
25
24
  if TYPE_CHECKING:
26
25
  from sqlalchemy.orm import Session
27
26
 
28
27
 
29
28
  @transactional_session
30
- def import_data(data, issuer, vo='def', *, session: "Session"):
29
+ def import_data(data: dict[str, Any], issuer: str, vo: str = 'def', *, session: "Session") -> None:
31
30
  """
32
31
  Import data to add/update/delete records in Rucio.
33
32
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,21 +12,31 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Any, Optional
17
16
 
18
- from rucio.api import permission
19
17
  from rucio.common import exception
20
18
  from rucio.common.types import InternalAccount, InternalScope
21
- from rucio.common.utils import api_update_return_dict
19
+ from rucio.common.utils import gateway_update_return_dict
22
20
  from rucio.core import lifetime_exception
23
21
  from rucio.db.sqla.session import stream_session, transactional_session
22
+ from rucio.gateway import permission
24
23
 
25
24
  if TYPE_CHECKING:
25
+ from collections.abc import Iterable, Iterator
26
+
26
27
  from sqlalchemy.orm import Session
27
28
 
29
+ from rucio.db.sqla.constants import LifetimeExceptionsState
30
+
28
31
 
29
32
  @stream_session
30
- def list_exceptions(exception_id=None, states=None, vo='def', *, session: "Session"):
33
+ def list_exceptions(
34
+ exception_id: Optional[str] = None,
35
+ states: Optional["Iterable[LifetimeExceptionsState]"] = None,
36
+ vo: str = 'def',
37
+ *,
38
+ session: "Session"
39
+ ) -> 'Iterator[dict[str, Any]]':
31
40
  """
32
41
  List exceptions to Lifetime Model.
33
42
 
@@ -40,11 +49,20 @@ def list_exceptions(exception_id=None, states=None, vo='def', *, session: "Sessi
40
49
  exceptions = lifetime_exception.list_exceptions(exception_id=exception_id, states=states, session=session)
41
50
  for e in exceptions:
42
51
  if vo == e['scope'].vo:
43
- yield api_update_return_dict(e, session=session)
52
+ yield gateway_update_return_dict(e, session=session)
44
53
 
45
54
 
46
55
  @transactional_session
47
- def add_exception(dids, account, pattern, comments, expires_at, vo='def', *, session: "Session"):
56
+ def add_exception(
57
+ dids: "Iterable[dict[str, Any]]",
58
+ account: str,
59
+ pattern: str,
60
+ comments: str,
61
+ expires_at: str,
62
+ vo: str = 'def',
63
+ *,
64
+ session: "Session"
65
+ ) -> dict[str, Any]:
48
66
  """
49
67
  Add exceptions to Lifetime Model.
50
68
 
@@ -59,10 +77,10 @@ def add_exception(dids, account, pattern, comments, expires_at, vo='def', *, ses
59
77
  returns: The id of the exception.
60
78
  """
61
79
 
62
- account = InternalAccount(account, vo=vo)
80
+ internal_account = InternalAccount(account, vo=vo)
63
81
  for did in dids:
64
82
  did['scope'] = InternalScope(did['scope'], vo=vo)
65
- exceptions = lifetime_exception.add_exception(dids=dids, account=account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
83
+ exceptions = lifetime_exception.add_exception(dids=dids, account=internal_account, pattern=pattern, comments=comments, expires_at=expires_at, session=session)
66
84
 
67
85
  for key in exceptions:
68
86
  if key == 'exceptions':
@@ -79,7 +97,14 @@ def add_exception(dids, account, pattern, comments, expires_at, vo='def', *, ses
79
97
 
80
98
 
81
99
  @transactional_session
82
- def update_exception(exception_id, state, issuer, vo='def', *, session: "Session"):
100
+ def update_exception(
101
+ exception_id: str,
102
+ state: 'LifetimeExceptionsState',
103
+ issuer: str,
104
+ vo: str = 'def',
105
+ *,
106
+ session: "Session"
107
+ ) -> None:
83
108
  """
84
109
  Update exceptions state to Lifetime Model.
85
110
 
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,16 +13,18 @@
14
13
  # limitations under the License.
15
14
 
16
15
  import logging
17
- from typing import TYPE_CHECKING
16
+ from typing import TYPE_CHECKING, Any
18
17
 
19
18
  from rucio.common.types import InternalScope
20
- from rucio.common.utils import api_update_return_dict
19
+ from rucio.common.utils import gateway_update_return_dict
21
20
  from rucio.core import lock
22
21
  from rucio.core.rse import get_rse_id
23
22
  from rucio.db.sqla.constants import DIDType
24
23
  from rucio.db.sqla.session import stream_session
25
24
 
26
25
  if TYPE_CHECKING:
26
+ from collections.abc import Iterable, Iterator
27
+
27
28
  from sqlalchemy.orm import Session
28
29
 
29
30
 
@@ -32,7 +33,13 @@ LOGGER.setLevel(logging.DEBUG)
32
33
 
33
34
 
34
35
  @stream_session
35
- def get_dataset_locks(scope, name, vo='def', *, session: "Session"):
36
+ def get_dataset_locks(
37
+ scope: str,
38
+ name: str,
39
+ vo: str = 'def',
40
+ *,
41
+ session: "Session"
42
+ ) -> 'Iterator[dict[str, Any]]':
36
43
  """
37
44
  Get the dataset locks of a dataset.
38
45
 
@@ -43,16 +50,21 @@ def get_dataset_locks(scope, name, vo='def', *, session: "Session"):
43
50
  :return: List of dicts {'rse_id': ..., 'state': ...}
44
51
  """
45
52
 
46
- scope = InternalScope(scope, vo=vo)
53
+ internal_scope = InternalScope(scope, vo=vo)
47
54
 
48
- locks = lock.get_dataset_locks(scope=scope, name=name, session=session)
55
+ locks = lock.get_dataset_locks(scope=internal_scope, name=name, session=session)
49
56
 
50
57
  for lock_object in locks:
51
- yield api_update_return_dict(lock_object, session=session)
58
+ yield gateway_update_return_dict(lock_object, session=session)
52
59
 
53
60
 
54
61
  @stream_session
55
- def get_dataset_locks_bulk(dids, vo='def', *, session: "Session"):
62
+ def get_dataset_locks_bulk(
63
+ dids: 'Iterable[dict[str, Any]]',
64
+ vo: str = 'def',
65
+ *,
66
+ session: "Session"
67
+ ) -> 'Iterator[dict[str, Any]]':
56
68
  """
57
69
  Get the dataset locks for multiple datasets or containers.
58
70
 
@@ -94,7 +106,12 @@ def get_dataset_locks_bulk(dids, vo='def', *, session: "Session"):
94
106
 
95
107
 
96
108
  @stream_session
97
- def get_dataset_locks_by_rse(rse, vo='def', *, session: "Session"):
109
+ def get_dataset_locks_by_rse(
110
+ rse: str,
111
+ vo: str = 'def',
112
+ *,
113
+ session: "Session"
114
+ ) -> 'Iterator[dict[str, Any]]':
98
115
  """
99
116
  Get the dataset locks of an RSE.
100
117
 
@@ -108,11 +125,16 @@ def get_dataset_locks_by_rse(rse, vo='def', *, session: "Session"):
108
125
  locks = lock.get_dataset_locks_by_rse_id(rse_id=rse_id, session=session)
109
126
 
110
127
  for lock_object in locks:
111
- yield api_update_return_dict(lock_object, session=session)
128
+ yield gateway_update_return_dict(lock_object, session=session)
112
129
 
113
130
 
114
131
  @stream_session
115
- def get_replica_locks_for_rule_id(rule_id, vo='def', *, session: "Session"):
132
+ def get_replica_locks_for_rule_id(
133
+ rule_id: str,
134
+ vo: str = 'def',
135
+ *,
136
+ session: "Session"
137
+ ) -> 'Iterator[dict[str, Any]]':
116
138
  """
117
139
  Get the replica locks for a rule_id.
118
140
 
@@ -128,4 +150,4 @@ def get_replica_locks_for_rule_id(rule_id, vo='def', *, session: "Session"):
128
150
  if lock_object['scope'].vo != vo: # rule is on a different VO, so don't return any locks
129
151
  LOGGER.debug('rule id %s is not present on VO %s' % (rule_id, vo))
130
152
  break
131
- yield api_update_return_dict(lock_object, session=session)
153
+ yield gateway_update_return_dict(lock_object, session=session)
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,33 +12,36 @@
13
12
  # See the License for the specific language governing permissions and
14
13
  # limitations under the License.
15
14
 
16
- from typing import TYPE_CHECKING
15
+ from typing import TYPE_CHECKING, Optional, Union
17
16
 
18
- from rucio.api.permission import has_permission
19
17
  from rucio.common.exception import AccessDenied
20
- from rucio.core import meta
18
+ from rucio.core import meta_conventions
19
+ from rucio.db.sqla.constants import KeyType
21
20
  from rucio.db.sqla.session import read_session, transactional_session
21
+ from rucio.gateway.permission import has_permission
22
22
 
23
23
  if TYPE_CHECKING:
24
24
  from sqlalchemy.orm import Session
25
25
 
26
+ from rucio.common.types import InternalAccount
27
+
26
28
 
27
29
  @read_session
28
- def list_keys(*, session: "Session"):
30
+ def list_keys(*, session: "Session") -> list[str]:
29
31
  """
30
- Lists all keys.
32
+ Lists all keys for DID Metadata Conventions.
31
33
 
32
34
  :param session: The database session in use.
33
35
 
34
36
  :returns: A list containing all keys.
35
37
  """
36
- return meta.list_keys(session=session)
38
+ return meta_conventions.list_keys(session=session)
37
39
 
38
40
 
39
41
  @read_session
40
- def list_values(key, *, session: "Session"):
42
+ def list_values(key: str, *, session: "Session") -> list[str]:
41
43
  """
42
- Lists all values for a key.
44
+ Lists all allowed values for a DID key (all values for a key in DID Metadata Conventions).
43
45
 
44
46
  :param key: the name for the key.
45
47
  :param session: The database session in use.
@@ -47,13 +49,13 @@ def list_values(key, *, session: "Session"):
47
49
 
48
50
  :returns: A list containing all values.
49
51
  """
50
- return meta.list_values(key=key, session=session)
52
+ return meta_conventions.list_values(key=key, session=session)
51
53
 
52
54
 
53
55
  @transactional_session
54
- def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def', *, session: "Session"):
56
+ def add_key(key: str, key_type: Union[KeyType, str], issuer: "InternalAccount", value_type: Optional[str] = None, value_regexp: Optional[str] = None, vo: str = 'def', *, session: "Session") -> None:
55
57
  """
56
- Add a new allowed key.
58
+ Add an allowed key for DID metadata (update the DID Metadata Conventions table with a new key).
57
59
 
58
60
  :param key: the name for the new key.
59
61
  :param key_type: the type of the key: all(container, dataset, file), collection(dataset or container), file, derived(compute from file for collection).
@@ -66,13 +68,13 @@ def add_key(key, key_type, issuer, value_type=None, value_regexp=None, vo='def',
66
68
  kwargs = {'key': key, 'key_type': key_type, 'value_type': value_type, 'value_regexp': value_regexp}
67
69
  if not has_permission(issuer=issuer, vo=vo, action='add_key', kwargs=kwargs, session=session):
68
70
  raise AccessDenied('Account %s can not add key' % (issuer))
69
- return meta.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
71
+ return meta_conventions.add_key(key=key, key_type=key_type, value_type=value_type, value_regexp=value_regexp, session=session)
70
72
 
71
73
 
72
74
  @transactional_session
73
- def add_value(key, value, issuer, vo='def', *, session: "Session"):
75
+ def add_value(key: str, value: str, issuer: "InternalAccount", vo: str = 'def', *, session: "Session") -> None:
74
76
  """
75
- Add a new value to a key.
77
+ Add an allowed value for DID metadata (update a key in DID Metadata Conventions table).
76
78
 
77
79
  :param key: the name for the key.
78
80
  :param value: the value.
@@ -82,4 +84,4 @@ def add_value(key, value, issuer, vo='def', *, session: "Session"):
82
84
  kwargs = {'key': key, 'value': value}
83
85
  if not has_permission(issuer=issuer, vo=vo, action='add_value', kwargs=kwargs, session=session):
84
86
  raise AccessDenied('Account %s can not add value %s to key %s' % (issuer, value, key))
85
- return meta.add_value(key=key, value=value, session=session)
87
+ return meta_conventions.add_value(key=key, value=value, session=session)
@@ -1,4 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
1
  # Copyright European Organization for Nuclear Research (CERN) since 2012
3
2
  #
4
3
  # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +13,7 @@
14
13
  # limitations under the License.
15
14
 
16
15
  from copy import deepcopy
17
- from typing import TYPE_CHECKING
16
+ from typing import TYPE_CHECKING, Any
18
17
 
19
18
  from rucio.common.exception import RSENotFound
20
19
  from rucio.common.types import InternalAccount, InternalScope
@@ -27,7 +26,7 @@ if TYPE_CHECKING:
27
26
 
28
27
 
29
28
  @read_session
30
- def has_permission(issuer, action, kwargs, vo='def', *, session: "Session"):
29
+ def has_permission(issuer: str, action: str, kwargs: dict[str, Any], vo: str = 'def', *, session: "Session") -> bool:
31
30
  """
32
31
  Checks if an account has the specified permission to
33
32
  execute an action with parameters.
@@ -67,6 +66,6 @@ def has_permission(issuer, action, kwargs, vo='def', *, session: "Session"):
67
66
  for r in d['rules']:
68
67
  r['account'] = InternalAccount(r['account'], vo=vo)
69
68
 
70
- issuer = InternalAccount(issuer, vo=vo)
69
+ issuer_account = InternalAccount(issuer, vo=vo)
71
70
 
72
- return permission.has_permission(issuer=issuer, action=action, kwargs=kwargs, session=session)
71
+ return permission.has_permission(issuer=issuer_account, action=action, kwargs=kwargs, session=session)